dom/media/MediaFormatReader.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #if !defined(MediaFormatReader_h_)
   8 #  define MediaFormatReader_h_
   9
  10 #  include "FrameStatistics.h"
  11 #  include "MediaDataDemuxer.h"
  12 #  include "MediaEventSource.h"
  13 #  include "MediaMetadataManager.h"
  14 #  include "MediaPromiseDefs.h"
  15 #  include "PlatformDecoderModule.h"
  16 #  include "SeekTarget.h"
  17 #  include "mozilla/Atomics.h"
  18 #  include "mozilla/Maybe.h"
  19 #  include "mozilla/MozPromise.h"
  20 #  include "mozilla/Mutex.h"
  21 #  include "mozilla/StateMirroring.h"
  22 #  include "mozilla/StaticPrefs_media.h"
  23 #  include "mozilla/TaskQueue.h"
  24 #  include "mozilla/ThreadSafeWeakPtr.h"
  25 #  include "mozilla/dom/MediaDebugInfoBinding.h"
  26
  27 namespace mozilla {
  28
  29 class CDMProxy;
  30 class GMPCrashHelper;
  31 class MediaResource;
  32 class VideoFrameContainer;
  33
  34 struct WaitForDataRejectValue {
  35   enum Reason { SHUTDOWN, CANCELED };
  36
  37   WaitForDataRejectValue(MediaData::Type aType, Reason aReason)
  38       : mType(aType), mReason(aReason) {}
  39   MediaData::Type mType;
  40   Reason mReason;
  41 };
  42
  43 struct SeekRejectValue {
  44   MOZ_IMPLICIT SeekRejectValue(const MediaResult& aError)
  45       : mType(MediaData::Type::NULL_DATA), mError(aError) {}
  46   MOZ_IMPLICIT SeekRejectValue(nsresult aResult)
  47       : mType(MediaData::Type::NULL_DATA), mError(aResult) {}
  48   SeekRejectValue(MediaData::Type aType, const MediaResult& aError)
  49       : mType(aType), mError(aError) {}
  50   MediaData::Type mType;
  51   MediaResult mError;
  52 };
  53
  54 struct MetadataHolder {
  55   UniquePtr<MediaInfo> mInfo;
  56   UniquePtr<MetadataTags> mTags;
  57 };
  58
  59 typedef void* MediaDecoderOwnerID;
  60
  61 struct MOZ_STACK_CLASS MediaFormatReaderInit {
  62   MediaResource* mResource = nullptr;
  63   VideoFrameContainer* mVideoFrameContainer = nullptr;
  64   FrameStatistics* mFrameStats = nullptr;
  65   already_AddRefed<layers::KnowsCompositor> mKnowsCompositor;
  66   already_AddRefed<GMPCrashHelper> mCrashHelper;
  67   // Used in bug 1393399 for temporary telemetry.
  68   MediaDecoderOwnerID mMediaDecoderOwnerID = nullptr;
  69 };
  70
  71 DDLoggedTypeDeclName(MediaFormatReader);
  72
  73 class MediaFormatReader final
  74     : public SupportsThreadSafeWeakPtr<MediaFormatReader>,
  75       public DecoderDoctorLifeLogger<MediaFormatReader> {
  76   static const bool IsExclusive = true;
  77   typedef TrackInfo::TrackType TrackType;
  78   typedef MozPromise<bool, MediaResult, IsExclusive> NotifyDataArrivedPromise;
  79
  80  public:
  81   MOZ_DECLARE_REFCOUNTED_TYPENAME(MediaFormatReader)
  82
  83   using TrackSet = EnumSet<TrackInfo::TrackType>;
  84   using MetadataPromise = MozPromise<MetadataHolder, MediaResult, IsExclusive>;
  85
  86   template <typename Type>
  87   using DataPromise = MozPromise<RefPtr<Type>, MediaResult, IsExclusive>;
  88   using AudioDataPromise = DataPromise<AudioData>;
  89   using VideoDataPromise = DataPromise<VideoData>;
  90
  91   using SeekPromise = MozPromise<media::TimeUnit, SeekRejectValue, IsExclusive>;
  92
  93   // Note that, conceptually, WaitForData makes sense in a non-exclusive sense.
  94   // But in the current architecture it's only ever used exclusively (by MDSM),
  95   // so we mark it that way to verify our assumptions. If you have a use-case
  96   // for multiple WaitForData consumers, feel free to flip the exclusivity here.
  97   using WaitForDataPromise =
  98       MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>;
  99
 100   MediaFormatReader(MediaFormatReaderInit& aInit, MediaDataDemuxer* aDemuxer);
 101   virtual ~MediaFormatReader();
 102
 103   // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE
 104   // on failure.
 105   nsresult Init();
 106
 107   size_t SizeOfVideoQueueInFrames();
 108   size_t SizeOfAudioQueueInFrames();
 109
 110   // Requests one video sample from the reader.
 111   RefPtr<VideoDataPromise> RequestVideoData(
 112       const media::TimeUnit& aTimeThreshold,
 113       bool aRequestNextVideoKeyFrame = false);
 114
 115   // Requests one audio sample from the reader.
 116   //
 117   // The decode should be performed asynchronously, and the promise should
 118   // be resolved when it is complete.
 119   RefPtr<AudioDataPromise> RequestAudioData();
 120
 121   // The default implementation of AsyncReadMetadata is implemented in terms of
 122   // synchronous ReadMetadata() calls. Implementations may also
 123   // override AsyncReadMetadata to create a more proper async implementation.
 124   RefPtr<MetadataPromise> AsyncReadMetadata();
 125
 126   // Fills aInfo with the latest cached data required to present the media,
 127   // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded.
 128   void ReadUpdatedMetadata(MediaInfo* aInfo);
 129
 130   RefPtr<SeekPromise> Seek(const SeekTarget& aTarget);
 131
 132   // Called once new data has been cached by the MediaResource.
 133   // mBuffered should be recalculated and updated accordingly.
 134   void NotifyDataArrived();
 135
 136   // Update ID for the external playback engine. Currently it's only used on
 137   // Windows when the media engine playback is enabled.
 138   void UpdateMediaEngineId(uint64_t aMediaEngineId);
 139
 140  protected:
 141   // Recomputes mBuffered.
 142   void UpdateBuffered();
 143
 144  public:
 145   // Called by MDSM in dormant state to release resources allocated by this
 146   // reader. The reader can resume decoding by calling Seek() to a specific
 147   // position.
 148   void ReleaseResources();
 149
 150   bool OnTaskQueue() const { return OwnerThread()->IsCurrentThreadIn(); }
 151
 152   // Resets all state related to decoding, emptying all buffers etc.
 153   // Cancels all pending Request*Data() request callbacks, rejects any
 154   // outstanding seek promises, and flushes the decode pipeline. The
 155   // decoder must not call any of the callbacks for outstanding
 156   // Request*Data() calls after this is called. Calls to Request*Data()
 157   // made after this should be processed as usual.
 158   //
 159   // Normally this call preceedes a Seek() call, or shutdown.
 160   //
 161   // aParam is a set of TrackInfo::TrackType enums specifying which
 162   // queues need to be reset, defaulting to both audio and video tracks.
 163   nsresult ResetDecode(TrackSet aTracks);
 164
 165   // Destroys the decoding state. The reader cannot be made usable again.
 166   // This is different from ReleaseMediaResources() as it is irreversable,
 167   // whereas ReleaseMediaResources() is.  Must be called on the decode
 168   // thread.
 169   RefPtr<ShutdownPromise> Shutdown();
 170
 171   // Returns true if this decoder reader uses hardware accelerated video
 172   // decoding.
 173   bool VideoIsHardwareAccelerated() const;
 174
 175   // By default, the state machine polls the reader once per second when it's
 176   // in buffering mode. Some readers support a promise-based mechanism by which
 177   // they notify the state machine when the data arrives.
 178   bool IsWaitForDataSupported() const { return true; }
 179
 180   RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType);
 181
 182   // The MediaDecoderStateMachine uses various heuristics that assume that
 183   // raw media data is arriving sequentially from a network channel. This
 184   // makes sense in the <video src="foo"> case, but not for more advanced use
 185   // cases like MSE.
 186   bool UseBufferingHeuristics() const { return mTrackDemuxersMayBlock; }
 187
 188   RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy);
 189
 190   // Requests that the MediaFormatReader populates aInfo with debug information.
 191   // This may be done asynchronously, and aInfo should *not* be accessed by the
 192   // caller until the returned promise is resolved or rejected.
 193   RefPtr<GenericPromise> RequestDebugInfo(
 194       dom::MediaFormatReaderDebugInfo& aInfo);
 195
 196   // Switch the video decoder to NullDecoderModule. It might takes effective
 197   // since a few samples later depends on how much demuxed samples are already
 198   // queued in the original video decoder.
 199   void SetVideoNullDecode(bool aIsNullDecode);
 200
 201   void UpdateCompositor(already_AddRefed<layers::KnowsCompositor>);
 202
 203   void UpdateDuration(const media::TimeUnit& aDuration) {
 204     MOZ_ASSERT(OnTaskQueue());
 205     UpdateBuffered();
 206   }
 207
 208   AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() {
 209     return &mBuffered;
 210   }
 211
 212   TaskQueue* OwnerThread() const { return mTaskQueue; }
 213
 214   TimedMetadataEventSource& TimedMetadataEvent() { return mTimedMetadataEvent; }
 215
 216   // Notified by the OggDemuxer during playback when chained ogg is detected.
 217   MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; }
 218
 219   TimedMetadataEventProducer& TimedMetadataProducer() {
 220     return mTimedMetadataEvent;
 221   }
 222
 223   MediaEventProducer<void>& MediaNotSeekableProducer() {
 224     return mOnMediaNotSeekable;
 225   }
 226
 227   // Notified if the reader can't decode a sample due to a missing decryption
 228   // key.
 229   MediaEventSource<TrackInfo::TrackType>& OnTrackWaitingForKey() {
 230     return mOnTrackWaitingForKey;
 231   }
 232
 233   MediaEventProducer<TrackInfo::TrackType>& OnTrackWaitingForKeyProducer() {
 234     return mOnTrackWaitingForKey;
 235   }
 236
 237   MediaEventSource<nsTArray<uint8_t>, nsString>& OnEncrypted() {
 238     return mOnEncrypted;
 239   }
 240
 241   MediaEventSource<void>& OnWaitingForKey() { return mOnWaitingForKey; }
 242
 243   MediaEventSource<MediaResult>& OnDecodeWarning() { return mOnDecodeWarning; }
 244
 245   MediaEventSource<VideoInfo>& OnStoreDecoderBenchmark() {
 246     return mOnStoreDecoderBenchmark;
 247   }
 248
 249  private:
 250   bool HasVideo() const { return mVideo.mTrackDemuxer; }
 251   bool HasAudio() const { return mAudio.mTrackDemuxer; }
 252
 253   bool IsWaitingOnCDMResource();
 254
 255   bool InitDemuxer();
 256   // Notify the track demuxers that new data has been received.
 257   void NotifyTrackDemuxers();
 258   void ReturnOutput(MediaData* aData, TrackType aTrack);
 259
 260   // Enqueues a task to call Update(aTrack) on the decoder task queue.
 261   // Lock for corresponding track must be held.
 262   void ScheduleUpdate(TrackType aTrack);
 263   void Update(TrackType aTrack);
 264   // Handle actions should more data be received.
 265   // Returns true if no more action is required.
 266   bool UpdateReceivedNewData(TrackType aTrack);
 267   // Called when new samples need to be demuxed.
 268   void RequestDemuxSamples(TrackType aTrack);
 269   // Handle demuxed samples by the input behavior.
 270   void HandleDemuxedSamples(TrackType aTrack,
 271                             FrameStatistics::AutoNotifyDecoded& aA);
 272   // Decode any pending already demuxed samples.
 273   void DecodeDemuxedSamples(TrackType aTrack, MediaRawData* aSample);
 274
 275   struct InternalSeekTarget {
 276     InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
 277         : mTime(aTime),
 278           mDropTarget(aDropTarget),
 279           mWaiting(false),
 280           mHasSeeked(false) {}
 281
 282     media::TimeUnit Time() const { return mTime.mStart; }
 283     media::TimeUnit EndTime() const { return mTime.mEnd; }
 284     bool Contains(const media::TimeUnit& aTime) const {
 285       return mTime.Contains(aTime);
 286     }
 287
 288     media::TimeInterval mTime;
 289     bool mDropTarget;
 290     bool mWaiting;
 291     bool mHasSeeked;
 292   };
 293
 294   // Perform an internal seek to aTime. If aDropTarget is true then
 295   // the first sample past the target will be dropped.
 296   void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
 297   // Return the end time of the internal seek target if it exists. Otherwise,
 298   // return infinity.
 299   media::TimeUnit GetInternalSeekTargetEndTime() const;
 300
 301   // Drain the current decoder.
 302   void DrainDecoder(TrackType aTrack);
 303   void NotifyNewOutput(TrackType aTrack,
 304                        MediaDataDecoder::DecodedData&& aResults);
 305   void NotifyError(TrackType aTrack, const MediaResult& aError);
 306   void NotifyWaitingForData(TrackType aTrack);
 307   void NotifyWaitingForKey(TrackType aTrack);
 308   void NotifyEndOfStream(TrackType aTrack);
 309
 310   void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
 311
 312   // Initializes mLayersBackendType if possible.
 313   void InitLayersBackendType();
 314
 315   void Reset(TrackType aTrack);
 316   void DropDecodedSamples(TrackType aTrack);
 317
 318   // Return a target timeunit which the reader should skip to, this would be
 319   // either the timethreshold we pass, or the time of the next keyframe. Return
 320   // nothing if we don't need to skip.
 321   // @param aTimeThreshold
 322   // The time that we expect the time of next video frame should be or go beyond
 323   // @param aRequestNextVideoKeyFrame
 324   // If true and the next keyframe's time is larger than aTimeThreshold, skip to
 325   // the next keyframe time instead of aTimeThreshold.
 326   Maybe<media::TimeUnit> ShouldSkip(media::TimeUnit aTimeThreshold,
 327                                     bool aRequestNextVideoKeyFrame);
 328
 329   void SetVideoDecodeThreshold();
 330
 331   size_t SizeOfQueue(TrackType aTrack);
 332
 333   // Fire a new OnStoreDecoderBenchmark event that will create new
 334   // storage of the decoder benchmark.
 335   // This is called only on TaskQueue.
 336   void NotifyDecoderBenchmarkStore();
 337
 338   enum class DrainState {
 339     None,
 340     DrainRequested,
 341     Draining,
 342     PartialDrainPending,
 343     DrainCompleted,
 344     DrainAborted,
 345   };
 346
 347   class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise> {
 348     NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder)
 349    private:
 350     ~SharedShutdownPromiseHolder() = default;
 351   };
 352
 353   struct DecoderData {
 354     DecoderData(MediaFormatReader* aOwner, MediaData::Type aType,
 355                 uint32_t aNumOfMaxError)
 356         : mOwner(aOwner),
 357           mType(aType),
 358           mMutex("DecoderData"),
 359           mDescription("shutdown"),
 360           mUpdateScheduled(false),
 361           mDemuxEOS(false),
 362           mWaitingForData(false),
 363           mWaitingForKey(false),
 364           mReceivedNewData(false),
 365           mFlushing(false),
 366           mFlushed(true),
 367           mDrainState(DrainState::None),
 368           mNumOfConsecutiveDecodingError(0),
 369           mMaxConsecutiveDecodingError(aNumOfMaxError),
 370           mNumOfConsecutiveRDDOrGPUCrashes(0),
 371           mMaxConsecutiveRDDOrGPUCrashes(
 372               StaticPrefs::media_rdd_process_max_crashes()),
 373           mFirstFrameTime(Some(media::TimeUnit::Zero())),
 374           mNumSamplesInput(0),
 375           mNumSamplesOutput(0),
 376           mNumSamplesOutputTotal(0),
 377           mNumSamplesSkippedTotal(0),
 378           mSizeOfQueue(0),
 379           mIsHardwareAccelerated(false),
 380           mLastStreamSourceID(UINT32_MAX),
 381           mIsNullDecode(false),
 382           mHardwareDecodingDisabled(false) {
 383       DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderData",
 384                                            this);
 385     }
 386
 387     ~DecoderData() {
 388       DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderData",
 389                                           this);
 390     }
 391
 392     MediaFormatReader* mOwner;
 393     // Disambiguate Audio vs Video.
 394     MediaData::Type mType;
 395     RefPtr<MediaTrackDemuxer> mTrackDemuxer;
 396     // TaskQueue on which decoder can choose to decode.
 397     // Only non-null up until the decoder is created.
 398     RefPtr<TaskQueue> mTaskQueue;
 399
 400     // Mutex protecting mDescription, mDecoder, mTrackDemuxer and mWorkingInfo
 401     // as those can be read outside the TaskQueue.
 402     // They are only written on the TaskQueue however, as such mMutex doesn't
 403     // need to be held when those members are read on the TaskQueue.
 404     Mutex mMutex MOZ_UNANNOTATED;
 405     // The platform decoder.
 406     RefPtr<MediaDataDecoder> mDecoder;
 407     nsCString mDescription;
 408     void ShutdownDecoder();
 409
 410     // Only accessed from reader's task queue.
 411     bool mUpdateScheduled;
 412     bool mDemuxEOS;
 413     bool mWaitingForData;
 414     bool mWaitingForKey;
 415     bool mReceivedNewData;
 416
 417     // Pending seek.
 418     MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
 419
 420     // Queued demux samples waiting to be decoded.
 421     nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
 422     MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
 423     // A WaitingPromise is pending if the demuxer is waiting for data or
 424     // if the decoder is waiting for a key.
 425     MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
 426     bool HasWaitingPromise() const {
 427       MOZ_ASSERT(mOwner->OnTaskQueue());
 428       return !mWaitingPromise.IsEmpty();
 429     }
 430
 431     bool IsWaitingForData() const {
 432       MOZ_ASSERT(mOwner->OnTaskQueue());
 433       return mWaitingForData;
 434     }
 435
 436     bool IsWaitingForKey() const {
 437       MOZ_ASSERT(mOwner->OnTaskQueue());
 438       return mWaitingForKey && mDecodeRequest.Exists();
 439     }
 440
 441     // MediaDataDecoder handler's variables.
 442     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest;
 443     bool mFlushing;  // True if flush is in action.
 444     // Set to true if the last operation run on the decoder was a flush.
 445     bool mFlushed;
 446     RefPtr<SharedShutdownPromiseHolder> mShutdownPromise;
 447
 448     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest;
 449     DrainState mDrainState;
 450     bool HasPendingDrain() const { return mDrainState != DrainState::None; }
 451     bool HasCompletedDrain() const {
 452       return mDrainState == DrainState::DrainCompleted ||
 453              mDrainState == DrainState::DrainAborted;
 454     }
 455     void RequestDrain() {
 456       MOZ_RELEASE_ASSERT(mDrainState == DrainState::None);
 457       mDrainState = DrainState::DrainRequested;
 458     }
 459
 460     // Track decoding error and fail when we hit the limit.
 461     uint32_t mNumOfConsecutiveDecodingError;
 462     uint32_t mMaxConsecutiveDecodingError;
 463
 464     // Track RDD process crashes and fail when we hit the limit.
 465     uint32_t mNumOfConsecutiveRDDOrGPUCrashes;
 466     uint32_t mMaxConsecutiveRDDOrGPUCrashes;
 467
 468     // Set when we haven't yet decoded the first frame.
 469     // Cleared once the first frame has been decoded.
 470     // This is used to determine, upon error, if we should try again to decode
 471     // the frame, or skip to the next keyframe.
 472     Maybe<media::TimeUnit> mFirstFrameTime;
 473
 474     Maybe<MediaResult> mError;
 475     bool HasFatalError() const {
 476       if (!mError.isSome()) {
 477         return false;
 478       }
 479       if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
 480         // Allow decode errors to be non-fatal, but give up
 481         // if we have too many, or if warnings should be treated as errors.
 482         return mNumOfConsecutiveDecodingError > mMaxConsecutiveDecodingError ||
 483                StaticPrefs::media_playback_warnings_as_errors();
 484       } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
 485         // If the caller asked for a new decoder we shouldn't treat
 486         // it as fatal.
 487         return false;
 488       } else if (mError.ref() ==
 489                  NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR) {
 490         // Allow RDD crashes to be non-fatal, but give up
 491         // if we have too many, or if warnings should be treated as errors.
 492         return mNumOfConsecutiveRDDOrGPUCrashes >
 493                    mMaxConsecutiveRDDOrGPUCrashes ||
 494                StaticPrefs::media_playback_warnings_as_errors();
 495       } else {
 496         // All other error types are fatal
 497         return true;
 498       }
 499     }
 500
 501     // If set, all decoded samples prior mTimeThreshold will be dropped.
 502     // Used for internal seeking when a change of stream is detected or when
 503     // encountering data discontinuity.
 504     Maybe<InternalSeekTarget> mTimeThreshold;
 505     // Time of last decoded sample returned.
 506     Maybe<media::TimeInterval> mLastDecodedSampleTime;
 507
 508     // Decoded samples returned my mDecoder awaiting being returned to
 509     // state machine upon request.
 510     nsTArray<RefPtr<MediaData>> mOutput;
 511     uint64_t mNumSamplesInput;
 512     uint64_t mNumSamplesOutput;
 513     uint64_t mNumSamplesOutputTotal;
 514     uint64_t mNumSamplesSkippedTotal;
 515
 516     // These get overridden in the templated concrete class.
 517     // Indicate if we have a pending promise for decoded frame.
 518     // Rejecting the promise will stop the reader from decoding ahead.
 519     virtual bool HasPromise() const = 0;
 520     virtual void RejectPromise(const MediaResult& aError,
 521                                const char* aMethodName) = 0;
 522
 523     // Clear track demuxer related data.
 524     void ResetDemuxer() {
 525       mDemuxRequest.DisconnectIfExists();
 526       mSeekRequest.DisconnectIfExists();
 527       mTrackDemuxer->Reset();
 528       mQueuedSamples.Clear();
 529     }
 530
 531     // Flush the decoder if present and reset decoding related data.
 532     // Following a flush, the decoder is ready to accept any new data.
 533     void Flush();
 534
 535     bool CancelWaitingForKey() {
 536       if (!mWaitingForKey) {
 537         return false;
 538       }
 539       mWaitingForKey = false;
 540       if (IsWaitingForData() || !HasWaitingPromise()) {
 541         return false;
 542       }
 543       mWaitingPromise.Resolve(mType, __func__);
 544       return true;
 545     }
 546
 547     // Reset the state of the DecoderData, clearing all queued frames
 548     // (pending demuxed and decoded).
 549     // The track demuxer is *not* reset.
 550     void ResetState() {
 551       MOZ_ASSERT(mOwner->OnTaskQueue());
 552       mDemuxEOS = false;
 553       mWaitingForData = false;
 554       mQueuedSamples.Clear();
 555       mDecodeRequest.DisconnectIfExists();
 556       mDrainRequest.DisconnectIfExists();
 557       mDrainState = DrainState::None;
 558       CancelWaitingForKey();
 559       mTimeThreshold.reset();
 560       mLastDecodedSampleTime.reset();
 561       mOutput.Clear();
 562       mNumSamplesInput = 0;
 563       mNumSamplesOutput = 0;
 564       mSizeOfQueue = 0;
 565       mNextStreamSourceID.reset();
 566       if (!HasFatalError()) {
 567         mError.reset();
 568       }
 569     }
 570
 571     bool HasInternalSeekPending() const {
 572       return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
 573     }
 574
 575     // Return the current TrackInfo in the stream. If the stream content never
 576     // changed since AsyncReadMetadata was called then the TrackInfo used is
 577     // mOriginalInfo, other it will be mInfo. The later case is only ever true
 578     // with MSE or the WebMDemuxer.
 579     const TrackInfo* GetCurrentInfo() const {
 580       if (mInfo) {
 581         return *mInfo;
 582       }
 583       return mOriginalInfo.get();
 584     }
 585     // Return the current TrackInfo updated as per the decoder output.
 586     // Typically for audio, the number of channels and/or sampling rate can vary
 587     // between what was found in the metadata and what the decoder returned.
 588     const TrackInfo* GetWorkingInfo() const { return mWorkingInfo.get(); }
 589     bool IsEncrypted() const { return GetCurrentInfo()->mCrypto.IsEncrypted(); }
 590
 591     // Used by the MDSM for logging purposes.
 592     Atomic<size_t> mSizeOfQueue;
 593     // Used by the MDSM to determine if video decoding is hardware accelerated.
 594     // This value is updated after a frame is successfully decoded.
 595     Atomic<bool> mIsHardwareAccelerated;
 596     // Sample format monitoring.
 597     uint32_t mLastStreamSourceID;
 598     Maybe<uint32_t> mNextStreamSourceID;
 599     media::TimeIntervals mTimeRanges;
 600     Maybe<media::TimeUnit> mLastTimeRangesEnd;
 601     // TrackInfo as first discovered during ReadMetadata.
 602     UniquePtr<TrackInfo> mOriginalInfo;
 603     // Written exclusively on the TaskQueue, can be read on MDSM's TaskQueue.
 604     // Must be read with parent's mutex held.
 605     UniquePtr<TrackInfo> mWorkingInfo;
 606     RefPtr<TrackInfoSharedPtr> mInfo;
 607     Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
 608     // Use NullDecoderModule or not.
 609     bool mIsNullDecode;
 610     bool mHardwareDecodingDisabled;
 611     // Whether we have reported hardware decoding support for video. Used only
 612     // on reader's task queue,
 613     bool mHasReportedVideoHardwareSupportTelemtry = false;
 614
 615     class {
 616      public:
 617       float Mean() const { return mMean; }
 618
 619       void Update(const media::TimeUnit& aValue) {
 620         if (aValue == media::TimeUnit::Zero()) {
 621           return;
 622         }
 623         mMean += (1.0f / aValue.ToSeconds() - mMean) / ++mCount;
 624       }
 625
 626       void Reset() {
 627         mMean = 0;
 628         mCount = 0;
 629       }
 630
 631      private:
 632       float mMean = 0;
 633       uint64_t mCount = 0;
 634     } mMeanRate;
 635   };
 636
 637   template <typename Type>
 638   class DecoderDataWithPromise : public DecoderData {
 639    public:
 640     DecoderDataWithPromise(MediaFormatReader* aOwner, MediaData::Type aType,
 641                            uint32_t aNumOfMaxError)
 642         : DecoderData(aOwner, aType, aNumOfMaxError), mHasPromise(false) {
 643       DecoderDoctorLogger::LogConstructionAndBase(
 644           "MediaFormatReader::DecoderDataWithPromise", this,
 645           "MediaFormatReader::DecoderData",
 646           static_cast<const MediaFormatReader::DecoderData*>(this));
 647     }
 648
 649     ~DecoderDataWithPromise() {
 650       DecoderDoctorLogger::LogDestruction(
 651           "MediaFormatReader::DecoderDataWithPromise", this);
 652     }
 653
 654     bool HasPromise() const override { return mHasPromise; }
 655
 656     RefPtr<DataPromise<Type>> EnsurePromise(const char* aMethodName) {
 657       MOZ_ASSERT(mOwner->OnTaskQueue());
 658       mHasPromise = true;
 659       return mPromise.Ensure(aMethodName);
 660     }
 661
 662     void ResolvePromise(Type* aData, const char* aMethodName) {
 663       MOZ_ASSERT(mOwner->OnTaskQueue());
 664       mPromise.Resolve(aData, aMethodName);
 665       mHasPromise = false;
 666     }
 667
 668     void RejectPromise(const MediaResult& aError,
 669                        const char* aMethodName) override {
 670       MOZ_ASSERT(mOwner->OnTaskQueue());
 671       mPromise.Reject(aError, aMethodName);
 672       mHasPromise = false;
 673     }
 674
 675    private:
 676     MozPromiseHolder<DataPromise<Type>> mPromise;
 677     Atomic<bool> mHasPromise;
 678   };
 679
 680   // Decode task queue.
 681   RefPtr<TaskQueue> mTaskQueue;
 682
 683   DecoderDataWithPromise<AudioData> mAudio;
 684   DecoderDataWithPromise<VideoData> mVideo;
 685
 686   // Returns true when the decoder for this track needs input.
 687   bool NeedInput(DecoderData& aDecoder);
 688
 689   DecoderData& GetDecoderData(TrackType aTrack);
 690
 691   // Demuxer objects.
 692   class DemuxerProxy;
 693   UniquePtr<DemuxerProxy> mDemuxer;
 694   bool mDemuxerInitDone;
 695   void OnDemuxerInitDone(const MediaResult& aResult);
 696   void OnDemuxerInitFailed(const MediaResult& aError);
 697   MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
 698   MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise;
 699   bool mPendingNotifyDataArrived;
 700   void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
 701
 702   void DoDemuxVideo();
 703   void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
 704   void OnVideoDemuxFailed(const MediaResult& aError) {
 705     OnDemuxFailed(TrackType::kVideoTrack, aError);
 706   }
 707
 708   void DoDemuxAudio();
 709   void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
 710   void OnAudioDemuxFailed(const MediaResult& aError) {
 711     OnDemuxFailed(TrackType::kAudioTrack, aError);
 712   }
 713
 714   void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
 715   MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise>
 716       mSkipRequest;
 717   void VideoSkipReset(uint32_t aSkipped);
 718   void OnVideoSkipCompleted(uint32_t aSkipped);
 719   void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
 720
 721   // The last number of decoded output frames that we've reported to
 722   // MediaDecoder::NotifyDecoded(). We diff the number of output video
 723   // frames every time that DecodeVideoData() is called, and report the
 724   // delta there.
 725   uint64_t mLastReportedNumDecodedFrames;
 726
 727   // Timestamp of the previous decoded keyframe, in microseconds.
 728   int64_t mPreviousDecodedKeyframeTime_us;
 729   // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
 730   static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
 731
 732   RefPtr<layers::KnowsCompositor> mKnowsCompositor;
 733
 734   // Metadata objects
 735   // True if we've read the streams' metadata.
 736   bool mInitDone;
 737   MozPromiseHolder<MetadataPromise> mMetadataPromise;
 738   bool IsEncrypted() const;
 739
 740   // Set to true if any of our track buffers may be blocking.
 741   bool mTrackDemuxersMayBlock;
 742
 743   // Seeking objects.
 744   void SetSeekTarget(const SeekTarget& aTarget);
 745   bool IsSeeking() const { return mPendingSeekTime.isSome(); }
 746   bool IsVideoSeeking() const {
 747     return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
 748   }
 749   void ScheduleSeek();
 750   void AttemptSeek();
 751   void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
 752   void DoVideoSeek();
 753   void OnVideoSeekCompleted(media::TimeUnit aTime);
 754   void OnVideoSeekFailed(const MediaResult& aError);
 755   bool mSeekScheduled;
 756
 757   void DoAudioSeek();
 758   void OnAudioSeekCompleted(media::TimeUnit aTime);
 759   void OnAudioSeekFailed(const MediaResult& aError);
 760   // The SeekTarget that was last given to Seek()
 761   SeekTarget mOriginalSeekTarget;
 762   // Temporary seek information while we wait for the data
 763   Maybe<media::TimeUnit> mFallbackSeekTime;
 764   Maybe<media::TimeUnit> mPendingSeekTime;
 765   MozPromiseHolder<SeekPromise> mSeekPromise;
 766
 767   RefPtr<VideoFrameContainer> mVideoFrameContainer;
 768   layers::ImageContainer* GetImageContainer();
 769
 770   RefPtr<CDMProxy> mCDMProxy;
 771
 772   RefPtr<GMPCrashHelper> mCrashHelper;
 773
 774   void SetNullDecode(TrackType aTrack, bool aIsNullDecode);
 775
 776   class DecoderFactory;
 777   UniquePtr<DecoderFactory> mDecoderFactory;
 778
 779   class ShutdownPromisePool;
 780   UniquePtr<ShutdownPromisePool> mShutdownPromisePool;
 781
 782   MediaEventListener mOnTrackWaitingForKeyListener;
 783
 784   void OnFirstDemuxCompleted(TrackInfo::TrackType aType,
 785                              RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
 786
 787   void OnFirstDemuxFailed(TrackInfo::TrackType aType,
 788                           const MediaResult& aError);
 789
 790   void MaybeResolveMetadataPromise();
 791
 792   // Stores presentation info required for playback.
 793   MediaInfo mInfo;
 794
 795   UniquePtr<MetadataTags> mTags;
 796
 797   // A flag indicating if the start time is known or not.
 798   bool mHasStartTime = false;
 799
 800   void ShutdownDecoder(TrackType aTrack);
 801   RefPtr<ShutdownPromise> TearDownDecoders();
 802
 803   bool mShutdown = false;
 804
 805   // Buffered range.
 806   Canonical<media::TimeIntervals> mBuffered;
 807
 808   // Used to send TimedMetadata to the listener.
 809   TimedMetadataEventProducer mTimedMetadataEvent;
 810
 811   // Notify if this media is not seekable.
 812   MediaEventProducer<void> mOnMediaNotSeekable;
 813
 814   // Notify if we are waiting for a decryption key.
 815   MediaEventProducer<TrackInfo::TrackType> mOnTrackWaitingForKey;
 816
 817   MediaEventProducer<nsTArray<uint8_t>, nsString> mOnEncrypted;
 818
 819   MediaEventProducer<void> mOnWaitingForKey;
 820
 821   MediaEventProducer<MediaResult> mOnDecodeWarning;
 822
 823   MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark;
 824
 825   RefPtr<FrameStatistics> mFrameStats;
 826
 827   // Used in bug 1393399 for telemetry.
 828   const MediaDecoderOwnerID mMediaDecoderOwnerID;
 829
 830   bool ResolveSetCDMPromiseIfDone(TrackType aTrack);
 831   void PrepareToSetCDMForTrack(TrackType aTrack);
 832   MozPromiseHolder<SetCDMPromise> mSetCDMPromise;
 833   TrackSet mSetCDMForTracks{};
 834   bool IsDecoderWaitingForCDM(TrackType aTrack);
 835
 836   void GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo);
 837
 838   // Only be used on Windows when the media engine playback is enabled.
 839   Maybe<uint64_t> mMediaEngineId;
 840 };
 841
 842 }  // namespace mozilla
 843
 844 #endif