dom/media/MediaFormatReader.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #if !defined(MediaFormatReader_h_)
   8 #  define MediaFormatReader_h_
   9
  10 #  include "FrameStatistics.h"
  11 #  include "MediaDataDemuxer.h"
  12 #  include "MediaEventSource.h"
  13 #  include "MediaMetadataManager.h"
  14 #  include "MediaPromiseDefs.h"
  15 #  include "PlatformDecoderModule.h"
  16 #  include "SeekTarget.h"
  17 #  include "mozilla/Atomics.h"
  18 #  include "mozilla/Maybe.h"
  19 #  include "mozilla/MozPromise.h"
  20 #  include "mozilla/Mutex.h"
  21 #  include "mozilla/StateMirroring.h"
  22 #  include "mozilla/StaticPrefs_media.h"
  23 #  include "mozilla/TaskQueue.h"
  24 #  include "mozilla/TimeStamp.h"
  25 #  include "mozilla/ThreadSafeWeakPtr.h"
  26 #  include "mozilla/dom/MediaDebugInfoBinding.h"
  27
  28 namespace mozilla {
  29
  30 class CDMProxy;
  31 class GMPCrashHelper;
  32 class MediaResource;
  33 class VideoFrameContainer;
  34
  35 struct WaitForDataRejectValue {
  36   enum Reason { SHUTDOWN, CANCELED };
  37
  38   WaitForDataRejectValue(MediaData::Type aType, Reason aReason)
  39       : mType(aType), mReason(aReason) {}
  40   MediaData::Type mType;
  41   Reason mReason;
  42 };
  43
  44 struct SeekRejectValue {
  45   MOZ_IMPLICIT SeekRejectValue(const MediaResult& aError)
  46       : mType(MediaData::Type::NULL_DATA), mError(aError) {}
  47   MOZ_IMPLICIT SeekRejectValue(nsresult aResult)
  48       : mType(MediaData::Type::NULL_DATA), mError(aResult) {}
  49   SeekRejectValue(MediaData::Type aType, const MediaResult& aError)
  50       : mType(aType), mError(aError) {}
  51   MediaData::Type mType;
  52   MediaResult mError;
  53 };
  54
  55 struct MetadataHolder {
  56   UniquePtr<MediaInfo> mInfo;
  57   UniquePtr<MetadataTags> mTags;
  58 };
  59
  60 using MediaDecoderOwnerID = void*;
  61
  62 struct MOZ_STACK_CLASS MediaFormatReaderInit {
  63   MediaResource* mResource = nullptr;
  64   VideoFrameContainer* mVideoFrameContainer = nullptr;
  65   FrameStatistics* mFrameStats = nullptr;
  66   already_AddRefed<layers::KnowsCompositor> mKnowsCompositor;
  67   already_AddRefed<GMPCrashHelper> mCrashHelper;
  68   // Used in bug 1393399 for temporary telemetry.
  69   MediaDecoderOwnerID mMediaDecoderOwnerID = nullptr;
  70   Maybe<TrackingId> mTrackingId;
  71 };
  72
  73 DDLoggedTypeDeclName(MediaFormatReader);
  74
  75 class MediaFormatReader final
  76     : public SupportsThreadSafeWeakPtr<MediaFormatReader>,
  77       public DecoderDoctorLifeLogger<MediaFormatReader> {
  78   static const bool IsExclusive = true;
  79   using TrackType = TrackInfo::TrackType;
  80   using NotifyDataArrivedPromise = MozPromise<bool, MediaResult, IsExclusive>;
  81
  82  public:
  83   MOZ_DECLARE_REFCOUNTED_TYPENAME(MediaFormatReader)
  84
  85   using TrackSet = EnumSet<TrackInfo::TrackType>;
  86   using MetadataPromise = MozPromise<MetadataHolder, MediaResult, IsExclusive>;
  87
  88   template <typename Type>
  89   using DataPromise = MozPromise<RefPtr<Type>, MediaResult, IsExclusive>;
  90   using AudioDataPromise = DataPromise<AudioData>;
  91   using VideoDataPromise = DataPromise<VideoData>;
  92
  93   using SeekPromise = MozPromise<media::TimeUnit, SeekRejectValue, IsExclusive>;
  94
  95   // Note that, conceptually, WaitForData makes sense in a non-exclusive sense.
  96   // But in the current architecture it's only ever used exclusively (by MDSM),
  97   // so we mark it that way to verify our assumptions. If you have a use-case
  98   // for multiple WaitForData consumers, feel free to flip the exclusivity here.
  99   using WaitForDataPromise =
 100       MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>;
 101
 102   MediaFormatReader(MediaFormatReaderInit& aInit, MediaDataDemuxer* aDemuxer);
 103   virtual ~MediaFormatReader();
 104
 105   // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE
 106   // on failure.
 107   nsresult Init();
 108
 109   size_t SizeOfVideoQueueInFrames();
 110   size_t SizeOfAudioQueueInFrames();
 111
 112   // Requests one video sample from the reader.
 113   RefPtr<VideoDataPromise> RequestVideoData(
 114       const media::TimeUnit& aTimeThreshold,
 115       bool aRequestNextVideoKeyFrame = false);
 116
 117   // Requests one audio sample from the reader.
 118   //
 119   // The decode should be performed asynchronously, and the promise should
 120   // be resolved when it is complete.
 121   RefPtr<AudioDataPromise> RequestAudioData();
 122
 123   // The default implementation of AsyncReadMetadata is implemented in terms of
 124   // synchronous ReadMetadata() calls. Implementations may also
 125   // override AsyncReadMetadata to create a more proper async implementation.
 126   RefPtr<MetadataPromise> AsyncReadMetadata();
 127
 128   // Fills aInfo with the latest cached data required to present the media,
 129   // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded.
 130   void ReadUpdatedMetadata(MediaInfo* aInfo);
 131
 132   RefPtr<SeekPromise> Seek(const SeekTarget& aTarget);
 133
 134   // Called once new data has been cached by the MediaResource.
 135   // mBuffered should be recalculated and updated accordingly.
 136   void NotifyDataArrived();
 137
 138   // Update ID for the external playback engine. Currently it's only used on
 139   // Windows when the media engine playback is enabled.
 140   void UpdateMediaEngineId(uint64_t aMediaEngineId);
 141
 142  protected:
 143   // Recomputes mBuffered.
 144   void UpdateBuffered();
 145
 146  public:
 147   // Called by MDSM in dormant state to release resources allocated by this
 148   // reader. The reader can resume decoding by calling Seek() to a specific
 149   // position.
 150   void ReleaseResources();
 151
 152   bool OnTaskQueue() const { return OwnerThread()->IsCurrentThreadIn(); }
 153
 154   // Resets all state related to decoding, emptying all buffers etc.
 155   // Cancels all pending Request*Data() request callbacks, rejects any
 156   // outstanding seek promises, and flushes the decode pipeline. The
 157   // decoder must not call any of the callbacks for outstanding
 158   // Request*Data() calls after this is called. Calls to Request*Data()
 159   // made after this should be processed as usual.
 160   //
 161   // Normally this call preceedes a Seek() call, or shutdown.
 162   //
 163   // aParam is a set of TrackInfo::TrackType enums specifying which
 164   // queues need to be reset, defaulting to both audio and video tracks.
 165   nsresult ResetDecode(const TrackSet& aTracks);
 166
 167   // Destroys the decoding state. The reader cannot be made usable again.
 168   // This is different from ReleaseMediaResources() as it is irreversable,
 169   // whereas ReleaseMediaResources() is.  Must be called on the decode
 170   // thread.
 171   RefPtr<ShutdownPromise> Shutdown();
 172
 173   // Returns true if this decoder reader uses hardware accelerated video
 174   // decoding.
 175   bool VideoIsHardwareAccelerated() const;
 176
 177   // By default, the state machine polls the reader once per second when it's
 178   // in buffering mode. Some readers support a promise-based mechanism by which
 179   // they notify the state machine when the data arrives.
 180   bool IsWaitForDataSupported() const { return true; }
 181
 182   RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType);
 183
 184   // The MediaDecoderStateMachine uses various heuristics that assume that
 185   // raw media data is arriving sequentially from a network channel. This
 186   // makes sense in the <video src="foo"> case, but not for more advanced use
 187   // cases like MSE.
 188   bool UseBufferingHeuristics() const { return mTrackDemuxersMayBlock; }
 189
 190   RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy);
 191
 192   // Requests that the MediaFormatReader populates aInfo with debug information.
 193   // This may be done asynchronously, and aInfo should *not* be accessed by the
 194   // caller until the returned promise is resolved or rejected.
 195   RefPtr<GenericPromise> RequestDebugInfo(
 196       dom::MediaFormatReaderDebugInfo& aInfo);
 197
 198   Maybe<nsCString> GetAudioProcessPerCodec();
 199
 200   // Switch the video decoder to NullDecoderModule. It might takes effective
 201   // since a few samples later depends on how much demuxed samples are already
 202   // queued in the original video decoder.
 203   void SetVideoNullDecode(bool aIsNullDecode);
 204
 205   void UpdateCompositor(already_AddRefed<layers::KnowsCompositor>);
 206
 207   void UpdateDuration(const media::TimeUnit& aDuration) {
 208     MOZ_ASSERT(OnTaskQueue());
 209     UpdateBuffered();
 210   }
 211
 212   AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() {
 213     return &mBuffered;
 214   }
 215
 216   TaskQueue* OwnerThread() const { return mTaskQueue; }
 217
 218   TimedMetadataEventSource& TimedMetadataEvent() { return mTimedMetadataEvent; }
 219
 220   // Notified by the OggDemuxer during playback when chained ogg is detected.
 221   MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; }
 222
 223   TimedMetadataEventProducer& TimedMetadataProducer() {
 224     return mTimedMetadataEvent;
 225   }
 226
 227   MediaEventProducer<void>& MediaNotSeekableProducer() {
 228     return mOnMediaNotSeekable;
 229   }
 230
 231   // Notified if the reader can't decode a sample due to a missing decryption
 232   // key.
 233   MediaEventSource<TrackInfo::TrackType>& OnTrackWaitingForKey() {
 234     return mOnTrackWaitingForKey;
 235   }
 236
 237   MediaEventProducer<TrackInfo::TrackType>& OnTrackWaitingForKeyProducer() {
 238     return mOnTrackWaitingForKey;
 239   }
 240
 241   MediaEventSource<nsTArray<uint8_t>, nsString>& OnEncrypted() {
 242     return mOnEncrypted;
 243   }
 244
 245   MediaEventSource<void>& OnWaitingForKey() { return mOnWaitingForKey; }
 246
 247   MediaEventSource<MediaResult>& OnDecodeWarning() { return mOnDecodeWarning; }
 248
 249   MediaEventSource<VideoInfo>& OnStoreDecoderBenchmark() {
 250     return mOnStoreDecoderBenchmark;
 251   }
 252
 253   MediaEventProducer<VideoInfo, AudioInfo>& OnTrackInfoUpdatedEvent() {
 254     return mTrackInfoUpdatedEvent;
 255   }
 256
 257  private:
 258   bool HasVideo() const { return mVideo.mTrackDemuxer; }
 259   bool HasAudio() const { return mAudio.mTrackDemuxer; }
 260
 261   bool IsWaitingOnCDMResource();
 262
 263   bool InitDemuxer();
 264   // Notify the track demuxers that new data has been received.
 265   void NotifyTrackDemuxers();
 266   void ReturnOutput(MediaData* aData, TrackType aTrack);
 267
 268   // Enqueues a task to call Update(aTrack) on the decoder task queue.
 269   // Lock for corresponding track must be held.
 270   void ScheduleUpdate(TrackType aTrack);
 271   void Update(TrackType aTrack);
 272   // Handle actions should more data be received.
 273   // Returns true if no more action is required.
 274   bool UpdateReceivedNewData(TrackType aTrack);
 275   // Called when new samples need to be demuxed.
 276   void RequestDemuxSamples(TrackType aTrack);
 277   // Handle demuxed samples by the input behavior.
 278   void HandleDemuxedSamples(TrackType aTrack,
 279                             FrameStatistics::AutoNotifyDecoded& aA);
 280   // Decode any pending already demuxed samples.
 281   void DecodeDemuxedSamples(TrackType aTrack, MediaRawData* aSample);
 282
 283   struct InternalSeekTarget {
 284     InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
 285         : mTime(aTime),
 286           mDropTarget(aDropTarget),
 287           mWaiting(false),
 288           mHasSeeked(false) {}
 289
 290     media::TimeUnit Time() const { return mTime.mStart; }
 291     media::TimeUnit EndTime() const { return mTime.mEnd; }
 292     bool Contains(const media::TimeUnit& aTime) const {
 293       return mTime.Contains(aTime);
 294     }
 295
 296     media::TimeInterval mTime;
 297     bool mDropTarget;
 298     bool mWaiting;
 299     bool mHasSeeked;
 300   };
 301
 302   // Perform an internal seek to aTime. If aDropTarget is true then
 303   // the first sample past the target will be dropped.
 304   void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
 305   // Return the end time of the internal seek target if it exists. Otherwise,
 306   // return infinity.
 307   media::TimeUnit GetInternalSeekTargetEndTime() const;
 308
 309   // Drain the current decoder.
 310   void DrainDecoder(TrackType aTrack);
 311   void NotifyNewOutput(TrackType aTrack,
 312                        MediaDataDecoder::DecodedData&& aResults);
 313   void NotifyError(TrackType aTrack, const MediaResult& aError);
 314   void NotifyWaitingForData(TrackType aTrack);
 315   void NotifyWaitingForKey(TrackType aTrack);
 316   void NotifyEndOfStream(TrackType aTrack);
 317
 318   void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
 319
 320   // Initializes mLayersBackendType if possible.
 321   void InitLayersBackendType();
 322
 323   void Reset(TrackType aTrack);
 324   void DropDecodedSamples(TrackType aTrack);
 325
 326   // Return a target timeunit which the reader should skip to, this would be
 327   // either the timethreshold we pass, or the time of the next keyframe. Return
 328   // nothing if we don't need to skip.
 329   // @param aTimeThreshold
 330   // The time that we expect the time of next video frame should be or go beyond
 331   // @param aRequestNextVideoKeyFrame
 332   // If true and the next keyframe's time is larger than aTimeThreshold, skip to
 333   // the next keyframe time instead of aTimeThreshold.
 334   Maybe<media::TimeUnit> ShouldSkip(media::TimeUnit aTimeThreshold,
 335                                     bool aRequestNextVideoKeyFrame);
 336
 337   void SetVideoDecodeThreshold();
 338
 339   size_t SizeOfQueue(TrackType aTrack);
 340
 341   // Fire a new OnStoreDecoderBenchmark event that will create new
 342   // storage of the decoder benchmark.
 343   // This is called only on TaskQueue.
 344   void NotifyDecoderBenchmarkStore();
 345
 346   void NotifyTrackInfoUpdated();
 347
 348   enum class DrainState {
 349     None,
 350     DrainRequested,
 351     Draining,
 352     PartialDrainPending,
 353     DrainCompleted,
 354     DrainAborted,
 355   };
 356
 357   class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise> {
 358     NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder)
 359    private:
 360     ~SharedShutdownPromiseHolder() = default;
 361   };
 362
 363   struct DecoderData {
 364     DecoderData(MediaFormatReader* aOwner, MediaData::Type aType,
 365                 uint32_t aNumOfMaxError)
 366         : mOwner(aOwner),
 367           mType(aType),
 368           mMutex("DecoderData"),
 369           mDescription("uninitialized"),
 370           mProcessName(""),
 371           mCodecName(""),
 372           mUpdateScheduled(false),
 373           mDemuxEOS(false),
 374           mWaitingForDataStartTime(Nothing()),
 375           mWaitingForKey(false),
 376           mReceivedNewData(false),
 377           mFlushing(false),
 378           mFlushed(true),
 379           mDrainState(DrainState::None),
 380           mNumOfConsecutiveDecodingError(0),
 381           mMaxConsecutiveDecodingError(aNumOfMaxError),
 382           mNumOfConsecutiveRDDOrGPUCrashes(0),
 383           mMaxConsecutiveRDDOrGPUCrashes(
 384               StaticPrefs::media_rdd_process_max_crashes()),
 385           mNumOfConsecutiveUtilityCrashes(0),
 386           mMaxConsecutiveUtilityCrashes(
 387               StaticPrefs::media_utility_process_max_crashes()),
 388           mFirstFrameTime(Some(media::TimeUnit::Zero())),
 389           mNumSamplesInput(0),
 390           mNumSamplesOutput(0),
 391           mNumSamplesOutputTotal(0),
 392           mNumSamplesSkippedTotal(0),
 393           mSizeOfQueue(0),
 394           mIsHardwareAccelerated(false),
 395           mLastStreamSourceID(UINT32_MAX),
 396           mIsNullDecode(false),
 397           mHardwareDecodingDisabled(false) {
 398       DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderData",
 399                                            this);
 400     }
 401
 402     ~DecoderData() {
 403       DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderData",
 404                                           this);
 405     }
 406
 407     MediaFormatReader* mOwner;
 408     // Disambiguate Audio vs Video.
 409     MediaData::Type mType;
 410     RefPtr<MediaTrackDemuxer> mTrackDemuxer;
 411     // TaskQueue on which decoder can choose to decode.
 412     // Only non-null up until the decoder is created.
 413     RefPtr<TaskQueue> mTaskQueue;
 414
 415     // Mutex protecting mDescription, mDecoder, mTrackDemuxer, mWorkingInfo,
 416     // mProcessName and mCodecName as those can be read outside the TaskQueue.
 417     // They are only written on the TaskQueue however, as such mMutex doesn't
 418     // need to be held when those members are read on the TaskQueue.
 419     Mutex mMutex MOZ_UNANNOTATED;
 420     // The platform decoder.
 421     RefPtr<MediaDataDecoder> mDecoder;
 422     nsCString mDescription;
 423     nsCString mProcessName;
 424     nsCString mCodecName;
 425     void ShutdownDecoder();
 426
 427     // Only accessed from reader's task queue.
 428     bool mUpdateScheduled;
 429     bool mDemuxEOS;
 430     Maybe<TimeStamp> mWaitingForDataStartTime;
 431     bool mWaitingForKey;
 432     bool mReceivedNewData;
 433
 434     // Pending seek.
 435     MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
 436
 437     // Queued demux samples waiting to be decoded.
 438     nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
 439     MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
 440     // A WaitingPromise is pending if the demuxer is waiting for data or
 441     // if the decoder is waiting for a key.
 442     MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
 443     bool HasWaitingPromise() const {
 444       MOZ_ASSERT(mOwner->OnTaskQueue());
 445       return !mWaitingPromise.IsEmpty();
 446     }
 447
 448     bool IsWaitingForData() const {
 449       MOZ_ASSERT(mOwner->OnTaskQueue());
 450       return !!mWaitingForDataStartTime;
 451     }
 452
 453     bool IsWaitingForKey() const {
 454       MOZ_ASSERT(mOwner->OnTaskQueue());
 455       return mWaitingForKey && mDecodeRequest.Exists();
 456     }
 457
 458     // MediaDataDecoder handler's variables.
 459     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest;
 460     bool mFlushing;  // True if flush is in action.
 461     // Set to true if the last operation run on the decoder was a flush.
 462     bool mFlushed;
 463     RefPtr<SharedShutdownPromiseHolder> mShutdownPromise;
 464
 465     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest;
 466     DrainState mDrainState;
 467     bool HasPendingDrain() const { return mDrainState != DrainState::None; }
 468     bool HasCompletedDrain() const {
 469       return mDrainState == DrainState::DrainCompleted ||
 470              mDrainState == DrainState::DrainAborted;
 471     }
 472     void RequestDrain() {
 473       MOZ_RELEASE_ASSERT(mDrainState == DrainState::None);
 474       mDrainState = DrainState::DrainRequested;
 475     }
 476
 477     // Track decoding error and fail when we hit the limit.
 478     uint32_t mNumOfConsecutiveDecodingError;
 479     uint32_t mMaxConsecutiveDecodingError;
 480
 481     // Track RDD or GPU process crashes and fail when we hit the limit.
 482     uint32_t mNumOfConsecutiveRDDOrGPUCrashes;
 483     uint32_t mMaxConsecutiveRDDOrGPUCrashes;
 484
 485     // Track Utility process crashes and fail when we hit the limit.
 486     uint32_t mNumOfConsecutiveUtilityCrashes;
 487     uint32_t mMaxConsecutiveUtilityCrashes;
 488
 489     // Set when we haven't yet decoded the first frame.
 490     // Cleared once the first frame has been decoded.
 491     // This is used to determine, upon error, if we should try again to decode
 492     // the frame, or skip to the next keyframe.
 493     Maybe<media::TimeUnit> mFirstFrameTime;
 494
 495     Maybe<MediaResult> mError;
 496     bool HasFatalError() const {
 497       if (!mError.isSome()) {
 498         return false;
 499       }
 500       if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
 501         // Allow decode errors to be non-fatal, but give up
 502         // if we have too many, or if warnings should be treated as errors.
 503         return mNumOfConsecutiveDecodingError > mMaxConsecutiveDecodingError ||
 504                StaticPrefs::media_playback_warnings_as_errors();
 505       }
 506       if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
 507         // If the caller asked for a new decoder we shouldn't treat
 508         // it as fatal.
 509         return false;
 510       }
 511       if (mError.ref() ==
 512           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR) {
 513         // Allow RDD crashes to be non-fatal, but give up
 514         // if we have too many, or if warnings should be treated as errors.
 515         return mNumOfConsecutiveRDDOrGPUCrashes >
 516                    mMaxConsecutiveRDDOrGPUCrashes ||
 517                StaticPrefs::media_playback_warnings_as_errors();
 518       }
 519       if (mError.ref() ==
 520           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR) {
 521         bool tooManyConsecutiveCrashes =
 522             mNumOfConsecutiveUtilityCrashes > mMaxConsecutiveUtilityCrashes;
 523         // TODO: Telemetry?
 524         return tooManyConsecutiveCrashes ||
 525                StaticPrefs::media_playback_warnings_as_errors();
 526       }
 527       if (mError.ref() ==
 528           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR) {
 529         return false;
 530       }
 531       // All other error types are fatal
 532       return true;
 533     }
 534
 535     // If set, all decoded samples prior mTimeThreshold will be dropped.
 536     // Used for internal seeking when a change of stream is detected or when
 537     // encountering data discontinuity.
 538     Maybe<InternalSeekTarget> mTimeThreshold;
 539     // Time of last decoded sample returned.
 540     Maybe<media::TimeInterval> mLastDecodedSampleTime;
 541
 542     // Decoded samples returned my mDecoder awaiting being returned to
 543     // state machine upon request.
 544     nsTArray<RefPtr<MediaData>> mOutput;
 545     uint64_t mNumSamplesInput;
 546     uint64_t mNumSamplesOutput;
 547     uint64_t mNumSamplesOutputTotal;
 548     uint64_t mNumSamplesSkippedTotal;
 549
 550     // These get overridden in the templated concrete class.
 551     // Indicate if we have a pending promise for decoded frame.
 552     // Rejecting the promise will stop the reader from decoding ahead.
 553     virtual bool HasPromise() const = 0;
 554     virtual void RejectPromise(const MediaResult& aError,
 555                                const char* aMethodName) = 0;
 556
 557     // Clear track demuxer related data.
 558     void ResetDemuxer() {
 559       mDemuxRequest.DisconnectIfExists();
 560       mSeekRequest.DisconnectIfExists();
 561       mTrackDemuxer->Reset();
 562       mQueuedSamples.Clear();
 563     }
 564
 565     // Flush the decoder if present and reset decoding related data.
 566     // Following a flush, the decoder is ready to accept any new data.
 567     void Flush();
 568
 569     bool CancelWaitingForKey() {
 570       if (!mWaitingForKey) {
 571         return false;
 572       }
 573       mWaitingForKey = false;
 574       if (IsWaitingForData() || !HasWaitingPromise()) {
 575         return false;
 576       }
 577       mWaitingPromise.Resolve(mType, __func__);
 578       return true;
 579     }
 580
 581     // Reset the state of the DecoderData, clearing all queued frames
 582     // (pending demuxed and decoded).
 583     // The track demuxer is *not* reset.
 584     void ResetState() {
 585       MOZ_ASSERT(mOwner->OnTaskQueue());
 586       mDemuxEOS = false;
 587       mWaitingForDataStartTime.reset();
 588       mQueuedSamples.Clear();
 589       mDecodeRequest.DisconnectIfExists();
 590       mDrainRequest.DisconnectIfExists();
 591       mDrainState = DrainState::None;
 592       CancelWaitingForKey();
 593       mTimeThreshold.reset();
 594       mLastDecodedSampleTime.reset();
 595       mOutput.Clear();
 596       mNumSamplesInput = 0;
 597       mNumSamplesOutput = 0;
 598       mSizeOfQueue = 0;
 599       mNextStreamSourceID.reset();
 600       if (!HasFatalError()) {
 601         mError.reset();
 602       }
 603     }
 604
 605     bool HasInternalSeekPending() const {
 606       return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
 607     }
 608
 609     // Return the current TrackInfo in the stream. If the stream content never
 610     // changed since AsyncReadMetadata was called then the TrackInfo used is
 611     // mOriginalInfo, other it will be mInfo. The later case is only ever true
 612     // with MSE or the WebMDemuxer.
 613     const TrackInfo* GetCurrentInfo() const {
 614       if (mInfo) {
 615         return *mInfo;
 616       }
 617       return mOriginalInfo.get();
 618     }
 619     // Return the current TrackInfo updated as per the decoder output.
 620     // Typically for audio, the number of channels and/or sampling rate can vary
 621     // between what was found in the metadata and what the decoder returned.
 622     const TrackInfo* GetWorkingInfo() const { return mWorkingInfo.get(); }
 623     bool IsEncrypted() const { return GetCurrentInfo()->mCrypto.IsEncrypted(); }
 624
 625     // Used by the MDSM for logging purposes.
 626     Atomic<size_t> mSizeOfQueue;
 627     // Used by the MDSM to determine if video decoding is hardware accelerated.
 628     // This value is updated after a frame is successfully decoded.
 629     Atomic<bool> mIsHardwareAccelerated;
 630     // Sample format monitoring.
 631     uint32_t mLastStreamSourceID;
 632     Maybe<uint32_t> mNextStreamSourceID;
 633     media::TimeIntervals mTimeRanges;
 634     Maybe<media::TimeUnit> mLastTimeRangesEnd;
 635     // TrackInfo as first discovered during ReadMetadata.
 636     UniquePtr<TrackInfo> mOriginalInfo;
 637     // Written exclusively on the TaskQueue, can be read on MDSM's TaskQueue.
 638     // Must be read with parent's mutex held.
 639     UniquePtr<TrackInfo> mWorkingInfo;
 640     RefPtr<TrackInfoSharedPtr> mInfo;
 641     Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
 642     // Use NullDecoderModule or not.
 643     bool mIsNullDecode;
 644     bool mHardwareDecodingDisabled;
 645     // Whether we have reported hardware decoding support for video. Used only
 646     // on reader's task queue,
 647     bool mHasReportedVideoHardwareSupportTelemtry = false;
 648
 649     class {
 650      public:
 651       float Mean() const { return mMean; }
 652
 653       void Update(const media::TimeUnit& aValue) {
 654         if (aValue == media::TimeUnit::Zero()) {
 655           return;
 656         }
 657         mMean += static_cast<float>((1.0f / aValue.ToSeconds() - mMean) /
 658                                     static_cast<double>(++mCount));
 659       }
 660
 661       void Reset() {
 662         mMean = 0;
 663         mCount = 0;
 664       }
 665
 666      private:
 667       float mMean = 0;
 668       uint64_t mCount = 0;
 669     } mMeanRate;
 670   };
 671
 672   template <typename Type>
 673   class DecoderDataWithPromise : public DecoderData {
 674    public:
 675     DecoderDataWithPromise(MediaFormatReader* aOwner, MediaData::Type aType,
 676                            uint32_t aNumOfMaxError)
 677         : DecoderData(aOwner, aType, aNumOfMaxError), mHasPromise(false) {
 678       DecoderDoctorLogger::LogConstructionAndBase(
 679           "MediaFormatReader::DecoderDataWithPromise", this,
 680           "MediaFormatReader::DecoderData",
 681           static_cast<const MediaFormatReader::DecoderData*>(this));
 682     }
 683
 684     ~DecoderDataWithPromise() {
 685       DecoderDoctorLogger::LogDestruction(
 686           "MediaFormatReader::DecoderDataWithPromise", this);
 687     }
 688
 689     bool HasPromise() const override { return mHasPromise; }
 690
 691     RefPtr<DataPromise<Type>> EnsurePromise(const char* aMethodName) {
 692       MOZ_ASSERT(mOwner->OnTaskQueue());
 693       mHasPromise = true;
 694       return mPromise.Ensure(aMethodName);
 695     }
 696
 697     void ResolvePromise(Type* aData, const char* aMethodName) {
 698       MOZ_ASSERT(mOwner->OnTaskQueue());
 699       mPromise.Resolve(aData, aMethodName);
 700       mHasPromise = false;
 701     }
 702
 703     void RejectPromise(const MediaResult& aError,
 704                        const char* aMethodName) override {
 705       MOZ_ASSERT(mOwner->OnTaskQueue());
 706       mPromise.Reject(aError, aMethodName);
 707       mHasPromise = false;
 708     }
 709
 710    private:
 711     MozPromiseHolder<DataPromise<Type>> mPromise;
 712     Atomic<bool> mHasPromise;
 713   };
 714
 715   // Decode task queue.
 716   RefPtr<TaskQueue> mTaskQueue;
 717
 718   DecoderDataWithPromise<AudioData> mAudio;
 719   DecoderDataWithPromise<VideoData> mVideo;
 720
 721   Watchable<bool> mWorkingInfoChanged;
 722   WatchManager<MediaFormatReader> mWatchManager;
 723   bool mIsWatchingWorkingInfo;
 724
 725   // Returns true when the decoder for this track needs input.
 726   bool NeedInput(DecoderData& aDecoder);
 727
 728   DecoderData& GetDecoderData(TrackType aTrack);
 729
 730   // Demuxer objects.
 731   class DemuxerProxy;
 732   UniquePtr<DemuxerProxy> mDemuxer;
 733   bool mDemuxerInitDone;
 734   void OnDemuxerInitDone(const MediaResult& aResult);
 735   void OnDemuxerInitFailed(const MediaResult& aError);
 736   MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
 737   MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise;
 738   bool mPendingNotifyDataArrived;
 739   void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
 740
 741   void DoDemuxVideo();
 742   void OnVideoDemuxCompleted(
 743       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 744   void OnVideoDemuxFailed(const MediaResult& aError) {
 745     OnDemuxFailed(TrackType::kVideoTrack, aError);
 746   }
 747
 748   void DoDemuxAudio();
 749   void OnAudioDemuxCompleted(
 750       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 751   void OnAudioDemuxFailed(const MediaResult& aError) {
 752     OnDemuxFailed(TrackType::kAudioTrack, aError);
 753   }
 754
 755   void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
 756   MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise>
 757       mSkipRequest;
 758   void VideoSkipReset(uint32_t aSkipped);
 759   void OnVideoSkipCompleted(uint32_t aSkipped);
 760   void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
 761
 762   // The last number of decoded output frames that we've reported to
 763   // MediaDecoder::NotifyDecoded(). We diff the number of output video
 764   // frames every time that DecodeVideoData() is called, and report the
 765   // delta there.
 766   uint64_t mLastReportedNumDecodedFrames;
 767
 768   // Timestamp of the previous decoded keyframe, in microseconds.
 769   int64_t mPreviousDecodedKeyframeTime_us;
 770   // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
 771   static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
 772
 773   RefPtr<layers::KnowsCompositor> mKnowsCompositor;
 774
 775   // Metadata objects
 776   // True if we've read the streams' metadata.
 777   bool mInitDone;
 778   MozPromiseHolder<MetadataPromise> mMetadataPromise;
 779   bool IsEncrypted() const;
 780
 781   // Set to true if any of our track buffers may be blocking.
 782   bool mTrackDemuxersMayBlock;
 783
 784   // Seeking objects.
 785   void SetSeekTarget(const SeekTarget& aTarget);
 786   bool IsSeeking() const { return mPendingSeekTime.isSome(); }
 787   bool IsVideoOnlySeeking() const {
 788     return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
 789   }
 790   bool IsAudioOnlySeeking() const {
 791     return IsSeeking() && mOriginalSeekTarget.IsAudioOnly();
 792   }
 793   void ScheduleSeek();
 794   void AttemptSeek();
 795   void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
 796   void DoVideoSeek();
 797   void OnVideoSeekCompleted(media::TimeUnit aTime);
 798   void OnVideoSeekFailed(const MediaResult& aError);
 799   bool mSeekScheduled;
 800
 801   void DoAudioSeek();
 802   void OnAudioSeekCompleted(media::TimeUnit aTime);
 803   void OnAudioSeekFailed(const MediaResult& aError);
 804   // The SeekTarget that was last given to Seek()
 805   SeekTarget mOriginalSeekTarget;
 806   // Temporary seek information while we wait for the data
 807   Maybe<media::TimeUnit> mFallbackSeekTime;
 808   Maybe<media::TimeUnit> mPendingSeekTime;
 809   MozPromiseHolder<SeekPromise> mSeekPromise;
 810
 811   RefPtr<VideoFrameContainer> mVideoFrameContainer;
 812   layers::ImageContainer* GetImageContainer();
 813
 814   RefPtr<CDMProxy> mCDMProxy;
 815
 816   RefPtr<GMPCrashHelper> mCrashHelper;
 817
 818   void SetNullDecode(TrackType aTrack, bool aIsNullDecode);
 819
 820   class DecoderFactory;
 821   UniquePtr<DecoderFactory> mDecoderFactory;
 822
 823   class ShutdownPromisePool;
 824   UniquePtr<ShutdownPromisePool> mShutdownPromisePool;
 825
 826   MediaEventListener mOnTrackWaitingForKeyListener;
 827
 828   void OnFirstDemuxCompleted(
 829       TrackInfo::TrackType aType,
 830       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 831
 832   void OnFirstDemuxFailed(TrackInfo::TrackType aType,
 833                           const MediaResult& aError);
 834
 835   void MaybeResolveMetadataPromise();
 836
 837   // Stores presentation info required for playback.
 838   MediaInfo mInfo;
 839
 840   UniquePtr<MetadataTags> mTags;
 841
 842   // A flag indicating if the start time is known or not.
 843   bool mHasStartTime = false;
 844
 845   void ShutdownDecoder(TrackType aTrack);
 846   RefPtr<ShutdownPromise> TearDownDecoders();
 847
 848   bool mShutdown = false;
 849
 850   // Buffered range.
 851   Canonical<media::TimeIntervals> mBuffered;
 852
 853   // Used to send TimedMetadata to the listener.
 854   TimedMetadataEventProducer mTimedMetadataEvent;
 855
 856   // Notify if this media is not seekable.
 857   MediaEventProducer<void> mOnMediaNotSeekable;
 858
 859   // Notify if we are waiting for a decryption key.
 860   MediaEventProducer<TrackInfo::TrackType> mOnTrackWaitingForKey;
 861
 862   MediaEventProducer<nsTArray<uint8_t>, nsString> mOnEncrypted;
 863
 864   MediaEventProducer<void> mOnWaitingForKey;
 865
 866   MediaEventProducer<MediaResult> mOnDecodeWarning;
 867
 868   MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark;
 869
 870   MediaEventProducer<VideoInfo, AudioInfo> mTrackInfoUpdatedEvent;
 871
 872   RefPtr<FrameStatistics> mFrameStats;
 873
 874   // Used in bug 1393399 for telemetry.
 875   const MediaDecoderOwnerID mMediaDecoderOwnerID;
 876
 877   bool ResolveSetCDMPromiseIfDone(TrackType aTrack);
 878   void PrepareToSetCDMForTrack(TrackType aTrack);
 879   MozPromiseHolder<SetCDMPromise> mSetCDMPromise;
 880   TrackSet mSetCDMForTracks{};
 881   bool IsDecoderWaitingForCDM(TrackType aTrack);
 882
 883   void GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo);
 884
 885   // Only be used on Windows when the media engine playback is enabled.
 886   Maybe<uint64_t> mMediaEngineId;
 887
 888   const Maybe<TrackingId> mTrackingId;
 889
 890   // The start time of reading the metdata and how long does it take. This
 891   // measurement includes the time of downloading media resource over the
 892   // internet.
 893   Maybe<TimeStamp> mReadMetadataStartTime;
 894   TimeDuration mReadMetaDataTime;
 895
 896   // The total amount of time we have been waiting for the video data due to
 897   // lacking of data.
 898   TimeDuration mTotalWaitingForVideoDataTime;
 899 };
 900
 901 }  // namespace mozilla
 902
 903 #endif