dom/media/MediaFormatReader.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #if !defined(MediaFormatReader_h_)
   8 #  define MediaFormatReader_h_
   9
  10 #  include "FrameStatistics.h"
  11 #  include "MediaDataDemuxer.h"
  12 #  include "MediaEventSource.h"
  13 #  include "MediaMetadataManager.h"
  14 #  include "MediaPromiseDefs.h"
  15 #  include "PlatformDecoderModule.h"
  16 #  include "SeekTarget.h"
  17 #  include "mozilla/Atomics.h"
  18 #  include "mozilla/Maybe.h"
  19 #  include "mozilla/MozPromise.h"
  20 #  include "mozilla/Mutex.h"
  21 #  include "mozilla/StateMirroring.h"
  22 #  include "mozilla/StaticPrefs_media.h"
  23 #  include "mozilla/TaskQueue.h"
  24 #  include "mozilla/TimeStamp.h"
  25 #  include "mozilla/ThreadSafeWeakPtr.h"
  26 #  include "mozilla/dom/MediaDebugInfoBinding.h"
  27
  28 namespace mozilla {
  29
  30 class CDMProxy;
  31 class GMPCrashHelper;
  32 class MediaResource;
  33 class VideoFrameContainer;
  34
  35 struct WaitForDataRejectValue {
  36   enum Reason { SHUTDOWN, CANCELED };
  37
  38   WaitForDataRejectValue(MediaData::Type aType, Reason aReason)
  39       : mType(aType), mReason(aReason) {}
  40   MediaData::Type mType;
  41   Reason mReason;
  42 };
  43
  44 struct SeekRejectValue {
  45   MOZ_IMPLICIT SeekRejectValue(const MediaResult& aError)
  46       : mType(MediaData::Type::NULL_DATA), mError(aError) {}
  47   MOZ_IMPLICIT SeekRejectValue(nsresult aResult)
  48       : mType(MediaData::Type::NULL_DATA), mError(aResult) {}
  49   SeekRejectValue(MediaData::Type aType, const MediaResult& aError)
  50       : mType(aType), mError(aError) {}
  51   MediaData::Type mType;
  52   MediaResult mError;
  53 };
  54
  55 struct MetadataHolder {
  56   UniquePtr<MediaInfo> mInfo;
  57   UniquePtr<MetadataTags> mTags;
  58 };
  59
  60 using MediaDecoderOwnerID = void*;
  61
  62 struct MOZ_STACK_CLASS MediaFormatReaderInit {
  63   MediaResource* mResource = nullptr;
  64   VideoFrameContainer* mVideoFrameContainer = nullptr;
  65   FrameStatistics* mFrameStats = nullptr;
  66   already_AddRefed<layers::KnowsCompositor> mKnowsCompositor;
  67   already_AddRefed<GMPCrashHelper> mCrashHelper;
  68   // Used in bug 1393399 for temporary telemetry.
  69   MediaDecoderOwnerID mMediaDecoderOwnerID = nullptr;
  70   Maybe<TrackingId> mTrackingId;
  71 };
  72
  73 DDLoggedTypeDeclName(MediaFormatReader);
  74
  75 class MediaFormatReader final
  76     : public SupportsThreadSafeWeakPtr<MediaFormatReader>,
  77       public DecoderDoctorLifeLogger<MediaFormatReader> {
  78   static const bool IsExclusive = true;
  79   using TrackType = TrackInfo::TrackType;
  80   using NotifyDataArrivedPromise = MozPromise<bool, MediaResult, IsExclusive>;
  81
  82  public:
  83   MOZ_DECLARE_REFCOUNTED_TYPENAME(MediaFormatReader)
  84
  85   using TrackSet = EnumSet<TrackInfo::TrackType>;
  86   using MetadataPromise = MozPromise<MetadataHolder, MediaResult, IsExclusive>;
  87
  88   template <typename Type>
  89   using DataPromise = MozPromise<RefPtr<Type>, MediaResult, IsExclusive>;
  90   using AudioDataPromise = DataPromise<AudioData>;
  91   using VideoDataPromise = DataPromise<VideoData>;
  92
  93   using SeekPromise = MozPromise<media::TimeUnit, SeekRejectValue, IsExclusive>;
  94
  95   // Note that, conceptually, WaitForData makes sense in a non-exclusive sense.
  96   // But in the current architecture it's only ever used exclusively (by MDSM),
  97   // so we mark it that way to verify our assumptions. If you have a use-case
  98   // for multiple WaitForData consumers, feel free to flip the exclusivity here.
  99   using WaitForDataPromise =
 100       MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>;
 101
 102   MediaFormatReader(MediaFormatReaderInit& aInit, MediaDataDemuxer* aDemuxer);
 103   virtual ~MediaFormatReader();
 104
 105   // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE
 106   // on failure.
 107   nsresult Init();
 108
 109   size_t SizeOfVideoQueueInFrames();
 110   size_t SizeOfAudioQueueInFrames();
 111
 112   // Requests one video sample from the reader.
 113   RefPtr<VideoDataPromise> RequestVideoData(
 114       const media::TimeUnit& aTimeThreshold,
 115       bool aRequestNextVideoKeyFrame = false);
 116
 117   // Requests one audio sample from the reader.
 118   //
 119   // The decode should be performed asynchronously, and the promise should
 120   // be resolved when it is complete.
 121   RefPtr<AudioDataPromise> RequestAudioData();
 122
 123   // The default implementation of AsyncReadMetadata is implemented in terms of
 124   // synchronous ReadMetadata() calls. Implementations may also
 125   // override AsyncReadMetadata to create a more proper async implementation.
 126   RefPtr<MetadataPromise> AsyncReadMetadata();
 127
 128   // Fills aInfo with the latest cached data required to present the media,
 129   // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded.
 130   void ReadUpdatedMetadata(MediaInfo* aInfo);
 131
 132   RefPtr<SeekPromise> Seek(const SeekTarget& aTarget);
 133
 134   // Called once new data has been cached by the MediaResource.
 135   // mBuffered should be recalculated and updated accordingly.
 136   void NotifyDataArrived();
 137
 138   // Update ID for the external playback engine. Currently it's only used on
 139   // Windows when the media engine playback is enabled.
 140   void UpdateMediaEngineId(uint64_t aMediaEngineId);
 141
 142   // This function will be called if the media key is set before playback
 143   // starts, indicating the playback should be encrypted.
 144   void SetEncryptedCustomIdent();
 145
 146   bool IsEncryptedCustomIdent() const { return mEncryptedCustomIdent; }
 147
 148  protected:
 149   // Recomputes mBuffered.
 150   void UpdateBuffered();
 151
 152  public:
 153   // Called by MDSM in dormant state to release resources allocated by this
 154   // reader. The reader can resume decoding by calling Seek() to a specific
 155   // position.
 156   void ReleaseResources();
 157
 158   bool OnTaskQueue() const { return OwnerThread()->IsCurrentThreadIn(); }
 159
 160   // Resets all state related to decoding, emptying all buffers etc.
 161   // Cancels all pending Request*Data() request callbacks, rejects any
 162   // outstanding seek promises, and flushes the decode pipeline. The
 163   // decoder must not call any of the callbacks for outstanding
 164   // Request*Data() calls after this is called. Calls to Request*Data()
 165   // made after this should be processed as usual.
 166   //
 167   // Normally this call preceedes a Seek() call, or shutdown.
 168   //
 169   // aParam is a set of TrackInfo::TrackType enums specifying which
 170   // queues need to be reset, defaulting to both audio and video tracks.
 171   nsresult ResetDecode(const TrackSet& aTracks);
 172
 173   // Destroys the decoding state. The reader cannot be made usable again.
 174   // This is different from ReleaseMediaResources() as it is irreversable,
 175   // whereas ReleaseMediaResources() is.  Must be called on the decode
 176   // thread.
 177   RefPtr<ShutdownPromise> Shutdown();
 178
 179   // Returns true if this decoder reader uses hardware accelerated video
 180   // decoding.
 181   bool VideoIsHardwareAccelerated() const;
 182
 183   // By default, the state machine polls the reader once per second when it's
 184   // in buffering mode. Some readers support a promise-based mechanism by which
 185   // they notify the state machine when the data arrives.
 186   bool IsWaitForDataSupported() const { return true; }
 187
 188   RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType);
 189
 190   // The MediaDecoderStateMachine uses various heuristics that assume that
 191   // raw media data is arriving sequentially from a network channel. This
 192   // makes sense in the <video src="foo"> case, but not for more advanced use
 193   // cases like MSE.
 194   bool UseBufferingHeuristics() const { return mTrackDemuxersMayBlock; }
 195
 196   RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy);
 197
 198   // Requests that the MediaFormatReader populates aInfo with debug information.
 199   // This may be done asynchronously, and aInfo should *not* be accessed by the
 200   // caller until the returned promise is resolved or rejected.
 201   RefPtr<GenericPromise> RequestDebugInfo(
 202       dom::MediaFormatReaderDebugInfo& aInfo);
 203
 204   // Switch the video decoder to NullDecoderModule. It might takes effective
 205   // since a few samples later depends on how much demuxed samples are already
 206   // queued in the original video decoder.
 207   void SetVideoNullDecode(bool aIsNullDecode);
 208
 209   void UpdateCompositor(already_AddRefed<layers::KnowsCompositor>);
 210
 211   void UpdateDuration(const media::TimeUnit& aDuration) {
 212     MOZ_ASSERT(OnTaskQueue());
 213     UpdateBuffered();
 214   }
 215
 216   AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() {
 217     return &mBuffered;
 218   }
 219
 220   TaskQueue* OwnerThread() const { return mTaskQueue; }
 221
 222   TimedMetadataEventSource& TimedMetadataEvent() { return mTimedMetadataEvent; }
 223
 224   // Notified by the OggDemuxer during playback when chained ogg is detected.
 225   MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; }
 226
 227   TimedMetadataEventProducer& TimedMetadataProducer() {
 228     return mTimedMetadataEvent;
 229   }
 230
 231   MediaEventProducer<void>& MediaNotSeekableProducer() {
 232     return mOnMediaNotSeekable;
 233   }
 234
 235   // Notified if the reader can't decode a sample due to a missing decryption
 236   // key.
 237   MediaEventSource<TrackInfo::TrackType>& OnTrackWaitingForKey() {
 238     return mOnTrackWaitingForKey;
 239   }
 240
 241   MediaEventProducer<TrackInfo::TrackType>& OnTrackWaitingForKeyProducer() {
 242     return mOnTrackWaitingForKey;
 243   }
 244
 245   MediaEventSource<nsTArray<uint8_t>, nsString>& OnEncrypted() {
 246     return mOnEncrypted;
 247   }
 248
 249   MediaEventSource<void>& OnWaitingForKey() { return mOnWaitingForKey; }
 250
 251   MediaEventSource<MediaResult>& OnDecodeWarning() { return mOnDecodeWarning; }
 252
 253   MediaEventSource<VideoInfo>& OnStoreDecoderBenchmark() {
 254     return mOnStoreDecoderBenchmark;
 255   }
 256
 257   MediaEventProducer<VideoInfo, AudioInfo>& OnTrackInfoUpdatedEvent() {
 258     return mTrackInfoUpdatedEvent;
 259   }
 260
 261  private:
 262   bool HasVideo() const { return mVideo.mTrackDemuxer; }
 263   bool HasAudio() const { return mAudio.mTrackDemuxer; }
 264
 265   bool IsWaitingOnCDMResource();
 266
 267   bool InitDemuxer();
 268   // Notify the track demuxers that new data has been received.
 269   void NotifyTrackDemuxers();
 270   void ReturnOutput(MediaData* aData, TrackType aTrack);
 271
 272   // Enqueues a task to call Update(aTrack) on the decoder task queue.
 273   // Lock for corresponding track must be held.
 274   void ScheduleUpdate(TrackType aTrack);
 275   void Update(TrackType aTrack);
 276   // Handle actions should more data be received.
 277   // Returns true if no more action is required.
 278   bool UpdateReceivedNewData(TrackType aTrack);
 279   // Called when new samples need to be demuxed.
 280   void RequestDemuxSamples(TrackType aTrack);
 281   // Handle demuxed samples by the input behavior.
 282   void HandleDemuxedSamples(TrackType aTrack,
 283                             FrameStatistics::AutoNotifyDecoded& aA);
 284   // Decode any pending already demuxed samples.
 285   void DecodeDemuxedSamples(TrackType aTrack, MediaRawData* aSample);
 286
 287   struct InternalSeekTarget {
 288     InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
 289         : mTime(aTime),
 290           mDropTarget(aDropTarget),
 291           mWaiting(false),
 292           mHasSeeked(false) {}
 293
 294     media::TimeUnit Time() const { return mTime.mStart; }
 295     media::TimeUnit EndTime() const { return mTime.mEnd; }
 296     bool Contains(const media::TimeUnit& aTime) const {
 297       return mTime.Contains(aTime);
 298     }
 299
 300     media::TimeInterval mTime;
 301     bool mDropTarget;
 302     bool mWaiting;
 303     bool mHasSeeked;
 304   };
 305
 306   // Perform an internal seek to aTime. If aDropTarget is true then
 307   // the first sample past the target will be dropped.
 308   void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
 309   // Return the end time of the internal seek target if it exists. Otherwise,
 310   // return infinity.
 311   media::TimeUnit GetInternalSeekTargetEndTime() const;
 312
 313   // Drain the current decoder.
 314   void DrainDecoder(TrackType aTrack);
 315   void NotifyNewOutput(TrackType aTrack,
 316                        MediaDataDecoder::DecodedData&& aResults);
 317   void NotifyError(TrackType aTrack, const MediaResult& aError);
 318   void NotifyWaitingForData(TrackType aTrack);
 319   void NotifyWaitingForKey(TrackType aTrack);
 320   void NotifyEndOfStream(TrackType aTrack);
 321
 322   void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
 323
 324   // Initializes mLayersBackendType if possible.
 325   void InitLayersBackendType();
 326
 327   void Reset(TrackType aTrack);
 328   void DropDecodedSamples(TrackType aTrack);
 329
 330   // Return a target timeunit which the reader should skip to, this would be
 331   // either the timethreshold we pass, or the time of the next keyframe. Return
 332   // nothing if we don't need to skip.
 333   // @param aTimeThreshold
 334   // The time that we expect the time of next video frame should be or go beyond
 335   // @param aRequestNextVideoKeyFrame
 336   // If true and the next keyframe's time is larger than aTimeThreshold, skip to
 337   // the next keyframe time instead of aTimeThreshold.
 338   Maybe<media::TimeUnit> ShouldSkip(media::TimeUnit aTimeThreshold,
 339                                     bool aRequestNextVideoKeyFrame);
 340
 341   void SetVideoDecodeThreshold();
 342
 343   size_t SizeOfQueue(TrackType aTrack);
 344
 345   // Fire a new OnStoreDecoderBenchmark event that will create new
 346   // storage of the decoder benchmark.
 347   // This is called only on TaskQueue.
 348   void NotifyDecoderBenchmarkStore();
 349
 350   void NotifyTrackInfoUpdated();
 351
 352   enum class DrainState {
 353     None,
 354     DrainRequested,
 355     Draining,
 356     PartialDrainPending,
 357     DrainCompleted,
 358     DrainAborted,
 359   };
 360
 361   class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise> {
 362     NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder)
 363    private:
 364     ~SharedShutdownPromiseHolder() = default;
 365   };
 366
 367   struct DecoderData {
 368     DecoderData(MediaFormatReader* aOwner, MediaData::Type aType,
 369                 uint32_t aNumOfMaxError)
 370         : mOwner(aOwner),
 371           mType(aType),
 372           mMutex("DecoderData"),
 373           mDescription("uninitialized"),
 374           mProcessName(""),
 375           mCodecName(""),
 376           mUpdateScheduled(false),
 377           mDemuxEOS(false),
 378           mWaitingForDataStartTime(Nothing()),
 379           mWaitingForKey(false),
 380           mReceivedNewData(false),
 381           mFlushing(false),
 382           mFlushed(true),
 383           mDrainState(DrainState::None),
 384           mNumOfConsecutiveDecodingError(0),
 385           mMaxConsecutiveDecodingError(aNumOfMaxError),
 386           mNumOfConsecutiveRDDOrGPUCrashes(0),
 387           mMaxConsecutiveRDDOrGPUCrashes(
 388               StaticPrefs::media_rdd_process_max_crashes()),
 389           mNumOfConsecutiveUtilityCrashes(0),
 390           mMaxConsecutiveUtilityCrashes(
 391               StaticPrefs::media_utility_process_max_crashes()),
 392           mFirstFrameTime(Some(media::TimeUnit::Zero())),
 393           mNumSamplesInput(0),
 394           mNumSamplesOutput(0),
 395           mNumSamplesOutputTotal(0),
 396           mNumSamplesSkippedTotal(0),
 397           mSizeOfQueue(0),
 398           mIsHardwareAccelerated(false),
 399           mLastStreamSourceID(UINT32_MAX),
 400           mIsNullDecode(false),
 401           mHardwareDecodingDisabled(false) {
 402       DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderData",
 403                                            this);
 404     }
 405
 406     ~DecoderData() {
 407       DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderData",
 408                                           this);
 409     }
 410
 411     MediaFormatReader* mOwner;
 412     // Disambiguate Audio vs Video.
 413     MediaData::Type mType;
 414     RefPtr<MediaTrackDemuxer> mTrackDemuxer;
 415     // TaskQueue on which decoder can choose to decode.
 416     // Only non-null up until the decoder is created.
 417     RefPtr<TaskQueue> mTaskQueue;
 418
 419     // Mutex protecting mDescription, mDecoder, mTrackDemuxer, mWorkingInfo,
 420     // mProcessName and mCodecName as those can be read outside the TaskQueue.
 421     // They are only written on the TaskQueue however, as such mMutex doesn't
 422     // need to be held when those members are read on the TaskQueue.
 423     Mutex mMutex MOZ_UNANNOTATED;
 424     // The platform decoder.
 425     RefPtr<MediaDataDecoder> mDecoder;
 426     nsCString mDescription;
 427     nsCString mProcessName;
 428     nsCString mCodecName;
 429     void ShutdownDecoder();
 430
 431     // Only accessed from reader's task queue.
 432     bool mUpdateScheduled;
 433     bool mDemuxEOS;
 434     Maybe<TimeStamp> mWaitingForDataStartTime;
 435     bool mWaitingForKey;
 436     bool mReceivedNewData;
 437     UniquePtr<PerformanceRecorderMulti<PlaybackStage>> mDecodePerfRecorder;
 438
 439     // Pending seek.
 440     MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
 441
 442     // Queued demux samples waiting to be decoded.
 443     nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
 444     MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
 445     // A WaitingPromise is pending if the demuxer is waiting for data or
 446     // if the decoder is waiting for a key.
 447     MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
 448     bool HasWaitingPromise() const {
 449       MOZ_ASSERT(mOwner->OnTaskQueue());
 450       return !mWaitingPromise.IsEmpty();
 451     }
 452
 453     bool IsWaitingForData() const {
 454       MOZ_ASSERT(mOwner->OnTaskQueue());
 455       return !!mWaitingForDataStartTime;
 456     }
 457
 458     bool IsWaitingForKey() const {
 459       MOZ_ASSERT(mOwner->OnTaskQueue());
 460       return mWaitingForKey && mDecodeRequest.Exists();
 461     }
 462
 463     // MediaDataDecoder handler's variables.
 464     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest;
 465     bool mFlushing;  // True if flush is in action.
 466     // Set to true if the last operation run on the decoder was a flush.
 467     bool mFlushed;
 468     RefPtr<SharedShutdownPromiseHolder> mShutdownPromise;
 469
 470     MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest;
 471     DrainState mDrainState;
 472     bool HasPendingDrain() const { return mDrainState != DrainState::None; }
 473     bool HasCompletedDrain() const {
 474       return mDrainState == DrainState::DrainCompleted ||
 475              mDrainState == DrainState::DrainAborted;
 476     }
 477     void RequestDrain() {
 478       MOZ_RELEASE_ASSERT(mDrainState == DrainState::None);
 479       mDrainState = DrainState::DrainRequested;
 480     }
 481
 482     void StartRecordDecodingPerf(const TrackType aTrack,
 483                                  const MediaRawData* aSample);
 484
 485     // Track decoding error and fail when we hit the limit.
 486     uint32_t mNumOfConsecutiveDecodingError;
 487     uint32_t mMaxConsecutiveDecodingError;
 488
 489     // Track RDD or GPU process crashes and fail when we hit the limit.
 490     uint32_t mNumOfConsecutiveRDDOrGPUCrashes;
 491     uint32_t mMaxConsecutiveRDDOrGPUCrashes;
 492
 493     // Track Utility process crashes and fail when we hit the limit.
 494     uint32_t mNumOfConsecutiveUtilityCrashes;
 495     uint32_t mMaxConsecutiveUtilityCrashes;
 496
 497     // Set when we haven't yet decoded the first frame.
 498     // Cleared once the first frame has been decoded.
 499     // This is used to determine, upon error, if we should try again to decode
 500     // the frame, or skip to the next keyframe.
 501     Maybe<media::TimeUnit> mFirstFrameTime;
 502
 503     Maybe<MediaResult> mError;
 504     bool HasFatalError() const {
 505       if (!mError.isSome()) {
 506         return false;
 507       }
 508       if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
 509         // Allow decode errors to be non-fatal, but give up
 510         // if we have too many, or if warnings should be treated as errors.
 511         return mNumOfConsecutiveDecodingError > mMaxConsecutiveDecodingError ||
 512                StaticPrefs::media_playback_warnings_as_errors();
 513       }
 514       if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
 515         // If the caller asked for a new decoder we shouldn't treat
 516         // it as fatal.
 517         return false;
 518       }
 519       if (mError.ref() ==
 520           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR) {
 521         // Allow RDD crashes to be non-fatal, but give up
 522         // if we have too many, or if warnings should be treated as errors.
 523         return mNumOfConsecutiveRDDOrGPUCrashes >
 524                    mMaxConsecutiveRDDOrGPUCrashes ||
 525                StaticPrefs::media_playback_warnings_as_errors();
 526       }
 527       if (mError.ref() ==
 528           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR) {
 529         bool tooManyConsecutiveCrashes =
 530             mNumOfConsecutiveUtilityCrashes > mMaxConsecutiveUtilityCrashes;
 531         // TODO: Telemetry?
 532         return tooManyConsecutiveCrashes ||
 533                StaticPrefs::media_playback_warnings_as_errors();
 534       }
 535       if (mError.ref() ==
 536           NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR) {
 537         return false;
 538       }
 539       // All other error types are fatal
 540       return true;
 541     }
 542
 543     // If set, all decoded samples prior mTimeThreshold will be dropped.
 544     // Used for internal seeking when a change of stream is detected or when
 545     // encountering data discontinuity.
 546     Maybe<InternalSeekTarget> mTimeThreshold;
 547     // Time of last decoded sample returned.
 548     Maybe<media::TimeInterval> mLastDecodedSampleTime;
 549
 550     // Decoded samples returned my mDecoder awaiting being returned to
 551     // state machine upon request.
 552     nsTArray<RefPtr<MediaData>> mOutput;
 553     uint64_t mNumSamplesInput;
 554     uint64_t mNumSamplesOutput;
 555     uint64_t mNumSamplesOutputTotal;
 556     uint64_t mNumSamplesSkippedTotal;
 557
 558     // These get overridden in the templated concrete class.
 559     // Indicate if we have a pending promise for decoded frame.
 560     // Rejecting the promise will stop the reader from decoding ahead.
 561     virtual bool HasPromise() const = 0;
 562     virtual void RejectPromise(const MediaResult& aError,
 563                                StaticString aMethodName) = 0;
 564
 565     // Clear track demuxer related data.
 566     void ResetDemuxer() {
 567       mDemuxRequest.DisconnectIfExists();
 568       mSeekRequest.DisconnectIfExists();
 569       mTrackDemuxer->Reset();
 570       mQueuedSamples.Clear();
 571     }
 572
 573     // Flush the decoder if present and reset decoding related data.
 574     // Following a flush, the decoder is ready to accept any new data.
 575     void Flush();
 576
 577     bool CancelWaitingForKey() {
 578       if (!mWaitingForKey) {
 579         return false;
 580       }
 581       mWaitingForKey = false;
 582       if (IsWaitingForData() || !HasWaitingPromise()) {
 583         return false;
 584       }
 585       mWaitingPromise.Resolve(mType, __func__);
 586       return true;
 587     }
 588
 589     // Reset the state of the DecoderData, clearing all queued frames
 590     // (pending demuxed and decoded).
 591     // The track demuxer is *not* reset.
 592     void ResetState() {
 593       MOZ_ASSERT(mOwner->OnTaskQueue());
 594       mDemuxEOS = false;
 595       mWaitingForDataStartTime.reset();
 596       mQueuedSamples.Clear();
 597       mDecodeRequest.DisconnectIfExists();
 598       mDrainRequest.DisconnectIfExists();
 599       mDrainState = DrainState::None;
 600       CancelWaitingForKey();
 601       mTimeThreshold.reset();
 602       mLastDecodedSampleTime.reset();
 603       mOutput.Clear();
 604       mNumSamplesInput = 0;
 605       mNumSamplesOutput = 0;
 606       mSizeOfQueue = 0;
 607       mNextStreamSourceID.reset();
 608       if (!HasFatalError()) {
 609         mError.reset();
 610       }
 611     }
 612
 613     bool HasInternalSeekPending() const {
 614       return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
 615     }
 616
 617     // Return the current TrackInfo in the stream. If the stream content never
 618     // changed since AsyncReadMetadata was called then the TrackInfo used is
 619     // mOriginalInfo, other it will be mInfo. The later case is only ever true
 620     // with MSE or the WebMDemuxer.
 621     const TrackInfo* GetCurrentInfo() const {
 622       if (mInfo) {
 623         return *mInfo;
 624       }
 625       return mOriginalInfo.get();
 626     }
 627     // Return the current TrackInfo updated as per the decoder output.
 628     // Typically for audio, the number of channels and/or sampling rate can vary
 629     // between what was found in the metadata and what the decoder returned.
 630     const TrackInfo* GetWorkingInfo() const { return mWorkingInfo.get(); }
 631     bool IsEncrypted() const { return GetCurrentInfo()->mCrypto.IsEncrypted(); }
 632
 633     // Used by the MDSM for logging purposes.
 634     Atomic<size_t> mSizeOfQueue;
 635     // Used by the MDSM to determine if video decoding is hardware accelerated.
 636     // This value is updated after a frame is successfully decoded.
 637     Atomic<bool> mIsHardwareAccelerated;
 638     // Sample format monitoring.
 639     uint32_t mLastStreamSourceID;
 640     Maybe<uint32_t> mNextStreamSourceID;
 641     media::TimeIntervals mTimeRanges;
 642     Maybe<media::TimeUnit> mLastTimeRangesEnd;
 643     // TrackInfo as first discovered during ReadMetadata.
 644     UniquePtr<TrackInfo> mOriginalInfo;
 645     // Written exclusively on the TaskQueue, can be read on MDSM's TaskQueue.
 646     // Must be read with parent's mutex held.
 647     UniquePtr<TrackInfo> mWorkingInfo;
 648     RefPtr<TrackInfoSharedPtr> mInfo;
 649     Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
 650     // Use NullDecoderModule or not.
 651     bool mIsNullDecode;
 652     bool mHardwareDecodingDisabled;
 653     // Whether we have reported hardware decoding support for video. Used only
 654     // on reader's task queue,
 655     bool mHasReportedVideoHardwareSupportTelemtry = false;
 656
 657     class {
 658      public:
 659       float Mean() const { return mMean; }
 660
 661       void Update(const media::TimeUnit& aValue) {
 662         if (aValue == media::TimeUnit::Zero()) {
 663           return;
 664         }
 665         mMean += static_cast<float>((1.0f / aValue.ToSeconds() - mMean) /
 666                                     static_cast<double>(++mCount));
 667       }
 668
 669       void Reset() {
 670         mMean = 0;
 671         mCount = 0;
 672       }
 673
 674      private:
 675       float mMean = 0;
 676       uint64_t mCount = 0;
 677     } mMeanRate;
 678   };
 679
 680   template <typename Type>
 681   class DecoderDataWithPromise : public DecoderData {
 682    public:
 683     DecoderDataWithPromise(MediaFormatReader* aOwner, MediaData::Type aType,
 684                            uint32_t aNumOfMaxError)
 685         : DecoderData(aOwner, aType, aNumOfMaxError), mHasPromise(false) {
 686       DecoderDoctorLogger::LogConstructionAndBase(
 687           "MediaFormatReader::DecoderDataWithPromise", this,
 688           "MediaFormatReader::DecoderData",
 689           static_cast<const MediaFormatReader::DecoderData*>(this));
 690     }
 691
 692     ~DecoderDataWithPromise() {
 693       DecoderDoctorLogger::LogDestruction(
 694           "MediaFormatReader::DecoderDataWithPromise", this);
 695     }
 696
 697     bool HasPromise() const override { return mHasPromise; }
 698
 699     RefPtr<DataPromise<Type>> EnsurePromise(StaticString aMethodName) {
 700       MOZ_ASSERT(mOwner->OnTaskQueue());
 701       mHasPromise = true;
 702       return mPromise.Ensure(aMethodName);
 703     }
 704
 705     void ResolvePromise(Type* aData, StaticString aMethodName) {
 706       MOZ_ASSERT(mOwner->OnTaskQueue());
 707       mPromise.Resolve(aData, aMethodName);
 708       mHasPromise = false;
 709     }
 710
 711     void RejectPromise(const MediaResult& aError,
 712                        StaticString aMethodName) override {
 713       MOZ_ASSERT(mOwner->OnTaskQueue());
 714       mPromise.Reject(aError, aMethodName);
 715       mHasPromise = false;
 716     }
 717
 718    private:
 719     MozPromiseHolder<DataPromise<Type>> mPromise;
 720     Atomic<bool> mHasPromise;
 721   };
 722
 723   // Decode task queue.
 724   RefPtr<TaskQueue> mTaskQueue;
 725
 726   DecoderDataWithPromise<AudioData> mAudio;
 727   DecoderDataWithPromise<VideoData> mVideo;
 728
 729   Watchable<bool> mWorkingInfoChanged;
 730   WatchManager<MediaFormatReader> mWatchManager;
 731   bool mIsWatchingWorkingInfo;
 732
 733   // Returns true when the decoder for this track needs input.
 734   bool NeedInput(DecoderData& aDecoder);
 735
 736   DecoderData& GetDecoderData(TrackType aTrack);
 737
 738   // Demuxer objects.
 739   class DemuxerProxy;
 740   UniquePtr<DemuxerProxy> mDemuxer;
 741   bool mDemuxerInitDone;
 742   void OnDemuxerInitDone(const MediaResult& aResult);
 743   void OnDemuxerInitFailed(const MediaResult& aError);
 744   MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
 745   MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise;
 746   bool mPendingNotifyDataArrived;
 747   void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
 748
 749   void DoDemuxVideo();
 750   void OnVideoDemuxCompleted(
 751       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 752   void OnVideoDemuxFailed(const MediaResult& aError) {
 753     OnDemuxFailed(TrackType::kVideoTrack, aError);
 754   }
 755
 756   void DoDemuxAudio();
 757   void OnAudioDemuxCompleted(
 758       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 759   void OnAudioDemuxFailed(const MediaResult& aError) {
 760     OnDemuxFailed(TrackType::kAudioTrack, aError);
 761   }
 762
 763   void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
 764   MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise>
 765       mSkipRequest;
 766   void VideoSkipReset(uint32_t aSkipped);
 767   void OnVideoSkipCompleted(uint32_t aSkipped);
 768   void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
 769
 770   // The last number of decoded output frames that we've reported to
 771   // MediaDecoder::NotifyDecoded(). We diff the number of output video
 772   // frames every time that DecodeVideoData() is called, and report the
 773   // delta there.
 774   uint64_t mLastReportedNumDecodedFrames;
 775
 776   // Timestamp of the previous decoded keyframe, in microseconds.
 777   int64_t mPreviousDecodedKeyframeTime_us;
 778   // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
 779   static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
 780
 781   RefPtr<layers::KnowsCompositor> mKnowsCompositor;
 782
 783   // Metadata objects
 784   // True if we've read the streams' metadata.
 785   bool mInitDone;
 786   MozPromiseHolder<MetadataPromise> mMetadataPromise;
 787   bool IsEncrypted() const;
 788
 789   // Set to true if any of our track buffers may be blocking.
 790   bool mTrackDemuxersMayBlock;
 791
 792   // Seeking objects.
 793   void SetSeekTarget(const SeekTarget& aTarget);
 794   bool IsSeeking() const { return mPendingSeekTime.isSome(); }
 795   bool IsVideoOnlySeeking() const {
 796     return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
 797   }
 798   bool IsAudioOnlySeeking() const {
 799     return IsSeeking() && mOriginalSeekTarget.IsAudioOnly();
 800   }
 801   void ScheduleSeek();
 802   void AttemptSeek();
 803   void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
 804   void DoVideoSeek();
 805   void OnVideoSeekCompleted(media::TimeUnit aTime);
 806   void OnVideoSeekFailed(const MediaResult& aError);
 807   bool mSeekScheduled;
 808
 809   void DoAudioSeek();
 810   void OnAudioSeekCompleted(media::TimeUnit aTime);
 811   void OnAudioSeekFailed(const MediaResult& aError);
 812   // The SeekTarget that was last given to Seek()
 813   SeekTarget mOriginalSeekTarget;
 814   // Temporary seek information while we wait for the data
 815   Maybe<media::TimeUnit> mFallbackSeekTime;
 816   Maybe<media::TimeUnit> mPendingSeekTime;
 817   MozPromiseHolder<SeekPromise> mSeekPromise;
 818
 819   RefPtr<VideoFrameContainer> mVideoFrameContainer;
 820   layers::ImageContainer* GetImageContainer();
 821
 822   RefPtr<CDMProxy> mCDMProxy;
 823
 824   RefPtr<GMPCrashHelper> mCrashHelper;
 825
 826   void SetNullDecode(TrackType aTrack, bool aIsNullDecode);
 827
 828   class DecoderFactory;
 829   UniquePtr<DecoderFactory> mDecoderFactory;
 830
 831   class ShutdownPromisePool;
 832   UniquePtr<ShutdownPromisePool> mShutdownPromisePool;
 833
 834   MediaEventListener mOnTrackWaitingForKeyListener;
 835
 836   void OnFirstDemuxCompleted(
 837       TrackInfo::TrackType aType,
 838       const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples);
 839
 840   void OnFirstDemuxFailed(TrackInfo::TrackType aType,
 841                           const MediaResult& aError);
 842
 843   void MaybeResolveMetadataPromise();
 844
 845   // Stores presentation info required for playback.
 846   MediaInfo mInfo;
 847
 848   UniquePtr<MetadataTags> mTags;
 849
 850   // A flag indicating if the start time is known or not.
 851   bool mHasStartTime = false;
 852
 853   void ShutdownDecoder(TrackType aTrack);
 854   RefPtr<ShutdownPromise> TearDownDecoders();
 855
 856   bool mShutdown = false;
 857
 858   // Buffered range.
 859   Canonical<media::TimeIntervals> mBuffered;
 860
 861   // Used to send TimedMetadata to the listener.
 862   TimedMetadataEventProducer mTimedMetadataEvent;
 863
 864   // Notify if this media is not seekable.
 865   MediaEventProducer<void> mOnMediaNotSeekable;
 866
 867   // Notify if we are waiting for a decryption key.
 868   MediaEventProducer<TrackInfo::TrackType> mOnTrackWaitingForKey;
 869
 870   MediaEventProducer<nsTArray<uint8_t>, nsString> mOnEncrypted;
 871
 872   MediaEventProducer<void> mOnWaitingForKey;
 873
 874   MediaEventProducer<MediaResult> mOnDecodeWarning;
 875
 876   MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark;
 877
 878   MediaEventProducer<VideoInfo, AudioInfo> mTrackInfoUpdatedEvent;
 879
 880   RefPtr<FrameStatistics> mFrameStats;
 881
 882   // Used in bug 1393399 for telemetry.
 883   const MediaDecoderOwnerID mMediaDecoderOwnerID;
 884
 885   bool ResolveSetCDMPromiseIfDone(TrackType aTrack);
 886   void PrepareToSetCDMForTrack(TrackType aTrack);
 887   MozPromiseHolder<SetCDMPromise> mSetCDMPromise;
 888   TrackSet mSetCDMForTracks{};
 889   bool IsDecoderWaitingForCDM(TrackType aTrack);
 890
 891   void GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo);
 892
 893   // Only be used on Windows when the media engine playback is enabled.
 894   Maybe<uint64_t> mMediaEngineId;
 895
 896   const Maybe<TrackingId> mTrackingId;
 897
 898   // The start time of reading the metdata and how long does it take. This
 899   // measurement includes the time of downloading media resource over the
 900   // internet.
 901   Maybe<TimeStamp> mReadMetadataStartTime;
 902   TimeDuration mReadMetaDataTime;
 903
 904   // The total amount of time we have been waiting for the video data due to
 905   // lacking of data.
 906   TimeDuration mTotalWaitingForVideoDataTime;
 907
 908   // https://github.com/w3c/encrypted-media/issues/251#issuecomment-819783073
 909   // Treat playback as encrypted if the media key is set before playback starts,
 910   // this allows websites to start with non-encrypted stream and switch to
 911   // encrypted stream later.
 912   Atomic<bool> mEncryptedCustomIdent;
 913 };
 914
 915 }  // namespace mozilla
 916
 917 #endif