dom/media/ExternalEngineStateMachine.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef DOM_MEDIA_EXTERNALENGINESTATEMACHINE_H_
   6 #define DOM_MEDIA_EXTERNALENGINESTATEMACHINE_H_
   7
   8 #include "MediaDecoderStateMachineBase.h"
   9 #include "SeekJob.h"
  10 #include "mozilla/Variant.h"
  11
  12 namespace mozilla {
  13
  14 /**
  15  * ExternalPlaybackEngine represents a media engine which is responsible for
  16  * decoding and playback, which are not controlled by Gecko.
  17  */
  18 class ExternalPlaybackEngine;
  19
  20 enum class ExternalEngineEvent {
  21   LoadedMetaData,
  22   LoadedFirstFrame,
  23   LoadedData,
  24   Waiting,
  25   Playing,
  26   Seeked,
  27   BufferingStarted,
  28   BufferingEnded,
  29   Timeupdate,
  30   Ended,
  31   RequestForAudio,
  32   RequestForVideo,
  33   AudioEnough,
  34   VideoEnough,
  35 };
  36 const char* ExternalEngineEventToStr(ExternalEngineEvent aEvent);
  37
  38 /**
  39  * When using ExternalEngineStateMachine, that means we use an external engine
  40  * to control decoding and playback (including A/V sync). Eg. Media Foundation
  41  * Media Engine on Windows.
  42  *
  43  * The external engine does most of playback works, and uses ExternalEngineEvent
  44  * to tell us its internal state. Therefore, this state machine is responsible
  45  * to address those events from the engine and coordinate the format reader in
  46  * order to provide data to the engine correctly.
  47  */
  48 DDLoggedTypeDeclName(ExternalEngineStateMachine);
  49
  50 class ExternalEngineStateMachine final
  51     : public MediaDecoderStateMachineBase,
  52       public DecoderDoctorLifeLogger<ExternalEngineStateMachine> {
  53  public:
  54   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ExternalEngineStateMachine, override)
  55
  56   ExternalEngineStateMachine(MediaDecoder* aDecoder,
  57                              MediaFormatReader* aReader);
  58
  59   RefPtr<MediaDecoder::SeekPromise> InvokeSeek(
  60       const SeekTarget& aTarget) override;
  61
  62   RefPtr<GenericPromise> InvokeSetSink(
  63       const RefPtr<AudioDeviceInfo>& aSink) override;
  64
  65   // The media sample would be managed by the external engine so we won't store
  66   // any samples in our side.
  67   size_t SizeOfVideoQueue() const override { return 0; }
  68   size_t SizeOfAudioQueue() const override { return 0; }
  69
  70   // Not supported.
  71   void SetVideoDecodeMode(VideoDecodeMode aMode) override {}
  72   void InvokeSuspendMediaSink() override {}
  73   void InvokeResumeMediaSink() override {}
  74   RefPtr<GenericPromise> RequestDebugInfo(
  75       dom::MediaDecoderStateMachineDebugInfo& aInfo) override {
  76     // This debug info doesn't fit in this scenario because most decoding
  77     // details are only visible inside the external engine.
  78     return GenericPromise::CreateAndResolve(true, __func__);
  79   }
  80
  81   void NotifyEvent(ExternalEngineEvent aEvent) {
  82     // On the engine manager thread.
  83     Unused << OwnerThread()->Dispatch(NS_NewRunnableFunction(
  84         "ExternalEngineStateMachine::NotifyEvent",
  85         [self = RefPtr{this}, aEvent] { self->NotifyEventInternal(aEvent); }));
  86   }
  87   void NotifyError(const MediaResult& aError) {
  88     // On the engine manager thread.
  89     Unused << OwnerThread()->Dispatch(NS_NewRunnableFunction(
  90         "ExternalEngineStateMachine::NotifyError",
  91         [self = RefPtr{this}, aError] { self->NotifyErrorInternal(aError); }));
  92   }
  93   void NotifyResizing(uint32_t aWidth, uint32_t aHeight) {
  94     // On the engine manager thread.
  95     Unused << OwnerThread()->Dispatch(
  96         NS_NewRunnableFunction("ExternalEngineStateMachine::NotifyResizing",
  97                                [self = RefPtr{this}, aWidth, aHeight] {
  98                                  self->NotifyResizingInternal(aWidth, aHeight);
  99                                }));
 100   }
 101
 102   const char* GetStateStr() const;
 103
 104   RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy) override;
 105
 106   bool IsCDMProxySupported(CDMProxy* aProxy) override;
 107
 108   bool IsExternalEngineStateMachine() const override { return true; }
 109
 110  private:
 111   ~ExternalEngineStateMachine();
 112
 113   void AssertOnTaskQueue() const { MOZ_ASSERT(OnTaskQueue()); }
 114
 115   // A light-weight state object that helps to store some variables which would
 116   // only be used in a certain state. Also be able to do the cleaning for the
 117   // state transition. Only modify on the task queue.
 118   struct StateObject final {
 119     enum class State {
 120       InitEngine,
 121       ReadingMetadata,
 122       RunningEngine,
 123       SeekingData,
 124       ShutdownEngine,
 125       RecoverEngine,
 126     };
 127     struct InitEngine {
 128       InitEngine() = default;
 129       ~InitEngine() { mEngineInitRequest.DisconnectIfExists(); }
 130       MozPromiseRequestHolder<GenericNonExclusivePromise> mEngineInitRequest;
 131       RefPtr<GenericNonExclusivePromise> mInitPromise;
 132     };
 133     struct ReadingMetadata {
 134       ReadingMetadata() = default;
 135       ~ReadingMetadata() { mMetadataRequest.DisconnectIfExists(); }
 136       MozPromiseRequestHolder<MediaFormatReader::MetadataPromise>
 137           mMetadataRequest;
 138     };
 139     struct RunningEngine {};
 140     struct SeekingData {
 141       SeekingData() = default;
 142       SeekingData(SeekingData&&) = default;
 143       SeekingData(const SeekingData&) = delete;
 144       SeekingData& operator=(const SeekingData&) = delete;
 145       ~SeekingData() {
 146         mSeekJob.RejectIfExists(__func__);
 147         mSeekRequest.DisconnectIfExists();
 148       }
 149       void SetTarget(const SeekTarget& aTarget) {
 150         // If there is any promise for previous seeking, reject it first.
 151         mSeekJob.RejectIfExists(__func__);
 152         mSeekRequest.DisconnectIfExists();
 153         // Then create a new seek job.
 154         mSeekJob = SeekJob();
 155         mSeekJob.mTarget = Some(aTarget);
 156       }
 157       void Resolve(const char* aCallSite) {
 158         MOZ_ASSERT(mSeekJob.Exists());
 159         mSeekJob.Resolve(aCallSite);
 160         mSeekJob = SeekJob();
 161       }
 162       void RejectIfExists(const char* aCallSite) {
 163         mSeekJob.RejectIfExists(aCallSite);
 164       }
 165       bool IsSeeking() const { return mSeekRequest.Exists(); }
 166       media::TimeUnit GetTargetTime() const {
 167         return mSeekJob.mTarget ? mSeekJob.mTarget->GetTime()
 168                                 : media::TimeUnit::Invalid();
 169       }
 170       // Set it to true when starting seeking, and would be set to false after
 171       // receiving engine's `seeked` event. Used on thhe task queue only.
 172       bool mWaitingEngineSeeked = false;
 173       bool mWaitingReaderSeeked = false;
 174       MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mSeekRequest;
 175       SeekJob mSeekJob;
 176     };
 177     struct ShutdownEngine {
 178       RefPtr<ShutdownPromise> mShutdown;
 179     };
 180     // This state is used to recover the media engine after the MF CDM process
 181     // crashes.
 182     struct RecoverEngine : public InitEngine {};
 183
 184     StateObject() : mData(ReadingMetadata()), mName(State::ReadingMetadata){};
 185     explicit StateObject(InitEngine&& aArg)
 186         : mData(std::move(aArg)), mName(State::InitEngine){};
 187     explicit StateObject(RunningEngine&& aArg)
 188         : mData(std::move(aArg)), mName(State::RunningEngine){};
 189     explicit StateObject(SeekingData&& aArg)
 190         : mData(std::move(aArg)), mName(State::SeekingData){};
 191     explicit StateObject(ShutdownEngine&& aArg)
 192         : mData(std::move(aArg)), mName(State::ShutdownEngine){};
 193     explicit StateObject(RecoverEngine&& aArg)
 194         : mData(std::move(aArg)), mName(State::RecoverEngine){};
 195
 196     bool IsInitEngine() const { return mData.is<InitEngine>(); }
 197     bool IsReadingMetadata() const { return mData.is<ReadingMetadata>(); }
 198     bool IsRunningEngine() const { return mData.is<RunningEngine>(); }
 199     bool IsSeekingData() const { return mData.is<SeekingData>(); }
 200     bool IsShutdownEngine() const { return mData.is<ShutdownEngine>(); }
 201     bool IsRecoverEngine() const { return mData.is<RecoverEngine>(); }
 202
 203     InitEngine* AsInitEngine() {
 204       if (IsInitEngine()) {
 205         return &mData.as<InitEngine>();
 206       }
 207       if (IsRecoverEngine()) {
 208         return &mData.as<RecoverEngine>();
 209       }
 210       return nullptr;
 211     }
 212     ReadingMetadata* AsReadingMetadata() {
 213       return IsReadingMetadata() ? &mData.as<ReadingMetadata>() : nullptr;
 214     }
 215     SeekingData* AsSeekingData() {
 216       return IsSeekingData() ? &mData.as<SeekingData>() : nullptr;
 217     }
 218     ShutdownEngine* AsShutdownEngine() {
 219       return IsShutdownEngine() ? &mData.as<ShutdownEngine>() : nullptr;
 220     }
 221
 222     Variant<InitEngine, ReadingMetadata, RunningEngine, SeekingData,
 223             ShutdownEngine, RecoverEngine>
 224         mData;
 225     State mName;
 226   } mState;
 227   using State = StateObject::State;
 228
 229   void NotifyEventInternal(ExternalEngineEvent aEvent);
 230   void NotifyErrorInternal(const MediaResult& aError);
 231   void NotifyResizingInternal(uint32_t aWidth, uint32_t aHeight);
 232
 233   RefPtr<ShutdownPromise> Shutdown() override;
 234
 235   void SetPlaybackRate(double aPlaybackRate) override;
 236   void BufferedRangeUpdated() override;
 237   void VolumeChanged() override;
 238   void PreservesPitchChanged() override;
 239   void PlayStateChanged() override;
 240   void LoopingChanged() override;
 241   void PlaybackRateChanged();
 242
 243   // Not supported.
 244   void SetIsLiveStream(bool aIsLiveStream) override {}
 245   void SetCanPlayThrough(bool aCanPlayThrough) override {}
 246   void SetFragmentEndTime(const media::TimeUnit& aFragmentEndTime) override {}
 247
 248   void InitEngine();
 249   void OnEngineInitSuccess();
 250   void OnEngineInitFailure();
 251
 252   void ReadMetadata();
 253   void OnMetadataRead(MetadataHolder&& aMetadata);
 254   void OnMetadataNotRead(const MediaResult& aError);
 255   bool IsFormatSupportedByExternalEngine(const MediaInfo& aInfo);
 256
 257   // Functions for handling external engine event.
 258   void OnLoadedFirstFrame();
 259   void OnLoadedData();
 260   void OnWaiting();
 261   void OnPlaying();
 262   void OnSeeked();
 263   void OnBufferingStarted();
 264   void OnBufferingEnded();
 265   void OnTimeupdate();
 266   void OnEnded();
 267   void OnRequestAudio();
 268   void OnRequestVideo();
 269
 270   void ResetDecode();
 271
 272   void EndOfStream(MediaData::Type aType);
 273   void WaitForData(MediaData::Type aType);
 274
 275   void StartRunningEngine();
 276   void RunningEngineUpdate(MediaData::Type aType);
 277
 278   void ChangeStateTo(State aNextState);
 279   static const char* StateToStr(State aState);
 280
 281   RefPtr<MediaDecoder::SeekPromise> Seek(const SeekTarget& aTarget) override;
 282   void SeekReader();
 283   void OnSeekResolved(const media::TimeUnit& aUnit);
 284   void OnSeekRejected(const SeekRejectValue& aReject);
 285   bool IsSeeking();
 286   void CheckIfSeekCompleted();
 287
 288   void MaybeFinishWaitForData();
 289
 290   void SetBlankVideoToVideoContainer();
 291
 292   media::TimeUnit GetVideoThreshold();
 293
 294   bool ShouldRunEngineUpdateForRequest();
 295
 296   void UpdateSecondaryVideoContainer() override;
 297
 298   void RecoverFromCDMProcessCrashIfNeeded();
 299
 300   void ReportTelemetry(const MediaResult& aError);
 301
 302   void DecodeError(const MediaResult& aError) override;
 303
 304   UniquePtr<ExternalPlaybackEngine> mEngine;
 305
 306   bool mHasEnoughAudio = false;
 307   bool mHasEnoughVideo = false;
 308   bool mSentPlaybackEndedEvent = false;
 309   bool mHasReceivedFirstDecodedVideoFrame = false;
 310
 311   // Only used if setting CDM happens before the engine finishes initialization.
 312   MozPromiseHolder<SetCDMPromise> mSetCDMProxyPromise;
 313   MozPromiseRequestHolder<SetCDMPromise> mSetCDMProxyRequest;
 314
 315   // If seek happens while the engine is still initializing, then we would
 316   // postpone the seek until the engine is ready.
 317   SeekJob mPendingSeek;
 318   MozPromiseRequestHolder<MediaDecoder::SeekPromise> mPendingSeekRequest;
 319
 320   // It would be zero for audio-only playback.
 321   gfx::IntSize mVideoDisplay;
 322
 323   // It would be set if playback is encrypted.
 324   nsCString mKeySystem;
 325
 326   // This array stores the tasks which needs to be executed only after the
 327   // engine is ready but is called before that. It will be executed when
 328   // starting running the engine.
 329   nsTArray<RefPtr<nsIRunnable>> mPendingTasks;
 330
 331   bool mHasFatalError = false;
 332 };
 333
 334 class ExternalPlaybackEngine {
 335  public:
 336   explicit ExternalPlaybackEngine(ExternalEngineStateMachine* aOwner)
 337       : mOwner(aOwner) {}
 338
 339   virtual ~ExternalPlaybackEngine() = default;
 340
 341   // Init the engine and specify the preload request.
 342   virtual RefPtr<GenericNonExclusivePromise> Init(const MediaInfo& aInfo,
 343                                                   bool aShouldPreload) = 0;
 344   virtual void Shutdown() = 0;
 345   virtual uint64_t Id() const = 0;
 346   virtual bool IsInited() const = 0;
 347
 348   // Following methods should only be called after successfully initialize the
 349   // external engine.
 350   virtual void Play() = 0;
 351   virtual void Pause() = 0;
 352   virtual void Seek(const media::TimeUnit& aTargetTime) = 0;
 353   virtual void SetPlaybackRate(double aPlaybackRate) = 0;
 354   virtual void SetVolume(double aVolume) = 0;
 355   virtual void SetLooping(bool aLooping) = 0;
 356   virtual void SetPreservesPitch(bool aPreservesPitch) = 0;
 357   virtual media::TimeUnit GetCurrentPosition() = 0;
 358   virtual void NotifyEndOfStream(TrackInfo::TrackType aType) = 0;
 359   virtual bool SetCDMProxy(CDMProxy* aProxy) = 0;
 360   virtual void NotifyResizing(uint32_t aWidth, uint32_t aHeight) = 0;
 361
 362   ExternalEngineStateMachine* const MOZ_NON_OWNING_REF mOwner;
 363 };
 364
 365 }  // namespace mozilla
 366
 367 #endif  // DOM_MEDIA_EXTERNALENGINESTATEMACHINE_H_