1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "ExternalEngineStateMachine.h"
7 #include "PerformanceRecorder.h"
8 #ifdef MOZ_WMF_MEDIA_ENGINE
9 # include "MFMediaEngineDecoderModule.h"
10 # include "mozilla/MFMediaEngineChild.h"
11 # include "mozilla/StaticPrefs_media.h"
13 #include "mozilla/Atomics.h"
14 #include "mozilla/ClearOnShutdown.h"
15 #include "mozilla/ProfilerLabels.h"
16 #include "mozilla/UniquePtr.h"
17 #include "mozilla/StaticMutex.h"
18 #include "nsThreadUtils.h"
22 extern LazyLogModule gMediaDecoderLog
;
25 "Decoder=%p, State=%s, " x, mDecoderID, GetStateStr(), ##__VA_ARGS__
27 DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p, State=%s, " x, \
28 mDecoderID, GetStateStr(), ##__VA_ARGS__)
29 #define LOGV(x, ...) \
30 DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p, State=%s, " x, \
31 mDecoderID, GetStateStr(), ##__VA_ARGS__)
32 #define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get())
33 #define LOGE(x, ...) \
34 NS_DebugBreak(NS_DEBUG_WARNING, \
35 nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \
38 const char* ExternalEngineEventToStr(ExternalEngineEvent aEvent
) {
39 #define EVENT_TO_STR(event) \
40 case ExternalEngineEvent::event: \
43 EVENT_TO_STR(LoadedMetaData
);
44 EVENT_TO_STR(LoadedFirstFrame
);
45 EVENT_TO_STR(LoadedData
);
46 EVENT_TO_STR(Waiting
);
47 EVENT_TO_STR(Playing
);
49 EVENT_TO_STR(BufferingStarted
);
50 EVENT_TO_STR(BufferingEnded
);
51 EVENT_TO_STR(Timeupdate
);
53 EVENT_TO_STR(RequestForAudio
);
54 EVENT_TO_STR(RequestForVideo
);
55 EVENT_TO_STR(AudioEnough
);
56 EVENT_TO_STR(VideoEnough
);
58 MOZ_ASSERT_UNREACHABLE("Undefined event!");
65 * This class monitors the amount of crash happened for a remote engine
66 * process. It the amount of crash of the remote process exceeds the defined
67 * threshold, then `ShouldRecoverProcess()` will return false to indicate that
68 * we should not keep spawning that remote process because it's too easy to
71 * In addition, we also have another mechanism in the media format reader
72 * (MFR) to detect crash amount of remote processes, but that would only
73 * happen during the decoding process. The main reason to choose using this
74 * simple monitor, instead of the mechanism in the MFR is because that
75 * mechanism can't detect every crash happening in the remote process, such as
76 * crash happening during initializing the remote engine, or setting the CDM
77 * pipepline, which can happen prior to decoding.
79 class ProcessCrashMonitor final
{
81 static void NotifyCrash() {
82 StaticMutexAutoLock
lock(sMutex
);
83 auto* monitor
= ProcessCrashMonitor::EnsureInstance();
87 monitor
->mCrashNums
++;
89 static bool ShouldRecoverProcess() {
90 StaticMutexAutoLock
lock(sMutex
);
91 auto* monitor
= ProcessCrashMonitor::EnsureInstance();
95 return monitor
->mCrashNums
<= monitor
->mMaxCrashes
;
99 ProcessCrashMonitor() : mCrashNums(0) {
100 #ifdef MOZ_WMF_MEDIA_ENGINE
101 mMaxCrashes
= StaticPrefs::media_wmf_media_engine_max_crashes();
106 ProcessCrashMonitor(const ProcessCrashMonitor
&) = delete;
107 ProcessCrashMonitor
& operator=(const ProcessCrashMonitor
&) = delete;
109 static ProcessCrashMonitor
* EnsureInstance() {
113 if (!sCrashMonitor
) {
114 sCrashMonitor
.reset(new ProcessCrashMonitor());
115 GetMainThreadSerialEventTarget()->Dispatch(
116 NS_NewRunnableFunction("ProcessCrashMonitor::EnsureInstance", [&] {
119 StaticMutexAutoLock
lock(sMutex
);
120 sCrashMonitor
.reset();
123 ShutdownPhase::XPCOMShutdown
);
126 return sCrashMonitor
.get();
129 static inline StaticMutex sMutex
;
130 static inline UniquePtr
<ProcessCrashMonitor
> sCrashMonitor
;
131 static inline Atomic
<bool> sIsShutdown
{false};
134 uint32_t mMaxCrashes
;
138 const char* ExternalEngineStateMachine::StateToStr(State aNextState
) {
139 #define STATE_TO_STR(state) \
142 switch (aNextState
) {
143 STATE_TO_STR(InitEngine
);
144 STATE_TO_STR(ReadingMetadata
);
145 STATE_TO_STR(RunningEngine
);
146 STATE_TO_STR(SeekingData
);
147 STATE_TO_STR(ShutdownEngine
);
148 STATE_TO_STR(RecoverEngine
);
150 MOZ_ASSERT_UNREACHABLE("Undefined state!");
156 const char* ExternalEngineStateMachine::GetStateStr() const {
157 return StateToStr(mState
.mName
);
160 void ExternalEngineStateMachine::ChangeStateTo(State aNextState
) {
161 LOG("Change state : '%s' -> '%s' (play-state=%d)", StateToStr(mState
.mName
),
162 StateToStr(aNextState
), mPlayState
.Ref());
163 // Assert the possible state transitions.
164 MOZ_ASSERT_IF(mState
.IsInitEngine(), aNextState
== State::ReadingMetadata
||
165 aNextState
== State::ShutdownEngine
);
166 MOZ_ASSERT_IF(mState
.IsReadingMetadata(),
167 aNextState
== State::RunningEngine
||
168 aNextState
== State::ShutdownEngine
);
169 MOZ_ASSERT_IF(mState
.IsRunningEngine(),
170 aNextState
== State::SeekingData
||
171 aNextState
== State::ShutdownEngine
||
172 aNextState
== State::RecoverEngine
);
173 MOZ_ASSERT_IF(mState
.IsSeekingData(),
174 aNextState
== State::RunningEngine
||
175 aNextState
== State::ShutdownEngine
||
176 aNextState
== State::RecoverEngine
);
177 MOZ_ASSERT_IF(mState
.IsShutdownEngine(), aNextState
== State::ShutdownEngine
);
179 mState
.IsRecoverEngine(),
180 aNextState
== State::SeekingData
|| aNextState
== State::ShutdownEngine
);
181 if (aNextState
== State::SeekingData
) {
182 mState
= StateObject({StateObject::SeekingData()});
183 } else if (aNextState
== State::ReadingMetadata
) {
184 mState
= StateObject({StateObject::ReadingMetadata()});
185 } else if (aNextState
== State::RunningEngine
) {
186 mState
= StateObject({StateObject::RunningEngine()});
187 } else if (aNextState
== State::ShutdownEngine
) {
188 mState
= StateObject({StateObject::ShutdownEngine()});
189 } else if (aNextState
== State::RecoverEngine
) {
190 mState
= StateObject({StateObject::RecoverEngine()});
192 MOZ_ASSERT_UNREACHABLE("Wrong state!");
196 ExternalEngineStateMachine::ExternalEngineStateMachine(
197 MediaDecoder
* aDecoder
, MediaFormatReader
* aReader
)
198 : MediaDecoderStateMachineBase(aDecoder
, aReader
) {
199 LOG("Created ExternalEngineStateMachine");
200 MOZ_ASSERT(mState
.IsInitEngine());
204 void ExternalEngineStateMachine::InitEngine() {
205 MOZ_ASSERT(mState
.IsInitEngine() || mState
.IsRecoverEngine());
206 #ifdef MOZ_WMF_MEDIA_ENGINE
207 mEngine
.reset(new MFMediaEngineWrapper(this, mFrameStats
));
210 auto* state
= mState
.AsInitEngine();
211 state
->mInitPromise
= mEngine
->Init(!mMinimizePreroll
);
213 ->Then(OwnerThread(), __func__
, this,
214 &ExternalEngineStateMachine::OnEngineInitSuccess
,
215 &ExternalEngineStateMachine::OnEngineInitFailure
)
216 ->Track(state
->mEngineInitRequest
);
220 void ExternalEngineStateMachine::OnEngineInitSuccess() {
222 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnEngineInitSuccess",
224 MOZ_ASSERT(mState
.IsInitEngine() || mState
.IsRecoverEngine());
225 LOG("Initialized the external playback engine %" PRIu64
, mEngine
->Id());
226 auto* state
= mState
.AsInitEngine();
227 state
->mEngineInitRequest
.Complete();
228 mReader
->UpdateMediaEngineId(mEngine
->Id());
229 state
->mInitPromise
= nullptr;
230 if (mState
.IsInitEngine()) {
231 ChangeStateTo(State::ReadingMetadata
);
235 // We just recovered from CDM process crash, so we need to update the media
236 // info to the new CDM process.
238 mEngine
->SetMediaInfo(*mInfo
);
239 SeekTarget
target(mCurrentPosition
.Ref(), SeekTarget::Type::Accurate
);
243 void ExternalEngineStateMachine::OnEngineInitFailure() {
245 MOZ_ASSERT(mState
.IsInitEngine() || mState
.IsRecoverEngine());
246 LOGE("Failed to initialize the external playback engine");
247 auto* state
= mState
.AsInitEngine();
248 state
->mEngineInitRequest
.Complete();
249 state
->mInitPromise
= nullptr;
250 // TODO : Should fallback to the normal playback with media engine.
251 DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR
, __func__
));
254 void ExternalEngineStateMachine::ReadMetadata() {
256 MOZ_ASSERT(mState
.IsReadingMetadata());
257 mReader
->ReadMetadata()
258 ->Then(OwnerThread(), __func__
, this,
259 &ExternalEngineStateMachine::OnMetadataRead
,
260 &ExternalEngineStateMachine::OnMetadataNotRead
)
261 ->Track(mState
.AsReadingMetadata()->mMetadataRequest
);
264 void ExternalEngineStateMachine::OnMetadataRead(MetadataHolder
&& aMetadata
) {
266 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnMetadataRead",
268 MOZ_ASSERT(mState
.IsReadingMetadata());
269 LOG("OnMetadataRead");
271 mState
.AsReadingMetadata()->mMetadataRequest
.Complete();
272 mInfo
.emplace(*aMetadata
.mInfo
);
273 mMediaSeekable
= Info().mMediaSeekable
;
274 mMediaSeekableOnlyInBufferedRanges
=
275 Info().mMediaSeekableOnlyInBufferedRanges
;
277 if (!IsFormatSupportedByExternalEngine(*mInfo
)) {
278 // The external engine doesn't support the type, try to notify the decoder
279 // to use our own state machine again.
281 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR
));
285 #ifdef MOZ_WMF_MEDIA_ENGINE
286 // Only support encrypted playback.
287 if (!mInfo
->IsEncrypted() &&
288 StaticPrefs::media_wmf_media_engine_enabled() == 2) {
289 LOG("External engine only supports encrypted playback by the pref");
291 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR
));
296 mEngine
->SetMediaInfo(*mInfo
);
298 if (Info().mMetadataDuration
.isSome()) {
299 mDuration
= Info().mMetadataDuration
;
300 } else if (Info().mUnadjustedMetadataEndTime
.isSome()) {
301 const media::TimeUnit unadjusted
= Info().mUnadjustedMetadataEndTime
.ref();
302 const media::TimeUnit adjustment
= Info().mStartTime
;
303 mInfo
->mMetadataDuration
.emplace(unadjusted
- adjustment
);
304 mDuration
= Info().mMetadataDuration
;
307 // If we don't know the duration by this point, we assume infinity, per spec.
308 if (mDuration
.Ref().isNothing()) {
309 mDuration
= Some(media::TimeUnit::FromInfinity());
311 MOZ_ASSERT(mDuration
.Ref().isSome());
313 mMetadataLoadedEvent
.Notify(std::move(aMetadata
.mInfo
),
314 std::move(aMetadata
.mTags
),
315 MediaDecoderEventVisibility::Observable
);
316 StartRunningEngine();
319 void ExternalEngineStateMachine::OnMetadataNotRead(const MediaResult
& aError
) {
321 MOZ_ASSERT(mState
.IsReadingMetadata());
322 LOGE("Decode metadata failed, shutting down decoder");
323 mState
.AsReadingMetadata()->mMetadataRequest
.Complete();
327 bool ExternalEngineStateMachine::IsFormatSupportedByExternalEngine(
328 const MediaInfo
& aInfo
) {
330 MOZ_ASSERT(mState
.IsReadingMetadata());
331 #ifdef MOZ_WMF_MEDIA_ENGINE
332 const bool audioSupported
=
334 MFMediaEngineDecoderModule::SupportsConfig(aInfo
.mAudio
);
335 const bool videoSupported
=
337 MFMediaEngineDecoderModule::SupportsConfig(aInfo
.mVideo
);
338 LOG("audio=%s (supported=%d), video=%s(supported=%d)",
339 aInfo
.HasAudio() ? aInfo
.mAudio
.mMimeType
.get() : "none", audioSupported
,
340 aInfo
.HasVideo() ? aInfo
.mVideo
.mMimeType
.get() : "none", videoSupported
);
341 return audioSupported
&& videoSupported
;
347 RefPtr
<MediaDecoder::SeekPromise
> ExternalEngineStateMachine::Seek(
348 const SeekTarget
& aTarget
) {
350 if (!mState
.IsRunningEngine() && !mState
.IsSeekingData() &&
351 !mState
.IsRecoverEngine()) {
352 MOZ_ASSERT(false, "Can't seek due to unsupported state.");
353 return MediaDecoder::SeekPromise::CreateAndReject(true, __func__
);
355 // We don't support these type of seek, because they're depending on the
356 // implementation of the external engine, which might not be supported.
357 if (aTarget
.IsNextFrame() || aTarget
.IsVideoOnly()) {
358 return MediaDecoder::SeekPromise::CreateAndReject(true, __func__
);
361 LOG("Start seeking to %" PRId64
, aTarget
.GetTime().ToMicroseconds());
362 auto* state
= mState
.AsSeekingData();
364 // We're in other states, so change the state to seeking.
365 ChangeStateTo(State::SeekingData
);
366 state
= mState
.AsSeekingData();
368 state
->SetTarget(aTarget
);
370 // Update related status.
371 mSentPlaybackEndedEvent
= false;
372 mOnPlaybackEvent
.Notify(MediaPlaybackEvent::SeekStarted
);
373 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING
);
375 // Notify the external playback engine about seeking. After the engine changes
376 // its current time, it would send `seeked` event.
377 mEngine
->Seek(aTarget
.GetTime());
378 state
->mWaitingEngineSeeked
= true;
380 return state
->mSeekJob
.mPromise
.Ensure(__func__
);
383 void ExternalEngineStateMachine::SeekReader() {
385 MOZ_ASSERT(mState
.IsSeekingData());
386 auto* state
= mState
.AsSeekingData();
388 // Reset the reader first and ask it to perform a demuxer seek.
390 state
->mWaitingReaderSeeked
= true;
391 LOG("Seek reader to %" PRId64
, state
->GetTargetTime().ToMicroseconds());
392 mReader
->Seek(state
->mSeekJob
.mTarget
.ref())
393 ->Then(OwnerThread(), __func__
, this,
394 &ExternalEngineStateMachine::OnSeekResolved
,
395 &ExternalEngineStateMachine::OnSeekRejected
)
396 ->Track(state
->mSeekRequest
);
399 void ExternalEngineStateMachine::OnSeekResolved(const media::TimeUnit
& aUnit
) {
400 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnSeekResolved",
403 MOZ_ASSERT(mState
.IsSeekingData());
404 auto* state
= mState
.AsSeekingData();
406 LOG("OnReaderSeekResolved");
407 state
->mSeekRequest
.Complete();
408 state
->mWaitingReaderSeeked
= false;
410 // Start sending new data to the external playback engine.
412 mHasEnoughAudio
= false;
416 mHasEnoughVideo
= false;
419 CheckIfSeekCompleted();
422 void ExternalEngineStateMachine::OnSeekRejected(
423 const SeekRejectValue
& aReject
) {
424 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::OnSeekRejected",
427 MOZ_ASSERT(mState
.IsSeekingData());
428 auto* state
= mState
.AsSeekingData();
430 LOG("OnReaderSeekRejected");
431 state
->mSeekRequest
.Complete();
432 if (aReject
.mError
== NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
) {
433 LOG("OnSeekRejected reason=WAITING_FOR_DATA type=%s",
434 MediaData::TypeToStr(aReject
.mType
));
435 MOZ_ASSERT_IF(aReject
.mType
== MediaData::Type::AUDIO_DATA
,
436 !IsRequestingAudioData());
437 MOZ_ASSERT_IF(aReject
.mType
== MediaData::Type::VIDEO_DATA
,
438 !IsRequestingVideoData());
439 MOZ_ASSERT_IF(aReject
.mType
== MediaData::Type::AUDIO_DATA
,
440 !IsWaitingAudioData());
441 MOZ_ASSERT_IF(aReject
.mType
== MediaData::Type::VIDEO_DATA
,
442 !IsWaitingVideoData());
444 // Fire 'waiting' to notify the player that we are waiting for data.
445 mOnNextFrameStatus
.Notify(
446 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING
);
447 WaitForData(aReject
.mType
);
451 if (aReject
.mError
== NS_ERROR_DOM_MEDIA_END_OF_STREAM
) {
452 EndOfStream(aReject
.mType
);
456 MOZ_ASSERT(NS_FAILED(aReject
.mError
),
457 "Cancels should also disconnect mSeekRequest");
458 state
->RejectIfExists(__func__
);
459 DecodeError(aReject
.mError
);
462 bool ExternalEngineStateMachine::IsSeeking() {
464 const auto* state
= mState
.AsSeekingData();
465 return state
&& state
->IsSeeking();
468 void ExternalEngineStateMachine::CheckIfSeekCompleted() {
470 MOZ_ASSERT(mState
.IsSeekingData());
471 auto* state
= mState
.AsSeekingData();
472 if (state
->mWaitingEngineSeeked
|| state
->mWaitingReaderSeeked
) {
473 LOG("Seek hasn't been completed yet, waitEngineSeeked=%d, "
474 "waitReaderSeeked=%d",
475 state
->mWaitingEngineSeeked
, state
->mWaitingReaderSeeked
);
479 LOG("Seek completed");
480 state
->Resolve(__func__
);
481 mOnPlaybackEvent
.Notify(MediaPlaybackEvent::Invalidate
);
482 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE
);
483 StartRunningEngine();
486 void ExternalEngineStateMachine::ResetDecode() {
493 MediaFormatReader::TrackSet tracks
;
495 mVideoDataRequest
.DisconnectIfExists();
496 mVideoWaitRequest
.DisconnectIfExists();
497 tracks
+= TrackInfo::kVideoTrack
;
500 mAudioDataRequest
.DisconnectIfExists();
501 mAudioWaitRequest
.DisconnectIfExists();
502 tracks
+= TrackInfo::kAudioTrack
;
504 mReader
->ResetDecode(tracks
);
507 RefPtr
<GenericPromise
> ExternalEngineStateMachine::InvokeSetSink(
508 const RefPtr
<AudioDeviceInfo
>& aSink
) {
509 MOZ_ASSERT(NS_IsMainThread());
510 // TODO : can media engine support this?
511 return GenericPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
514 RefPtr
<ShutdownPromise
> ExternalEngineStateMachine::Shutdown() {
516 if (mState
.IsShutdownEngine()) {
517 LOG("Already shutdown");
518 return mState
.AsShutdownEngine()->mShutdown
;
522 ChangeStateTo(State::ShutdownEngine
);
525 mAudioDataRequest
.DisconnectIfExists();
526 mVideoDataRequest
.DisconnectIfExists();
527 mAudioWaitRequest
.DisconnectIfExists();
528 mVideoWaitRequest
.DisconnectIfExists();
530 mDuration
.DisconnectAll();
531 mCurrentPosition
.DisconnectAll();
532 // TODO : implement audible check
533 mIsAudioDataAudible
.DisconnectAll();
535 mMetadataManager
.Disconnect();
537 mSetCDMProxyPromise
.RejectIfExists(NS_ERROR_DOM_MEDIA_ABORT_ERR
, __func__
);
538 mSetCDMProxyRequest
.DisconnectIfExists();
542 auto* state
= mState
.AsShutdownEngine();
543 state
->mShutdown
= mReader
->Shutdown()->Then(
544 OwnerThread(), __func__
, [self
= RefPtr
{this}, this]() {
545 LOG("Shutting down state machine task queue");
546 return OwnerThread()->BeginShutdown();
548 return state
->mShutdown
;
551 void ExternalEngineStateMachine::BufferedRangeUpdated() {
553 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::BufferedRangeUpdated",
556 // While playing an unseekable stream of unknown duration, mDuration
557 // is updated as we play. But if data is being downloaded
558 // faster than played, mDuration won't reflect the end of playable data
559 // since we haven't played the frame at the end of buffered data. So update
560 // mDuration here as new data is downloaded to prevent such a lag.
561 if (mBuffered
.Ref().IsInvalid()) {
566 media::TimeUnit end
{mBuffered
.Ref().GetEnd(&exists
)};
571 // Use estimated duration from buffer ranges when mDuration is unknown or
572 // the estimated duration is larger.
573 if (mDuration
.Ref().isNothing() || mDuration
.Ref()->IsInfinite() ||
574 end
> mDuration
.Ref().ref()) {
575 mDuration
= Some(end
);
576 DDLOG(DDLogCategory::Property
, "duration_us",
577 mDuration
.Ref()->ToMicroseconds());
581 // Note: the variadic only supports passing member variables.
582 #define PERFORM_WHEN_ALLOW(Func, ...) \
584 /* Initialzation is not done yet, postpone the operation */ \
585 if ((mState.IsInitEngine() || mState.IsRecoverEngine()) && \
586 mState.AsInitEngine()->mInitPromise) { \
587 LOG("%s is called before init", __func__); \
588 mState.AsInitEngine()->mInitPromise->Then( \
589 OwnerThread(), __func__, \
590 [self = RefPtr{this}, this]( \
591 const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { \
592 if (aVal.IsResolve()) { \
597 } else if (mState.IsShutdownEngine()) { \
602 void ExternalEngineStateMachine::SetPlaybackRate(double aPlaybackRate
) {
604 mPlaybackRate
= aPlaybackRate
;
605 PERFORM_WHEN_ALLOW(SetPlaybackRate
, mPlaybackRate
);
606 mEngine
->SetPlaybackRate(aPlaybackRate
);
609 void ExternalEngineStateMachine::VolumeChanged() {
611 PERFORM_WHEN_ALLOW(VolumeChanged
);
612 mEngine
->SetVolume(mVolume
);
615 void ExternalEngineStateMachine::PreservesPitchChanged() {
617 PERFORM_WHEN_ALLOW(PreservesPitchChanged
);
618 mEngine
->SetPreservesPitch(mPreservesPitch
);
621 void ExternalEngineStateMachine::PlayStateChanged() {
623 PERFORM_WHEN_ALLOW(PlayStateChanged
);
624 if (mPlayState
== MediaDecoder::PLAY_STATE_PLAYING
) {
626 } else if (mPlayState
== MediaDecoder::PLAY_STATE_PAUSED
) {
631 void ExternalEngineStateMachine::LoopingChanged() {
633 PERFORM_WHEN_ALLOW(LoopingChanged
);
634 mEngine
->SetLooping(mLooping
);
637 #undef PERFORM_WHEN_ALLOW
639 void ExternalEngineStateMachine::EndOfStream(MediaData::Type aType
) {
641 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
642 static auto DataTypeToTrackType
= [](const MediaData::Type
& aType
) {
643 if (aType
== MediaData::Type::VIDEO_DATA
) {
644 return TrackInfo::TrackType::kVideoTrack
;
646 if (aType
== MediaData::Type::AUDIO_DATA
) {
647 return TrackInfo::TrackType::kAudioTrack
;
649 return TrackInfo::TrackType::kUndefinedTrack
;
651 mEngine
->NotifyEndOfStream(DataTypeToTrackType(aType
));
654 void ExternalEngineStateMachine::WaitForData(MediaData::Type aType
) {
656 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
657 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::WaitForData",
659 MOZ_ASSERT(aType
== MediaData::Type::AUDIO_DATA
||
660 aType
== MediaData::Type::VIDEO_DATA
);
663 RefPtr
<ExternalEngineStateMachine
> self
= this;
664 if (aType
== MediaData::Type::AUDIO_DATA
) {
665 MOZ_ASSERT(HasAudio());
666 mReader
->WaitForData(MediaData::Type::AUDIO_DATA
)
668 OwnerThread(), __func__
,
669 [self
, this](MediaData::Type aType
) {
671 "ExternalEngineStateMachine::WaitForData:AudioResolved",
673 MOZ_ASSERT(aType
== MediaData::Type::AUDIO_DATA
);
674 LOG("Done waiting for audio data");
675 mAudioWaitRequest
.Complete();
676 MaybeFinishWaitForData();
678 [self
, this](const WaitForDataRejectValue
& aRejection
) {
680 "ExternalEngineStateMachine::WaitForData:AudioRejected",
682 mAudioWaitRequest
.Complete();
683 DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
);
685 ->Track(mAudioWaitRequest
);
687 MOZ_ASSERT(HasVideo());
688 mReader
->WaitForData(MediaData::Type::VIDEO_DATA
)
690 OwnerThread(), __func__
,
691 [self
, this](MediaData::Type aType
) {
693 "ExternalEngineStateMachine::WaitForData:VideoResolved",
695 MOZ_ASSERT(aType
== MediaData::Type::VIDEO_DATA
);
696 LOG("Done waiting for video data");
697 mVideoWaitRequest
.Complete();
698 MaybeFinishWaitForData();
700 [self
, this](const WaitForDataRejectValue
& aRejection
) {
702 "ExternalEngineStateMachine::WaitForData:VideoRejected",
704 mVideoWaitRequest
.Complete();
705 DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
);
707 ->Track(mVideoWaitRequest
);
711 void ExternalEngineStateMachine::MaybeFinishWaitForData() {
713 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
715 bool isWaitingForAudio
= HasAudio() && mAudioWaitRequest
.Exists();
716 bool isWaitingForVideo
= HasVideo() && mVideoWaitRequest
.Exists();
717 if (isWaitingForAudio
|| isWaitingForVideo
) {
718 LOG("Still waiting for data (waitAudio=%d, waitVideo=%d)",
719 isWaitingForAudio
, isWaitingForVideo
);
723 LOG("Finished waiting for data");
724 if (mState
.IsSeekingData()) {
729 RunningEngineUpdate(MediaData::Type::AUDIO_DATA
);
732 RunningEngineUpdate(MediaData::Type::VIDEO_DATA
);
736 void ExternalEngineStateMachine::StartRunningEngine() {
737 ChangeStateTo(State::RunningEngine
);
738 // Manually check the play state because the engine might be recovered from
739 // crash or just get recreated, so PlayStateChanged() won't be triggered.
740 if (mPlayState
== MediaDecoder::PLAY_STATE_PLAYING
) {
744 RunningEngineUpdate(MediaData::Type::AUDIO_DATA
);
747 RunningEngineUpdate(MediaData::Type::VIDEO_DATA
);
751 void ExternalEngineStateMachine::RunningEngineUpdate(MediaData::Type aType
) {
753 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
754 if (aType
== MediaData::Type::AUDIO_DATA
&& !mHasEnoughAudio
) {
757 if (aType
== MediaData::Type::VIDEO_DATA
&& !mHasEnoughVideo
) {
762 void ExternalEngineStateMachine::OnRequestAudio() {
764 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
765 LOGV("OnRequestAudio");
771 if (IsRequestingAudioData() || mAudioWaitRequest
.Exists() || IsSeeking()) {
773 "No need to request audio, isRequesting=%d, waitingAudio=%d, "
775 IsRequestingAudioData(), mAudioWaitRequest
.Exists(), IsSeeking());
779 LOGV("Start requesting audio");
780 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestData
);
781 RefPtr
<ExternalEngineStateMachine
> self
= this;
782 mReader
->RequestAudioData()
784 OwnerThread(), __func__
,
785 [this, self
, perfRecorder(std::move(perfRecorder
))](
786 const RefPtr
<AudioData
>& aAudio
) mutable {
787 perfRecorder
.Record();
788 mAudioDataRequest
.Complete();
789 LOGV("Completed requesting audio");
791 "ExternalEngineStateMachine::OnRequestAudio:Resolved",
794 RunningEngineUpdate(MediaData::Type::AUDIO_DATA
);
796 [this, self
](const MediaResult
& aError
) {
797 mAudioDataRequest
.Complete();
799 "ExternalEngineStateMachine::OnRequestAudio:Rejected",
801 LOG("OnRequestAudio ErrorName=%s Message=%s",
802 aError
.ErrorName().get(), aError
.Message().get());
803 switch (aError
.Code()) {
804 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
805 WaitForData(MediaData::Type::AUDIO_DATA
);
807 case NS_ERROR_DOM_MEDIA_CANCELED
:
810 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
811 LOG("Reach to the end, no more audio data");
812 EndOfStream(MediaData::Type::AUDIO_DATA
);
814 case NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR
:
815 // We will handle the process crash in `NotifyErrorInternal()`
816 // so here just silently ignore this.
822 ->Track(mAudioDataRequest
);
825 void ExternalEngineStateMachine::OnRequestVideo() {
827 MOZ_ASSERT(mState
.IsRunningEngine() || mState
.IsSeekingData());
828 LOGV("OnRequestVideo");
834 if (IsRequestingVideoData() || mVideoWaitRequest
.Exists() || IsSeeking()) {
836 "No need to request video, isRequesting=%d, waitingVideo=%d, "
838 IsRequestingVideoData(), mVideoWaitRequest
.Exists(), IsSeeking());
842 LOGV("Start requesting video");
843 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestData
,
844 Info().mVideo
.mImage
.height
);
845 RefPtr
<ExternalEngineStateMachine
> self
= this;
846 mReader
->RequestVideoData(GetVideoThreshold(), false)
848 OwnerThread(), __func__
,
849 [this, self
, perfRecorder(std::move(perfRecorder
))](
850 const RefPtr
<VideoData
>& aVideo
) mutable {
851 perfRecorder
.Record();
852 mVideoDataRequest
.Complete();
853 LOGV("Completed requesting video");
855 "ExternalEngineStateMachine::OnRequestVideo:Resolved",
858 if (!mHasReceivedFirstDecodedVideoFrame
) {
859 mHasReceivedFirstDecodedVideoFrame
= true;
860 OnLoadedFirstFrame();
862 RunningEngineUpdate(MediaData::Type::VIDEO_DATA
);
863 // Send image to PIP window.
864 if (mSecondaryVideoContainer
.Ref()) {
865 mSecondaryVideoContainer
.Ref()->SetCurrentFrame(
866 mInfo
->mVideo
.mDisplay
, aVideo
->mImage
, TimeStamp::Now());
868 mVideoFrameContainer
->SetCurrentFrame(
869 mInfo
->mVideo
.mDisplay
, aVideo
->mImage
, TimeStamp::Now());
872 [this, self
](const MediaResult
& aError
) {
873 mVideoDataRequest
.Complete();
875 "ExternalEngineStateMachine::OnRequestVideo:Rejected",
877 LOG("OnRequestVideo ErrorName=%s Message=%s",
878 aError
.ErrorName().get(), aError
.Message().get());
879 switch (aError
.Code()) {
880 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
881 WaitForData(MediaData::Type::VIDEO_DATA
);
883 case NS_ERROR_DOM_MEDIA_CANCELED
:
886 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
887 LOG("Reach to the end, no more video data");
888 EndOfStream(MediaData::Type::VIDEO_DATA
);
890 case NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR
:
891 // We will handle the process crash in `NotifyErrorInternal()`
892 // so here just silently ignore this.
898 ->Track(mVideoDataRequest
);
901 void ExternalEngineStateMachine::OnLoadedFirstFrame() {
903 // We will wait until receive the first video frame.
904 if (mInfo
->HasVideo() && !mHasReceivedFirstDecodedVideoFrame
) {
905 LOGV("Hasn't received first decoded video frame");
908 LOGV("OnLoadedFirstFrame");
909 MediaDecoderEventVisibility visibility
=
910 mSentFirstFrameLoadedEvent
? MediaDecoderEventVisibility::Suppressed
911 : MediaDecoderEventVisibility::Observable
;
912 mSentFirstFrameLoadedEvent
= true;
913 mFirstFrameLoadedEvent
.Notify(UniquePtr
<MediaInfo
>(new MediaInfo(Info())),
915 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE
);
918 void ExternalEngineStateMachine::OnLoadedData() {
920 // In case the external engine doesn't send the first frame loaded event
922 if (!mSentFirstFrameLoadedEvent
) {
923 OnLoadedFirstFrame();
925 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE
);
928 void ExternalEngineStateMachine::OnWaiting() {
930 mOnNextFrameStatus
.Notify(
931 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING
);
934 void ExternalEngineStateMachine::OnPlaying() {
936 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE
);
939 void ExternalEngineStateMachine::OnSeeked() {
941 if (!mState
.IsSeekingData()) {
942 LOG("Engine Seeking has been completed, ignore the event");
945 MOZ_ASSERT(mState
.IsSeekingData());
947 const auto currentTime
= mEngine
->GetCurrentPosition();
948 auto* state
= mState
.AsSeekingData();
949 LOG("OnEngineSeeked, target=%" PRId64
", currentTime=%" PRId64
,
950 state
->GetTargetTime().ToMicroseconds(), currentTime
.ToMicroseconds());
951 // It's possible to receive multiple seeked event if we seek the engine
952 // before the previous seeking finishes, so we would wait until the last
953 // seeking is finished.
954 if (currentTime
>= state
->GetTargetTime()) {
955 state
->mWaitingEngineSeeked
= false;
956 CheckIfSeekCompleted();
960 void ExternalEngineStateMachine::OnBufferingStarted() {
962 mOnNextFrameStatus
.Notify(
963 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING
);
965 WaitForData(MediaData::Type::AUDIO_DATA
);
968 WaitForData(MediaData::Type::VIDEO_DATA
);
972 void ExternalEngineStateMachine::OnBufferingEnded() {
974 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE
);
977 void ExternalEngineStateMachine::OnEnded() {
979 if (mSentPlaybackEndedEvent
) {
982 LOG("Playback is ended");
983 mOnNextFrameStatus
.Notify(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE
);
984 mOnPlaybackEvent
.Notify(MediaPlaybackEvent::PlaybackEnded
);
985 mSentPlaybackEndedEvent
= true;
988 void ExternalEngineStateMachine::OnTimeupdate() {
993 mCurrentPosition
= mEngine
->GetCurrentPosition();
994 if (mDuration
.Ref().ref() < mCurrentPosition
.Ref()) {
995 mDuration
= Some(mCurrentPosition
.Ref());
999 void ExternalEngineStateMachine::NotifyEventInternal(
1000 ExternalEngineEvent aEvent
) {
1001 AssertOnTaskQueue();
1002 AUTO_PROFILER_LABEL("ExternalEngineStateMachine::NotifyEventInternal",
1004 LOG("Receive event %s", ExternalEngineEventToStr(aEvent
));
1005 if (mState
.IsShutdownEngine()) {
1009 case ExternalEngineEvent::LoadedMetaData
:
1010 // We read metadata by ourselves, ignore this if there is any.
1013 case ExternalEngineEvent::LoadedFirstFrame
:
1014 OnLoadedFirstFrame();
1016 case ExternalEngineEvent::LoadedData
:
1019 case ExternalEngineEvent::Waiting
:
1022 case ExternalEngineEvent::Playing
:
1025 case ExternalEngineEvent::Seeked
:
1028 case ExternalEngineEvent::BufferingStarted
:
1029 OnBufferingStarted();
1031 case ExternalEngineEvent::BufferingEnded
:
1034 case ExternalEngineEvent::Timeupdate
:
1037 case ExternalEngineEvent::Ended
:
1040 case ExternalEngineEvent::RequestForAudio
:
1041 mHasEnoughAudio
= false;
1042 if (ShouldRunEngineUpdateForRequest()) {
1043 RunningEngineUpdate(MediaData::Type::AUDIO_DATA
);
1046 case ExternalEngineEvent::RequestForVideo
:
1047 mHasEnoughVideo
= false;
1048 if (ShouldRunEngineUpdateForRequest()) {
1049 RunningEngineUpdate(MediaData::Type::VIDEO_DATA
);
1052 case ExternalEngineEvent::AudioEnough
:
1053 mHasEnoughAudio
= true;
1055 case ExternalEngineEvent::VideoEnough
:
1056 mHasEnoughVideo
= true;
1059 MOZ_ASSERT_UNREACHABLE("Undefined event!");
1064 bool ExternalEngineStateMachine::ShouldRunEngineUpdateForRequest() {
1065 // Running engine update will request new data, which could be run on
1066 // `RunningEngine` or `SeekingData` state. However, in `SeekingData` we should
1067 // only request new data after finishing reader seek, otherwise the reader
1068 // would start requesting data from a wrong position.
1069 return mState
.IsRunningEngine() ||
1070 (mState
.AsSeekingData() &&
1071 !mState
.AsSeekingData()->mWaitingReaderSeeked
);
1074 void ExternalEngineStateMachine::NotifyErrorInternal(
1075 const MediaResult
& aError
) {
1076 AssertOnTaskQueue();
1077 LOG("Engine error: %s", aError
.Description().get());
1078 if (aError
== NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR
) {
1079 // The external engine doesn't support the type, try to notify the decoder
1080 // to use our own state machine again.
1082 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR
));
1083 } else if (aError
== NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR
) {
1084 RecoverFromCDMProcessCrashIfNeeded();
1086 DecodeError(aError
);
1090 void ExternalEngineStateMachine::RecoverFromCDMProcessCrashIfNeeded() {
1091 AssertOnTaskQueue();
1092 if (mState
.IsRecoverEngine()) {
1095 ProcessCrashMonitor::NotifyCrash();
1096 if (!ProcessCrashMonitor::ShouldRecoverProcess()) {
1097 LOG("CDM process has crashed too many times, abort recovery");
1099 MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR
));
1103 LOG("CDM process crashed, recover the engine again (last time=%" PRId64
")",
1104 mCurrentPosition
.Ref().ToMicroseconds());
1105 ChangeStateTo(State::RecoverEngine
);
1107 mVideoDataRequest
.DisconnectIfExists();
1108 mVideoWaitRequest
.DisconnectIfExists();
1111 mAudioDataRequest
.DisconnectIfExists();
1112 mAudioWaitRequest
.DisconnectIfExists();
1114 // Ask the reader to shutdown current decoders which are no longer available
1115 // due to the remote process crash.
1116 mReader
->ReleaseResources();
1120 media::TimeUnit
ExternalEngineStateMachine::GetVideoThreshold() {
1121 AssertOnTaskQueue();
1122 if (auto* state
= mState
.AsSeekingData()) {
1123 return state
->GetTargetTime();
1125 return mCurrentPosition
.Ref();
1128 void ExternalEngineStateMachine::UpdateSecondaryVideoContainer() {
1129 AssertOnTaskQueue();
1130 LOG("UpdateSecondaryVideoContainer=%p", mSecondaryVideoContainer
.Ref().get());
1131 mOnSecondaryVideoContainerInstalled
.Notify(mSecondaryVideoContainer
.Ref());
1134 RefPtr
<SetCDMPromise
> ExternalEngineStateMachine::SetCDMProxy(
1136 if (mState
.IsShutdownEngine()) {
1137 return SetCDMPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
1140 if (mState
.IsInitEngine() && mState
.AsInitEngine()->mInitPromise
) {
1141 LOG("SetCDMProxy is called before init");
1142 mState
.AsInitEngine()->mInitPromise
->Then(
1143 OwnerThread(), __func__
,
1144 [self
= RefPtr
{this}, proxy
= RefPtr
{aProxy
},
1145 this](const GenericNonExclusivePromise::ResolveOrRejectValue
& aVal
) {
1147 ->Then(OwnerThread(), __func__
,
1148 [self
= RefPtr
{this},
1149 this](const SetCDMPromise::ResolveOrRejectValue
& aVal
) {
1150 mSetCDMProxyRequest
.Complete();
1151 if (aVal
.IsResolve()) {
1152 mSetCDMProxyPromise
.Resolve(true, __func__
);
1154 mSetCDMProxyPromise
.Reject(NS_ERROR_DOM_MEDIA_CDM_ERR
,
1158 ->Track(mSetCDMProxyRequest
);
1160 return mSetCDMProxyPromise
.Ensure(__func__
);
1163 // TODO : set CDM proxy again if we recreate the media engine after crash.
1164 LOG("SetCDMProxy=%p", aProxy
);
1165 MOZ_DIAGNOSTIC_ASSERT(mEngine
);
1166 if (!mEngine
->SetCDMProxy(aProxy
)) {
1167 LOG("Failed to set CDM proxy on the engine");
1168 return SetCDMPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CDM_ERR
, __func__
);
1170 return MediaDecoderStateMachineBase::SetCDMProxy(aProxy
);
1173 bool ExternalEngineStateMachine::IsCDMProxySupported(CDMProxy
* aProxy
) {
1176 // 1=enabled encrypted and clear, 2=enabled encrytped
1177 if (StaticPrefs::media_wmf_media_engine_enabled() != 1 &&
1178 StaticPrefs::media_wmf_media_engine_enabled() != 2) {
1182 // The CDM needs to be hosted in the same process of the external engine, and
1183 // only WMFCDM meets this requirement.
1184 return aProxy
->AsWMFCDMProxy();
1196 } // namespace mozilla