1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "MediaFormatReader.h"
13 #include "AllocationPolicy.h"
15 # include "AOMDecoder.h"
17 #include "DecoderBenchmark.h"
18 #include "MediaData.h"
19 #include "MediaDataDecoderProxy.h"
20 #include "MediaInfo.h"
21 #include "MP4Decoder.h"
22 #include "PDMFactory.h"
23 #include "PerformanceRecorder.h"
24 #include "VideoFrameContainer.h"
25 #include "VideoUtils.h"
26 #include "VPXDecoder.h"
27 #include "mozilla/AbstractThread.h"
28 #include "mozilla/CDMProxy.h"
29 #include "mozilla/ClearOnShutdown.h"
30 #include "mozilla/NotNull.h"
31 #include "mozilla/Preferences.h"
32 #include "mozilla/ProfilerLabels.h"
33 #include "mozilla/ProfilerMarkers.h"
34 #include "mozilla/SharedThreadPool.h"
35 #include "mozilla/StaticPrefs_media.h"
36 #include "mozilla/TaskQueue.h"
37 #include "mozilla/Unused.h"
38 #include "nsContentUtils.h"
39 #include "nsLiteralString.h"
40 #include "nsPrintfCString.h"
41 #include "nsTHashSet.h"
43 using namespace mozilla::media
;
45 static mozilla::LazyLogModule
sFormatDecoderLog("MediaFormatReader");
46 mozilla::LazyLogModule
gMediaDemuxerLog("MediaDemuxer");
48 #define LOG(arg, ...) \
49 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
50 __func__, ##__VA_ARGS__)
51 #define LOGV(arg, ...) \
52 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
53 __func__, ##__VA_ARGS__)
55 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
59 using MediaDataDecoderID
= void*;
62 * This class tracks shutdown promises to ensure all decoders are shut down
63 * completely before MFR continues the rest of the shutdown procedure.
65 class MediaFormatReader::ShutdownPromisePool
{
68 : mOnShutdownComplete(new ShutdownPromise::Private(__func__
)) {}
70 // Return a promise which will be resolved when all the tracking promises
71 // are resolved. Note no more promises should be added for tracking once
72 // this function is called.
73 RefPtr
<ShutdownPromise
> Shutdown();
75 // Track a shutdown promise.
76 void Track(const RefPtr
<ShutdownPromise
>& aPromise
);
78 // Shut down a decoder and track its shutdown promise.
79 void ShutdownDecoder(already_AddRefed
<MediaDataDecoder
> aDecoder
) {
80 Track(RefPtr
<MediaDataDecoder
>(aDecoder
)->Shutdown());
84 bool mShutdown
= false;
85 const RefPtr
<ShutdownPromise::Private
> mOnShutdownComplete
;
86 nsTHashSet
<RefPtr
<ShutdownPromise
>> mPromises
;
89 RefPtr
<ShutdownPromise
> MediaFormatReader::ShutdownPromisePool::Shutdown() {
90 MOZ_DIAGNOSTIC_ASSERT(!mShutdown
);
92 if (mPromises
.Count() == 0) {
93 mOnShutdownComplete
->Resolve(true, __func__
);
95 return mOnShutdownComplete
;
98 void MediaFormatReader::ShutdownPromisePool::Track(
99 const RefPtr
<ShutdownPromise
>& aPromise
) {
100 MOZ_DIAGNOSTIC_ASSERT(!mShutdown
);
101 MOZ_DIAGNOSTIC_ASSERT(!mPromises
.Contains(aPromise
));
102 mPromises
.Insert(aPromise
);
103 aPromise
->Then(AbstractThread::GetCurrent(), __func__
, [aPromise
, this]() {
104 MOZ_DIAGNOSTIC_ASSERT(mPromises
.Contains(aPromise
));
105 mPromises
.Remove(aPromise
);
106 if (mShutdown
&& mPromises
.Count() == 0) {
107 mOnShutdownComplete
->Resolve(true, __func__
);
112 void MediaFormatReader::DecoderData::ShutdownDecoder() {
113 MOZ_ASSERT(mOwner
->OnTaskQueue());
115 MutexAutoLock
lock(mMutex
);
118 // No decoder to shut down.
123 // Flush is is in action. Shutdown will be initiated after flush completes.
124 MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise
);
125 mOwner
->mShutdownPromisePool
->Track(mShutdownPromise
->Ensure(__func__
));
126 // The order of decoder creation and shutdown is handled by LocalAllocPolicy
127 // and ShutdownPromisePool. MFR can now reset these members to a fresh state
128 // and be ready to create new decoders again without explicitly waiting for
129 // flush/shutdown to complete.
130 mShutdownPromise
= nullptr;
133 // No flush is in action. We can shut down the decoder now.
134 mOwner
->mShutdownPromisePool
->Track(mDecoder
->Shutdown());
137 // mShutdownPromisePool will handle the order of decoder shutdown so
138 // we can forget mDecoder and be ready to create a new one.
140 mDescription
= "shutdown"_ns
;
141 mHasReportedVideoHardwareSupportTelemtry
= false;
142 mOwner
->ScheduleUpdate(mType
== MediaData::Type::AUDIO_DATA
143 ? TrackType::kAudioTrack
144 : TrackType::kVideoTrack
);
147 void MediaFormatReader::DecoderData::Flush() {
148 AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK
);
149 MOZ_ASSERT(mOwner
->OnTaskQueue());
151 if (mFlushing
|| mFlushed
) {
152 // Flush still pending or already flushed, nothing more to do.
155 mDecodeRequest
.DisconnectIfExists();
156 mDrainRequest
.DisconnectIfExists();
157 mDrainState
= DrainState::None
;
158 CancelWaitingForKey();
160 mNumSamplesInput
= 0;
161 mNumSamplesOutput
= 0;
164 TrackType type
= mType
== MediaData::Type::AUDIO_DATA
165 ? TrackType::kAudioTrack
166 : TrackType::kVideoTrack
;
168 MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise
);
169 mShutdownPromise
= new SharedShutdownPromiseHolder();
170 RefPtr
<SharedShutdownPromiseHolder
> p
= mShutdownPromise
;
171 RefPtr
<MediaDataDecoder
> d
= mDecoder
;
172 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
173 "flushing", DDNoValue
{});
174 mDecoder
->Flush()->Then(
175 mOwner
->OwnerThread(), __func__
,
176 [type
, this, p
, d
]() {
177 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
179 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
180 "flushed", DDNoValue
{});
182 // Shutdown happened before flush completes.
183 // Let's continue to shut down the decoder. Note
184 // we don't access |this| because this decoder
185 // is no longer managed by MFR::DecoderData.
186 d
->Shutdown()->ChainTo(p
->Steal(), __func__
);
190 mShutdownPromise
= nullptr;
191 mOwner
->ScheduleUpdate(type
);
193 [type
, this, p
, d
](const MediaResult
& aError
) {
194 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
196 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
197 "flush_error", aError
);
199 d
->Shutdown()->ChainTo(p
->Steal(), __func__
);
203 mShutdownPromise
= nullptr;
204 mOwner
->NotifyError(type
, aError
);
210 class MediaFormatReader::DecoderFactory
{
211 using InitPromise
= MediaDataDecoder::InitPromise
;
212 using TokenPromise
= AllocPolicy::Promise
;
213 using Token
= AllocPolicy::Token
;
214 using CreateDecoderPromise
= PlatformDecoderModule::CreateDecoderPromise
;
217 explicit DecoderFactory(MediaFormatReader
* aOwner
)
218 : mAudio(aOwner
->mAudio
, TrackInfo::kAudioTrack
, aOwner
->OwnerThread()),
219 mVideo(aOwner
->mVideo
, TrackInfo::kVideoTrack
, aOwner
->OwnerThread()),
220 mOwner(WrapNotNull(aOwner
)) {
221 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
223 DecoderDoctorLogger::LinkParentAndChild(
224 aOwner
, "decoder factory", "MediaFormatReader::DecoderFactory", this);
228 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
232 void CreateDecoder(TrackType aTrack
);
234 // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
235 // pristine state so CreateDecoder() is ready to be called again immediately.
236 void ShutdownDecoder(TrackType aTrack
) {
237 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
238 aTrack
== TrackInfo::kVideoTrack
);
239 auto& data
= aTrack
== TrackInfo::kAudioTrack
? mAudio
: mVideo
;
240 data
.mPolicy
->Cancel();
241 data
.mTokenRequest
.DisconnectIfExists();
242 if (data
.mLiveToken
) {
243 // We haven't completed creation of the decoder, and it hasn't been
245 data
.mLiveToken
= nullptr;
246 // The decoder will be shutdown as soon as it's available and tracked by
247 // the ShutdownPromisePool.
248 mOwner
->mShutdownPromisePool
->Track(data
.mCreateDecoderPromise
->Then(
249 mOwner
->mTaskQueue
, __func__
,
250 [](CreateDecoderPromise::ResolveOrRejectValue
&& aResult
) {
251 if (aResult
.IsReject()) {
252 return ShutdownPromise::CreateAndResolve(true, __func__
);
254 return aResult
.ResolveValue()->Shutdown();
256 // Free the token to leave room for a new decoder.
257 data
.mToken
= nullptr;
259 data
.mInitRequest
.DisconnectIfExists();
261 mOwner
->mShutdownPromisePool
->ShutdownDecoder(data
.mDecoder
.forget());
263 data
.mStage
= Stage::None
;
264 MOZ_ASSERT(!data
.mToken
);
268 enum class Stage
: int8_t { None
, WaitForToken
, CreateDecoder
, WaitForInit
};
271 Data(DecoderData
& aOwnerData
, TrackType aTrack
, TaskQueue
* aThread
)
272 : mOwnerData(aOwnerData
),
274 mPolicy(new SingleAllocPolicy(aTrack
, aThread
)) {}
275 DecoderData
& mOwnerData
;
276 const TrackType mTrack
;
277 RefPtr
<SingleAllocPolicy
> mPolicy
;
278 Stage mStage
= Stage::None
;
279 RefPtr
<Token
> mToken
;
280 RefPtr
<MediaDataDecoder
> mDecoder
;
281 MozPromiseRequestHolder
<TokenPromise
> mTokenRequest
;
282 struct DecoderCancelled
: public SupportsWeakPtr
{
283 NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled
)
285 ~DecoderCancelled() = default;
287 // Set when decoder is about to be created. If cleared before the decoder
288 // creation promise is resolved; it indicates that Shutdown() was called and
289 // further processing such as initialization should stop.
290 RefPtr
<DecoderCancelled
> mLiveToken
;
291 RefPtr
<CreateDecoderPromise
> mCreateDecoderPromise
;
292 MozPromiseRequestHolder
<InitPromise
> mInitRequest
;
295 void RunStage(Data
& aData
);
296 void DoCreateDecoder(Data
& aData
);
297 void DoInitDecoder(Data
& aData
);
299 // guaranteed to be valid by the owner.
300 const NotNull
<MediaFormatReader
*> mOwner
;
303 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack
) {
304 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
305 aTrack
== TrackInfo::kVideoTrack
);
306 Data
& data
= aTrack
== TrackInfo::kAudioTrack
? mAudio
: mVideo
;
307 MOZ_DIAGNOSTIC_ASSERT_IF(mOwner
->GetDecoderData(data
.mTrack
).IsEncrypted(),
312 void MediaFormatReader::DecoderFactory::RunStage(Data
& aData
) {
313 switch (aData
.mStage
) {
315 MOZ_DIAGNOSTIC_ASSERT(!aData
.mToken
);
316 aData
.mPolicy
->Alloc()
318 mOwner
->OwnerThread(), __func__
,
319 [this, &aData
](RefPtr
<Token
> aToken
) {
320 aData
.mTokenRequest
.Complete();
321 aData
.mToken
= std::move(aToken
);
322 aData
.mStage
= Stage::CreateDecoder
;
326 aData
.mTokenRequest
.Complete();
327 aData
.mStage
= Stage::None
;
329 ->Track(aData
.mTokenRequest
);
330 aData
.mStage
= Stage::WaitForToken
;
334 case Stage::WaitForToken
: {
335 MOZ_DIAGNOSTIC_ASSERT(!aData
.mToken
);
336 MOZ_DIAGNOSTIC_ASSERT(aData
.mTokenRequest
.Exists());
340 case Stage::CreateDecoder
: {
341 MOZ_DIAGNOSTIC_ASSERT(aData
.mToken
);
342 MOZ_DIAGNOSTIC_ASSERT(!aData
.mDecoder
);
343 MOZ_DIAGNOSTIC_ASSERT(!aData
.mInitRequest
.Exists());
345 DoCreateDecoder(aData
);
346 aData
.mStage
= Stage::WaitForInit
;
350 case Stage::WaitForInit
: {
351 MOZ_DIAGNOSTIC_ASSERT((aData
.mDecoder
&& aData
.mInitRequest
.Exists()) ||
358 void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data
& aData
) {
359 AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK
);
360 auto& ownerData
= aData
.mOwnerData
;
361 auto& decoder
= mOwner
->GetDecoderData(aData
.mTrack
);
363 RefPtr
<PDMFactory
> platform
= new PDMFactory();
364 if (decoder
.IsEncrypted()) {
365 MOZ_DIAGNOSTIC_ASSERT(mOwner
->mCDMProxy
);
366 platform
->SetCDMProxy(mOwner
->mCDMProxy
);
369 RefPtr
<PlatformDecoderModule::CreateDecoderPromise
> p
;
370 MediaFormatReader
* owner
= mOwner
;
371 auto onWaitingForKeyEvent
=
372 [owner
= ThreadSafeWeakPtr
<MediaFormatReader
>(owner
)]() {
373 RefPtr
<MediaFormatReader
> mfr(owner
);
374 MOZ_DIAGNOSTIC_ASSERT(mfr
, "The MediaFormatReader didn't wait for us");
375 return mfr
? &mfr
->OnTrackWaitingForKeyProducer() : nullptr;
378 switch (aData
.mTrack
) {
379 case TrackInfo::kAudioTrack
: {
380 p
= platform
->CreateDecoder(
381 {*ownerData
.GetCurrentInfo()->GetAsAudioInfo(), mOwner
->mCrashHelper
,
382 CreateDecoderParams::UseNullDecoder(ownerData
.mIsNullDecode
),
383 TrackInfo::kAudioTrack
, std::move(onWaitingForKeyEvent
),
384 mOwner
->mMediaEngineId
, mOwner
->mTrackingId
});
388 case TrackType::kVideoTrack
: {
389 // Decoders use the layers backend to decide if they can use hardware
390 // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
391 using Option
= CreateDecoderParams::Option
;
392 using OptionSet
= CreateDecoderParams::OptionSet
;
394 p
= platform
->CreateDecoder(
395 {*ownerData
.GetCurrentInfo()->GetAsVideoInfo(),
396 mOwner
->mKnowsCompositor
, mOwner
->GetImageContainer(),
397 mOwner
->mCrashHelper
,
398 CreateDecoderParams::UseNullDecoder(ownerData
.mIsNullDecode
),
399 TrackType::kVideoTrack
, std::move(onWaitingForKeyEvent
),
400 CreateDecoderParams::VideoFrameRate(ownerData
.mMeanRate
.Mean()),
401 OptionSet(ownerData
.mHardwareDecodingDisabled
402 ? Option::HardwareDecoderNotAllowed
404 mOwner
->mMediaEngineId
, mOwner
->mTrackingId
});
409 p
= PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(
410 NS_ERROR_DOM_MEDIA_FATAL_ERR
, __func__
);
413 aData
.mLiveToken
= MakeRefPtr
<Data::DecoderCancelled
>();
415 aData
.mCreateDecoderPromise
= p
->Then(
416 mOwner
->OwnerThread(), __func__
,
417 [this, &aData
, &ownerData
, live
= WeakPtr
{aData
.mLiveToken
},
418 owner
= ThreadSafeWeakPtr
<MediaFormatReader
>(owner
)](
419 RefPtr
<MediaDataDecoder
>&& aDecoder
) {
421 return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder
),
424 aData
.mLiveToken
= nullptr;
425 aData
.mDecoder
= new MediaDataDecoderProxy(
426 aDecoder
.forget(), do_AddRef(ownerData
.mTaskQueue
.get()));
427 aData
.mDecoder
= new AllocationWrapper(aData
.mDecoder
.forget(),
428 aData
.mToken
.forget());
429 DecoderDoctorLogger::LinkParentAndChild(
430 aData
.mDecoder
.get(), "decoder",
431 "MediaFormatReader::DecoderFactory", this);
433 DoInitDecoder(aData
);
435 return CreateDecoderPromise::CreateAndResolve(aData
.mDecoder
, __func__
);
438 live
= WeakPtr
{aData
.mLiveToken
}](const MediaResult
& aError
) {
439 NS_WARNING("Error constructing decoders");
441 return CreateDecoderPromise::CreateAndReject(aError
, __func__
);
443 aData
.mLiveToken
= nullptr;
444 aData
.mToken
= nullptr;
445 aData
.mStage
= Stage::None
;
446 aData
.mOwnerData
.mDescription
= aError
.Description();
447 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log
,
448 "create_decoder_error", aError
);
449 mOwner
->NotifyError(aData
.mTrack
, aError
);
451 return CreateDecoderPromise::CreateAndReject(aError
, __func__
);
455 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data
& aData
) {
456 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK
);
457 auto& ownerData
= aData
.mOwnerData
;
459 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log
,
460 "initialize_decoder", DDNoValue
{});
461 aData
.mDecoder
->Init()
463 mOwner
->OwnerThread(), __func__
,
464 [this, &aData
, &ownerData
](TrackType aTrack
) {
465 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
467 aData
.mInitRequest
.Complete();
468 aData
.mStage
= Stage::None
;
469 MutexAutoLock
lock(ownerData
.mMutex
);
470 ownerData
.mDecoder
= std::move(aData
.mDecoder
);
471 ownerData
.mDescription
= ownerData
.mDecoder
->GetDescriptionName();
472 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
473 DDLogCategory::Log
, "decoder_initialized", DDNoValue
{});
474 DecoderDoctorLogger::LinkParentAndChild(
475 "MediaFormatReader::DecoderData", &ownerData
, "decoder",
476 ownerData
.mDecoder
.get());
477 mOwner
->SetVideoDecodeThreshold();
478 mOwner
->ScheduleUpdate(aTrack
);
479 if (aTrack
== TrackInfo::kVideoTrack
) {
480 DecoderBenchmark::CheckVersion(
481 ownerData
.GetCurrentInfo()->mMimeType
);
483 if (aTrack
== TrackInfo::kAudioTrack
) {
484 ownerData
.mProcessName
= ownerData
.mDecoder
->GetProcessName();
485 ownerData
.mCodecName
= ownerData
.mDecoder
->GetCodecName();
488 [this, &aData
, &ownerData
](const MediaResult
& aError
) {
489 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
491 aData
.mInitRequest
.Complete();
492 MOZ_RELEASE_ASSERT(!ownerData
.mDecoder
,
493 "Can't have a decoder already set");
494 aData
.mStage
= Stage::None
;
495 mOwner
->mShutdownPromisePool
->ShutdownDecoder(
496 aData
.mDecoder
.forget());
497 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
498 DDLogCategory::Log
, "initialize_decoder_error", aError
);
499 mOwner
->NotifyError(aData
.mTrack
, aError
);
501 ->Track(aData
.mInitRequest
);
504 // DemuxerProxy ensures that the original main demuxer is only ever accessed
505 // via its own dedicated task queue.
506 // This ensure that the reader's taskqueue will never blocked while a demuxer
507 // is itself blocked attempting to access the MediaCache or the MediaResource.
508 class MediaFormatReader::DemuxerProxy
{
509 using TrackType
= TrackInfo::TrackType
;
513 explicit DemuxerProxy(MediaDataDemuxer
* aDemuxer
)
514 : mTaskQueue(TaskQueue::Create(
515 GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
516 "DemuxerProxy::mTaskQueue")),
517 mData(new Data(aDemuxer
)) {
518 MOZ_COUNT_CTOR(DemuxerProxy
);
521 MOZ_COUNTED_DTOR(DemuxerProxy
)
523 RefPtr
<ShutdownPromise
> Shutdown() {
524 RefPtr
<Data
> data
= std::move(mData
);
525 return InvokeAsync(mTaskQueue
, __func__
, [data
]() {
526 // We need to clear our reference to the demuxer now. So that in the event
527 // the init promise wasn't resolved, such as what can happen with the
528 // mediasource demuxer that is waiting on more data, it will force the
529 // init promise to be rejected.
530 data
->mDemuxer
= nullptr;
531 data
->mAudioDemuxer
= nullptr;
532 data
->mVideoDemuxer
= nullptr;
533 return ShutdownPromise::CreateAndResolve(true, __func__
);
537 RefPtr
<MediaDataDemuxer::InitPromise
> Init();
539 Wrapper
* GetTrackDemuxer(TrackType aTrack
, uint32_t aTrackNumber
) {
540 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
543 case TrackInfo::kAudioTrack
:
544 return mData
->mAudioDemuxer
;
545 case TrackInfo::kVideoTrack
:
546 return mData
->mVideoDemuxer
;
552 uint32_t GetNumberTracks(TrackType aTrack
) const {
553 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
556 case TrackInfo::kAudioTrack
:
557 return mData
->mNumAudioTrack
;
558 case TrackInfo::kVideoTrack
:
559 return mData
->mNumVideoTrack
;
565 bool IsSeekable() const {
566 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
568 return mData
->mSeekable
;
571 bool IsSeekableOnlyInBufferedRanges() const {
572 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
574 return mData
->mSeekableOnlyInBufferedRange
;
577 UniquePtr
<EncryptionInfo
> GetCrypto() const {
578 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
580 if (!mData
->mCrypto
) {
583 auto crypto
= MakeUnique
<EncryptionInfo
>();
584 *crypto
= *mData
->mCrypto
;
588 RefPtr
<NotifyDataArrivedPromise
> NotifyDataArrived();
590 bool ShouldComputeStartTime() const {
591 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
593 return mData
->mShouldComputeStartTime
;
597 const RefPtr
<TaskQueue
> mTaskQueue
;
599 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data
)
601 explicit Data(MediaDataDemuxer
* aDemuxer
)
602 : mInitDone(false), mDemuxer(aDemuxer
) {}
604 Atomic
<bool> mInitDone
;
605 // Only ever accessed over mTaskQueue once.
606 RefPtr
<MediaDataDemuxer
> mDemuxer
;
607 // Only accessed once InitPromise has been resolved and immutable after.
608 // So we can safely access them without the use of the mutex.
609 uint32_t mNumAudioTrack
= 0;
610 RefPtr
<Wrapper
> mAudioDemuxer
;
611 uint32_t mNumVideoTrack
= 0;
612 RefPtr
<Wrapper
> mVideoDemuxer
;
613 bool mSeekable
= false;
614 bool mSeekableOnlyInBufferedRange
= false;
615 bool mShouldComputeStartTime
= true;
616 UniquePtr
<EncryptionInfo
> mCrypto
;
624 class MediaFormatReader::DemuxerProxy::Wrapper
: public MediaTrackDemuxer
{
626 Wrapper(MediaTrackDemuxer
* aTrackDemuxer
, TaskQueue
* aTaskQueue
)
627 : mMutex("TrackDemuxer Mutex"),
628 mTaskQueue(aTaskQueue
),
629 mGetSamplesMayBlock(aTrackDemuxer
->GetSamplesMayBlock()),
630 mInfo(aTrackDemuxer
->GetInfo()),
631 mTrackDemuxer(aTrackDemuxer
) {
632 DecoderDoctorLogger::LogConstructionAndBase(
633 "MediaFormatReader::DemuxerProxy::Wrapper", this,
634 static_cast<const MediaTrackDemuxer
*>(this));
635 DecoderDoctorLogger::LinkParentAndChild(
636 "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
640 UniquePtr
<TrackInfo
> GetInfo() const override
{
644 return mInfo
->Clone();
647 RefPtr
<SeekPromise
> Seek(const TimeUnit
& aTime
) override
{
648 RefPtr
<Wrapper
> self
= this;
650 mTaskQueue
, __func__
,
651 [self
, aTime
]() { return self
->mTrackDemuxer
->Seek(aTime
); })
653 mTaskQueue
, __func__
,
654 [self
](const TimeUnit
& aTime
) {
655 self
->UpdateRandomAccessPoint();
656 return SeekPromise::CreateAndResolve(aTime
, __func__
);
658 [self
](const MediaResult
& aError
) {
659 self
->UpdateRandomAccessPoint();
660 return SeekPromise::CreateAndReject(aError
, __func__
);
664 RefPtr
<SamplesPromise
> GetSamples(int32_t aNumSamples
) override
{
665 RefPtr
<Wrapper
> self
= this;
666 return InvokeAsync(mTaskQueue
, __func__
,
667 [self
, aNumSamples
]() {
668 return self
->mTrackDemuxer
->GetSamples(aNumSamples
);
671 mTaskQueue
, __func__
,
672 [self
](RefPtr
<SamplesHolder
> aSamples
) {
673 self
->UpdateRandomAccessPoint();
674 return SamplesPromise::CreateAndResolve(aSamples
.forget(),
677 [self
](const MediaResult
& aError
) {
678 self
->UpdateRandomAccessPoint();
679 return SamplesPromise::CreateAndReject(aError
, __func__
);
683 bool GetSamplesMayBlock() const override
{ return mGetSamplesMayBlock
; }
685 void Reset() override
{
686 RefPtr
<Wrapper
> self
= this;
687 nsresult rv
= mTaskQueue
->Dispatch(NS_NewRunnableFunction(
688 "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
689 [self
]() { self
->mTrackDemuxer
->Reset(); }));
690 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
694 nsresult
GetNextRandomAccessPoint(TimeUnit
* aTime
) override
{
695 MutexAutoLock
lock(mMutex
);
696 if (NS_SUCCEEDED(mNextRandomAccessPointResult
)) {
697 *aTime
= mNextRandomAccessPoint
;
699 return mNextRandomAccessPointResult
;
702 RefPtr
<SkipAccessPointPromise
> SkipToNextRandomAccessPoint(
703 const TimeUnit
& aTimeThreshold
) override
{
704 RefPtr
<Wrapper
> self
= this;
706 mTaskQueue
, __func__
,
707 [self
, aTimeThreshold
]() {
708 return self
->mTrackDemuxer
->SkipToNextRandomAccessPoint(
712 mTaskQueue
, __func__
,
713 [self
](uint32_t aVal
) {
714 self
->UpdateRandomAccessPoint();
715 return SkipAccessPointPromise::CreateAndResolve(aVal
, __func__
);
717 [self
](const SkipFailureHolder
& aError
) {
718 self
->UpdateRandomAccessPoint();
719 return SkipAccessPointPromise::CreateAndReject(aError
, __func__
);
723 TimeIntervals
GetBuffered() override
{
724 MutexAutoLock
lock(mMutex
);
728 void BreakCycles() override
{}
731 Mutex mMutex MOZ_UNANNOTATED
;
732 const RefPtr
<TaskQueue
> mTaskQueue
;
733 const bool mGetSamplesMayBlock
;
734 const UniquePtr
<TrackInfo
> mInfo
;
735 // mTrackDemuxer is only ever accessed on demuxer's task queue.
736 RefPtr
<MediaTrackDemuxer
> mTrackDemuxer
;
737 // All following members are protected by mMutex
738 nsresult mNextRandomAccessPointResult
= NS_OK
;
739 TimeUnit mNextRandomAccessPoint
;
740 TimeIntervals mBuffered
;
741 friend class DemuxerProxy
;
744 RefPtr
<MediaTrackDemuxer
> trackDemuxer
= std::move(mTrackDemuxer
);
745 nsresult rv
= mTaskQueue
->Dispatch(NS_NewRunnableFunction(
746 "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
747 [trackDemuxer
]() { trackDemuxer
->BreakCycles(); }));
748 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
750 DecoderDoctorLogger::LogDestruction(
751 "MediaFormatReader::DemuxerProxy::Wrapper", this);
754 void UpdateRandomAccessPoint() {
755 MOZ_ASSERT(mTaskQueue
->IsCurrentThreadIn());
756 if (!mTrackDemuxer
) {
760 MutexAutoLock
lock(mMutex
);
761 mNextRandomAccessPointResult
=
762 mTrackDemuxer
->GetNextRandomAccessPoint(&mNextRandomAccessPoint
);
765 void UpdateBuffered() {
766 MOZ_ASSERT(mTaskQueue
->IsCurrentThreadIn());
767 if (!mTrackDemuxer
) {
771 MutexAutoLock
lock(mMutex
);
772 mBuffered
= mTrackDemuxer
->GetBuffered();
776 RefPtr
<MediaDataDemuxer::InitPromise
> MediaFormatReader::DemuxerProxy::Init() {
777 AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK
);
778 using InitPromise
= MediaDataDemuxer::InitPromise
;
780 RefPtr
<Data
> data
= mData
;
781 RefPtr
<TaskQueue
> taskQueue
= mTaskQueue
;
782 return InvokeAsync(mTaskQueue
, __func__
,
783 [data
, taskQueue
]() {
784 if (!data
->mDemuxer
) {
785 return InitPromise::CreateAndReject(
786 NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
788 return data
->mDemuxer
->Init();
792 [data
, taskQueue
]() {
793 AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK
);
794 if (!data
->mDemuxer
) { // Was shutdown.
795 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
798 data
->mNumAudioTrack
=
799 data
->mDemuxer
->GetNumberTracks(TrackInfo::kAudioTrack
);
800 if (data
->mNumAudioTrack
) {
801 RefPtr
<MediaTrackDemuxer
> d
=
802 data
->mDemuxer
->GetTrackDemuxer(TrackInfo::kAudioTrack
, 0);
804 RefPtr
<Wrapper
> wrapper
=
805 new DemuxerProxy::Wrapper(d
, taskQueue
);
806 wrapper
->UpdateBuffered();
807 data
->mAudioDemuxer
= wrapper
;
808 DecoderDoctorLogger::LinkParentAndChild(
809 data
->mDemuxer
.get(), "decoder factory wrapper",
810 "MediaFormatReader::DecoderFactory::Wrapper",
814 data
->mNumVideoTrack
=
815 data
->mDemuxer
->GetNumberTracks(TrackInfo::kVideoTrack
);
816 if (data
->mNumVideoTrack
) {
817 RefPtr
<MediaTrackDemuxer
> d
=
818 data
->mDemuxer
->GetTrackDemuxer(TrackInfo::kVideoTrack
, 0);
820 RefPtr
<Wrapper
> wrapper
=
821 new DemuxerProxy::Wrapper(d
, taskQueue
);
822 wrapper
->UpdateBuffered();
823 data
->mVideoDemuxer
= wrapper
;
824 DecoderDoctorLogger::LinkParentAndChild(
825 data
->mDemuxer
.get(), "decoder factory wrapper",
826 "MediaFormatReader::DecoderFactory::Wrapper",
830 data
->mCrypto
= data
->mDemuxer
->GetCrypto();
831 data
->mSeekable
= data
->mDemuxer
->IsSeekable();
832 data
->mSeekableOnlyInBufferedRange
=
833 data
->mDemuxer
->IsSeekableOnlyInBufferedRanges();
834 data
->mShouldComputeStartTime
=
835 data
->mDemuxer
->ShouldComputeStartTime();
836 data
->mInitDone
= true;
837 return InitPromise::CreateAndResolve(NS_OK
, __func__
);
839 [](const MediaResult
& aError
) {
840 return InitPromise::CreateAndReject(aError
, __func__
);
844 RefPtr
<MediaFormatReader::NotifyDataArrivedPromise
>
845 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
846 RefPtr
<Data
> data
= mData
;
847 return InvokeAsync(mTaskQueue
, __func__
, [data
]() {
848 if (!data
->mDemuxer
) {
850 return NotifyDataArrivedPromise::CreateAndReject(
851 NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
853 data
->mDemuxer
->NotifyDataArrived();
854 if (data
->mAudioDemuxer
) {
855 data
->mAudioDemuxer
->UpdateBuffered();
857 if (data
->mVideoDemuxer
) {
858 data
->mVideoDemuxer
->UpdateBuffered();
860 return NotifyDataArrivedPromise::CreateAndResolve(true, __func__
);
864 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit
& aInit
,
865 MediaDataDemuxer
* aDemuxer
)
867 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR
),
868 "MediaFormatReader::mTaskQueue",
869 /* aSupportsTailDispatch = */ true)),
870 mAudio(this, MediaData::Type::AUDIO_DATA
,
871 StaticPrefs::media_audio_max_decode_error()),
872 mVideo(this, MediaData::Type::VIDEO_DATA
,
873 StaticPrefs::media_video_max_decode_error()),
874 mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),
875 mWatchManager(this, OwnerThread()),
876 mIsWatchingWorkingInfo(false),
877 mDemuxer(new DemuxerProxy(aDemuxer
)),
878 mDemuxerInitDone(false),
879 mPendingNotifyDataArrived(false),
880 mLastReportedNumDecodedFrames(0),
881 mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe
),
882 mKnowsCompositor(aInit
.mKnowsCompositor
),
884 mTrackDemuxersMayBlock(false),
885 mSeekScheduled(false),
886 mVideoFrameContainer(aInit
.mVideoFrameContainer
),
887 mCrashHelper(aInit
.mCrashHelper
),
888 mDecoderFactory(new DecoderFactory(this)),
889 mShutdownPromisePool(new ShutdownPromisePool()),
890 mBuffered(mTaskQueue
, TimeIntervals(),
891 "MediaFormatReader::mBuffered (Canonical)"),
892 mFrameStats(aInit
.mFrameStats
),
893 mMediaDecoderOwnerID(aInit
.mMediaDecoderOwnerID
),
894 mTrackingId(std::move(aInit
.mTrackingId
)),
895 mReadMetadataStartTime(Nothing()),
896 mReadMetaDataTime(TimeDuration::Zero()),
897 mTotalWaitingForVideoDataTime(TimeDuration::Zero()) {
898 MOZ_ASSERT(aDemuxer
);
899 MOZ_COUNT_CTOR(MediaFormatReader
);
900 DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
902 DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
904 DDLINKCHILD("demuxer", aDemuxer
);
905 mOnTrackWaitingForKeyListener
= OnTrackWaitingForKey().Connect(
906 mTaskQueue
, this, &MediaFormatReader::NotifyWaitingForKey
);
909 MediaFormatReader::~MediaFormatReader() {
910 MOZ_COUNT_DTOR(MediaFormatReader
);
911 MOZ_ASSERT(mShutdown
);
914 RefPtr
<ShutdownPromise
> MediaFormatReader::Shutdown() {
915 MOZ_ASSERT(OnTaskQueue());
918 mDemuxerInitRequest
.DisconnectIfExists();
919 mNotifyDataArrivedPromise
.DisconnectIfExists();
920 mMetadataPromise
.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
921 mSeekPromise
.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
922 mSkipRequest
.DisconnectIfExists();
923 mSetCDMPromise
.RejectIfExists(
924 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
925 "MediaFormatReader is shutting down"),
928 if (mIsWatchingWorkingInfo
) {
929 mWatchManager
.Unwatch(mWorkingInfoChanged
,
930 &MediaFormatReader::NotifyTrackInfoUpdated
);
932 mWatchManager
.Shutdown();
934 if (mAudio
.HasPromise()) {
935 mAudio
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
937 if (mVideo
.HasPromise()) {
938 mVideo
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
942 mAudio
.ResetDemuxer();
943 mAudio
.mTrackDemuxer
->BreakCycles();
945 MutexAutoLock
lock(mAudio
.mMutex
);
946 mAudio
.mTrackDemuxer
= nullptr;
949 ShutdownDecoder(TrackInfo::kAudioTrack
);
953 mVideo
.ResetDemuxer();
954 mVideo
.mTrackDemuxer
->BreakCycles();
956 MutexAutoLock
lock(mVideo
.mMutex
);
957 mVideo
.mTrackDemuxer
= nullptr;
960 ShutdownDecoder(TrackInfo::kVideoTrack
);
963 mShutdownPromisePool
->Track(mDemuxer
->Shutdown());
966 mOnTrackWaitingForKeyListener
.Disconnect();
969 return mShutdownPromisePool
->Shutdown()->Then(
970 OwnerThread(), __func__
, this, &MediaFormatReader::TearDownDecoders
,
971 &MediaFormatReader::TearDownDecoders
);
974 void MediaFormatReader::ShutdownDecoder(TrackType aTrack
) {
975 LOGV("%s", TrackTypeToStr(aTrack
));
977 // Shut down the pending decoder if any.
978 mDecoderFactory
->ShutdownDecoder(aTrack
);
980 auto& decoder
= GetDecoderData(aTrack
);
981 // Flush the decoder if necessary.
984 // Shut down the decoder if any.
985 decoder
.ShutdownDecoder();
988 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
989 MOZ_ASSERT(OnTaskQueue());
990 if (!StaticPrefs::media_mediacapabilities_from_database()) {
993 auto& decoder
= GetDecoderData(TrackInfo::kVideoTrack
);
994 if (decoder
.GetCurrentInfo() && decoder
.GetCurrentInfo()->GetAsVideoInfo()) {
995 VideoInfo info
= *(decoder
.GetCurrentInfo()->GetAsVideoInfo());
996 info
.SetFrameRate(static_cast<int32_t>(ceil(decoder
.mMeanRate
.Mean())));
997 mOnStoreDecoderBenchmark
.Notify(std::move(info
));
1001 void MediaFormatReader::NotifyTrackInfoUpdated() {
1002 MOZ_ASSERT(OnTaskQueue());
1003 if (mWorkingInfoChanged
) {
1004 mWorkingInfoChanged
= false;
1006 VideoInfo videoInfo
;
1007 AudioInfo audioInfo
;
1009 MutexAutoLock
lock(mVideo
.mMutex
);
1011 videoInfo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
1015 MutexAutoLock
lock(mAudio
.mMutex
);
1017 audioInfo
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
1021 mTrackInfoUpdatedEvent
.Notify(videoInfo
, audioInfo
);
1025 RefPtr
<ShutdownPromise
> MediaFormatReader::TearDownDecoders() {
1026 if (mAudio
.mTaskQueue
) {
1027 mAudio
.mTaskQueue
->BeginShutdown();
1028 mAudio
.mTaskQueue
->AwaitShutdownAndIdle();
1029 mAudio
.mTaskQueue
= nullptr;
1031 if (mVideo
.mTaskQueue
) {
1032 mVideo
.mTaskQueue
->BeginShutdown();
1033 mVideo
.mTaskQueue
->AwaitShutdownAndIdle();
1034 mVideo
.mTaskQueue
= nullptr;
1037 mDecoderFactory
= nullptr;
1038 mVideoFrameContainer
= nullptr;
1041 mBuffered
.DisconnectAll();
1042 return mTaskQueue
->BeginShutdown();
1045 nsresult
MediaFormatReader::Init() {
1046 MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1049 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
1050 "MFR::mAudio::mTaskQueue");
1053 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
1054 "MFR::mVideo::mTaskQueue");
1059 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack
) {
1060 // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
1061 // and would create new one for specific track in the next Update.
1062 MOZ_ASSERT(OnTaskQueue());
1064 if (mSetCDMPromise
.IsEmpty()) {
1068 MOZ_ASSERT(mCDMProxy
);
1069 if (mSetCDMForTracks
.contains(aTrack
)) {
1070 mSetCDMForTracks
-= aTrack
;
1073 if (mSetCDMForTracks
.isEmpty()) {
1074 LOGV("%s : Done ", __func__
);
1075 mSetCDMPromise
.Resolve(/* aResolveValue = */ true, __func__
);
1077 ScheduleUpdate(TrackInfo::kAudioTrack
);
1080 ScheduleUpdate(TrackInfo::kVideoTrack
);
1084 LOGV("%s : %s track is ready.", __func__
, TrackTypeToStr(aTrack
));
1088 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack
) {
1089 MOZ_ASSERT(OnTaskQueue());
1090 LOGV("%s : %s", __func__
, TrackTypeToStr(aTrack
));
1092 mSetCDMForTracks
+= aTrack
;
1094 // An old cdm proxy exists, so detaching old cdm proxy by shutting down
1095 // MediaDataDecoder.
1096 ShutdownDecoder(aTrack
);
1098 ScheduleUpdate(aTrack
);
1101 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack
) {
1102 MOZ_ASSERT(OnTaskQueue());
1103 return GetDecoderData(aTrack
).IsEncrypted() &&
1104 mSetCDMForTracks
.contains(aTrack
) && !mCDMProxy
;
1107 RefPtr
<SetCDMPromise
> MediaFormatReader::SetCDMProxy(CDMProxy
* aProxy
) {
1108 MOZ_ASSERT(OnTaskQueue());
1109 LOGV("SetCDMProxy (%p)", aProxy
);
1112 return SetCDMPromise::CreateAndReject(
1113 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
1114 "MediaFormatReader is shutting down"),
1118 mSetCDMPromise
.RejectIfExists(
1119 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
1120 "Another new CDM proxy is being set."),
1123 // Shutdown all decoders as switching CDM proxy indicates that it's
1124 // inappropriate for the existing decoders to continue decoding via the old
1127 PrepareToSetCDMForTrack(TrackInfo::kAudioTrack
);
1130 PrepareToSetCDMForTrack(TrackInfo::kVideoTrack
);
1135 if (!mInitDone
|| mSetCDMForTracks
.isEmpty() || !mCDMProxy
) {
1136 // 1) MFR is not initialized yet or
1137 // 2) Demuxer is initialized without active audio and video or
1138 // 3) A null cdm proxy is set
1139 // the promise can be resolved directly.
1140 mSetCDMForTracks
.clear();
1141 return SetCDMPromise::CreateAndResolve(/* aResolveValue = */ true,
1145 RefPtr
<SetCDMPromise
> p
= mSetCDMPromise
.Ensure(__func__
);
1149 bool MediaFormatReader::IsWaitingOnCDMResource() {
1150 MOZ_ASSERT(OnTaskQueue());
1151 return IsEncrypted() && !mCDMProxy
;
1154 RefPtr
<MediaFormatReader::MetadataPromise
>
1155 MediaFormatReader::AsyncReadMetadata() {
1156 AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK
);
1157 MOZ_ASSERT(OnTaskQueue());
1159 MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise
.IsEmpty());
1162 // We are returning from dormant.
1163 MetadataHolder metadata
;
1164 metadata
.mInfo
= MakeUnique
<MediaInfo
>(mInfo
);
1165 return MetadataPromise::CreateAndResolve(std::move(metadata
), __func__
);
1168 if (!mReadMetadataStartTime
) {
1169 mReadMetadataStartTime
= Some(TimeStamp::Now());
1172 RefPtr
<MetadataPromise
> p
= mMetadataPromise
.Ensure(__func__
);
1175 ->Then(OwnerThread(), __func__
, this,
1176 &MediaFormatReader::OnDemuxerInitDone
,
1177 &MediaFormatReader::OnDemuxerInitFailed
)
1178 ->Track(mDemuxerInitRequest
);
1182 void MediaFormatReader::OnDemuxerInitDone(const MediaResult
& aResult
) {
1183 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK
);
1184 MOZ_ASSERT(OnTaskQueue());
1185 mDemuxerInitRequest
.Complete();
1187 if (NS_FAILED(aResult
) && StaticPrefs::media_playback_warnings_as_errors()) {
1188 mMetadataPromise
.Reject(aResult
, __func__
);
1192 mDemuxerInitDone
= true;
1194 UniquePtr
<MetadataTags
> tags(MakeUnique
<MetadataTags
>());
1196 RefPtr
<PDMFactory
> platform
;
1197 if (!IsWaitingOnCDMResource()) {
1198 platform
= new PDMFactory();
1201 // To decode, we need valid video and a place to put it.
1202 bool videoActive
= !!mDemuxer
->GetNumberTracks(TrackInfo::kVideoTrack
) &&
1203 GetImageContainer();
1206 // We currently only handle the first video track.
1207 MutexAutoLock
lock(mVideo
.mMutex
);
1208 mVideo
.mTrackDemuxer
= mDemuxer
->GetTrackDemuxer(TrackInfo::kVideoTrack
, 0);
1209 if (!mVideo
.mTrackDemuxer
) {
1210 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1214 UniquePtr
<TrackInfo
> videoInfo
= mVideo
.mTrackDemuxer
->GetInfo();
1215 videoActive
= videoInfo
&& videoInfo
->IsValid();
1218 platform
->SupportsMimeType(videoInfo
->mMimeType
).isEmpty()) {
1219 // We have no decoder for this track. Error.
1220 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1223 mInfo
.mVideo
= *videoInfo
->GetAsVideoInfo();
1224 mVideo
.mWorkingInfo
= MakeUnique
<VideoInfo
>(mInfo
.mVideo
);
1225 for (const MetadataTag
& tag
: videoInfo
->mTags
) {
1226 tags
->InsertOrUpdate(tag
.mKey
, tag
.mValue
);
1228 mWorkingInfoChanged
= true;
1229 mVideo
.mOriginalInfo
= std::move(videoInfo
);
1230 mTrackDemuxersMayBlock
|= mVideo
.mTrackDemuxer
->GetSamplesMayBlock();
1232 mVideo
.mTrackDemuxer
->BreakCycles();
1233 mVideo
.mTrackDemuxer
= nullptr;
1237 bool audioActive
= !!mDemuxer
->GetNumberTracks(TrackInfo::kAudioTrack
);
1239 MutexAutoLock
lock(mAudio
.mMutex
);
1240 mAudio
.mTrackDemuxer
= mDemuxer
->GetTrackDemuxer(TrackInfo::kAudioTrack
, 0);
1241 if (!mAudio
.mTrackDemuxer
) {
1242 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1246 UniquePtr
<TrackInfo
> audioInfo
= mAudio
.mTrackDemuxer
->GetInfo();
1247 // We actively ignore audio tracks that we know we can't play.
1248 audioActive
= audioInfo
&& audioInfo
->IsValid() &&
1250 !platform
->SupportsMimeType(audioInfo
->mMimeType
).isEmpty());
1253 mInfo
.mAudio
= *audioInfo
->GetAsAudioInfo();
1254 mAudio
.mWorkingInfo
= MakeUnique
<AudioInfo
>(mInfo
.mAudio
);
1255 for (const MetadataTag
& tag
: audioInfo
->mTags
) {
1256 tags
->InsertOrUpdate(tag
.mKey
, tag
.mValue
);
1258 mWorkingInfoChanged
= true;
1259 mAudio
.mOriginalInfo
= std::move(audioInfo
);
1260 mTrackDemuxersMayBlock
|= mAudio
.mTrackDemuxer
->GetSamplesMayBlock();
1262 mAudio
.mTrackDemuxer
->BreakCycles();
1263 mAudio
.mTrackDemuxer
= nullptr;
1267 UniquePtr
<EncryptionInfo
> crypto
= mDemuxer
->GetCrypto();
1268 if (crypto
&& crypto
->IsEncrypted()) {
1269 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1270 for (uint32_t i
= 0; i
< crypto
->mInitDatas
.Length(); i
++) {
1271 mOnEncrypted
.Notify(crypto
->mInitDatas
[i
].mInitData
,
1272 crypto
->mInitDatas
[i
].mType
);
1274 mInfo
.mCrypto
= *crypto
;
1277 auto videoDuration
= HasVideo() ? mInfo
.mVideo
.mDuration
: TimeUnit::Zero();
1278 auto audioDuration
= HasAudio() ? mInfo
.mAudio
.mDuration
: TimeUnit::Zero();
1280 // If the duration is 0 on both audio and video, it mMetadataDuration is to be
1281 // Nothing(). Duration will use buffered ranges.
1282 if (videoDuration
.IsPositive() || audioDuration
.IsPositive()) {
1283 auto duration
= std::max(videoDuration
, audioDuration
);
1284 mInfo
.mMetadataDuration
= Some(duration
);
1287 mInfo
.mMediaSeekable
= mDemuxer
->IsSeekable();
1288 mInfo
.mMediaSeekableOnlyInBufferedRanges
=
1289 mDemuxer
->IsSeekableOnlyInBufferedRanges();
1291 if (!videoActive
&& !audioActive
) {
1292 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1296 mTags
= std::move(tags
);
1299 // Try to get the start time.
1300 // For MSE case, the start time of each track is assumed to be 0.
1301 // For others, we must demux the first sample to know the start time for each
1303 if (!mDemuxer
->ShouldComputeStartTime()) {
1304 mAudio
.mFirstDemuxedSampleTime
.emplace(TimeUnit::Zero());
1305 mVideo
.mFirstDemuxedSampleTime
.emplace(TimeUnit::Zero());
1308 RequestDemuxSamples(TrackInfo::kAudioTrack
);
1312 RequestDemuxSamples(TrackInfo::kVideoTrack
);
1316 if (aResult
!= NS_OK
) {
1317 mOnDecodeWarning
.Notify(aResult
);
1320 MaybeResolveMetadataPromise();
1323 void MediaFormatReader::MaybeResolveMetadataPromise() {
1324 MOZ_ASSERT(OnTaskQueue());
1326 if ((HasAudio() && mAudio
.mFirstDemuxedSampleTime
.isNothing()) ||
1327 (HasVideo() && mVideo
.mFirstDemuxedSampleTime
.isNothing())) {
1331 TimeUnit startTime
=
1332 std::min(mAudio
.mFirstDemuxedSampleTime
.refOr(TimeUnit::FromInfinity()),
1333 mVideo
.mFirstDemuxedSampleTime
.refOr(TimeUnit::FromInfinity()));
1335 if (!startTime
.IsInfinite()) {
1336 mInfo
.mStartTime
= startTime
; // mInfo.mStartTime is initialized to 0.
1339 MetadataHolder metadata
;
1340 metadata
.mInfo
= MakeUnique
<MediaInfo
>(mInfo
);
1341 metadata
.mTags
= mTags
->Count() ? std::move(mTags
) : nullptr;
1343 // We now have all the informations required to calculate the initial buffered
1345 mHasStartTime
= true;
1348 mWatchManager
.Watch(mWorkingInfoChanged
,
1349 &MediaFormatReader::NotifyTrackInfoUpdated
);
1350 mIsWatchingWorkingInfo
= true;
1352 if (mReadMetadataStartTime
) {
1353 mReadMetaDataTime
= TimeStamp::Now() - *mReadMetadataStartTime
;
1354 mReadMetadataStartTime
.reset();
1357 mMetadataPromise
.Resolve(std::move(metadata
), __func__
);
1360 bool MediaFormatReader::IsEncrypted() const {
1361 return (HasAudio() && mAudio
.GetCurrentInfo()->mCrypto
.IsEncrypted()) ||
1362 (HasVideo() && mVideo
.GetCurrentInfo()->mCrypto
.IsEncrypted());
1365 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult
& aError
) {
1366 mDemuxerInitRequest
.Complete();
1367 mMetadataPromise
.Reject(aError
, __func__
);
1370 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo
* aInfo
) {
1371 // Called on the MDSM's TaskQueue.
1373 MutexAutoLock
lock(mVideo
.mMutex
);
1375 aInfo
->mVideo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
1379 MutexAutoLock
lock(mAudio
.mMutex
);
1381 aInfo
->mAudio
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
1382 Maybe
<nsCString
> audioProcessPerCodecName
= GetAudioProcessPerCodec();
1383 if (audioProcessPerCodecName
.isSome()) {
1384 Telemetry::ScalarAdd(
1385 Telemetry::ScalarID::MEDIA_AUDIO_PROCESS_PER_CODEC_NAME
,
1386 NS_ConvertUTF8toUTF16(*audioProcessPerCodecName
), 1);
1392 MediaFormatReader::DecoderData
& MediaFormatReader::GetDecoderData(
1394 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
1395 aTrack
== TrackInfo::kVideoTrack
);
1396 if (aTrack
== TrackInfo::kAudioTrack
) {
1402 Maybe
<TimeUnit
> MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold
,
1403 bool aRequestNextVideoKeyFrame
) {
1404 MOZ_ASSERT(OnTaskQueue());
1405 MOZ_ASSERT(HasVideo());
1407 if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1411 // Ensure we have no pending seek going as skip-to-keyframe could return out
1412 // of date information.
1413 if (mVideo
.HasInternalSeekPending()) {
1417 TimeUnit nextKeyframe
;
1418 nsresult rv
= mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
);
1419 if (NS_FAILED(rv
)) {
1420 // Only OggTrackDemuxer with video type gets into here.
1421 // We don't support skip-to-next-frame for this case.
1425 const bool isNextKeyframeValid
=
1426 nextKeyframe
.ToMicroseconds() >= 0 && !nextKeyframe
.IsInfinite();
1427 // If we request the next keyframe, only return times greater than
1428 // aTimeThreshold. Otherwise, data will be already behind the threshold and
1429 // will be eventually discarded somewhere in the media pipeline.
1430 if (aRequestNextVideoKeyFrame
&& isNextKeyframeValid
&&
1431 nextKeyframe
> aTimeThreshold
) {
1432 return Some(nextKeyframe
);
1435 const bool isNextVideoBehindTheThreshold
=
1436 (isNextKeyframeValid
&& nextKeyframe
<= aTimeThreshold
) ||
1437 GetInternalSeekTargetEndTime() < aTimeThreshold
;
1438 return isNextVideoBehindTheThreshold
? Some(aTimeThreshold
) : Nothing();
1441 RefPtr
<MediaFormatReader::VideoDataPromise
> MediaFormatReader::RequestVideoData(
1442 const TimeUnit
& aTimeThreshold
, bool aRequestNextVideoKeyFrame
) {
1443 MOZ_ASSERT(OnTaskQueue());
1444 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise(), "No duplicate sample requests");
1445 // Requesting video can be done independently from audio, even during audio
1446 // seeking. But it shouldn't happen if we're doing video seek.
1447 if (!IsAudioOnlySeeking()) {
1448 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty(),
1449 "No sample requests allowed while seeking");
1450 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.mSeekRequest
.Exists() ||
1451 mVideo
.mTimeThreshold
.isSome());
1452 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1454 LOGV("RequestVideoData(%" PRId64
"), requestNextKeyFrame=%d",
1455 aTimeThreshold
.ToMicroseconds(), aRequestNextVideoKeyFrame
);
1458 LOG("called with no video track");
1459 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR
,
1464 LOG("called mid-seek. Rejecting.");
1465 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1470 NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1471 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1475 if (Maybe
<TimeUnit
> target
=
1476 ShouldSkip(aTimeThreshold
, aRequestNextVideoKeyFrame
)) {
1477 PROFILER_MARKER_UNTYPED("RequestVideoData SkipVideoDemuxToNextKeyFrame",
1479 RefPtr
<VideoDataPromise
> p
= mVideo
.EnsurePromise(__func__
);
1480 SkipVideoDemuxToNextKeyFrame(*target
);
1484 RefPtr
<VideoDataPromise
> p
= mVideo
.EnsurePromise(__func__
);
1485 ScheduleUpdate(TrackInfo::kVideoTrack
);
1490 void MediaFormatReader::OnDemuxFailed(TrackType aTrack
,
1491 const MediaResult
& aError
) {
1492 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK
);
1493 MOZ_ASSERT(OnTaskQueue());
1494 LOG("Failed to demux %s, failure:%s",
1495 aTrack
== TrackType::kVideoTrack
? "video" : "audio",
1496 aError
.ErrorName().get());
1497 auto& decoder
= GetDecoderData(aTrack
);
1498 decoder
.mDemuxRequest
.Complete();
1499 switch (aError
.Code()) {
1500 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
1501 DDLOG(DDLogCategory::Log
,
1502 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1503 : "audio_demux_interruption",
1505 if (!decoder
.mWaitingForDataStartTime
) {
1506 decoder
.RequestDrain();
1508 NotifyEndOfStream(aTrack
);
1510 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
1511 DDLOG(DDLogCategory::Log
,
1512 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1513 : "audio_demux_interruption",
1515 if (!decoder
.mWaitingForDataStartTime
) {
1516 decoder
.RequestDrain();
1518 NotifyWaitingForData(aTrack
);
1520 case NS_ERROR_DOM_MEDIA_CANCELED
:
1521 DDLOG(DDLogCategory::Log
,
1522 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1523 : "audio_demux_interruption",
1525 if (decoder
.HasPromise()) {
1526 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
1530 DDLOG(DDLogCategory::Log
,
1531 aTrack
== TrackType::kVideoTrack
? "video_demux_error"
1532 : "audio_demux_error",
1534 NotifyError(aTrack
, aError
);
1539 void MediaFormatReader::DoDemuxVideo() {
1540 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK
);
1541 using SamplesPromise
= MediaTrackDemuxer::SamplesPromise
;
1543 DDLOG(DDLogCategory::Log
, "video_demuxing", DDNoValue
{});
1544 PerformanceRecorder
<PlaybackStage
> perfRecorder(
1545 MediaStage::RequestDemux
,
1546 mVideo
.GetCurrentInfo()->GetAsVideoInfo()->mImage
.height
);
1547 auto p
= mVideo
.mTrackDemuxer
->GetSamples(1);
1549 RefPtr
<MediaFormatReader
> self
= this;
1550 if (mVideo
.mFirstDemuxedSampleTime
.isNothing()) {
1552 OwnerThread(), __func__
,
1553 [self
](RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1554 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1556 DDLOGEX(self
.get(), DDLogCategory::Log
, "video_first_demuxed",
1558 self
->OnFirstDemuxCompleted(TrackInfo::kVideoTrack
, aSamples
);
1559 return SamplesPromise::CreateAndResolve(aSamples
.forget(), __func__
);
1561 [self
](const MediaResult
& aError
) {
1562 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1564 DDLOGEX(self
.get(), DDLogCategory::Log
, "video_first_demuxing_error",
1566 self
->OnFirstDemuxFailed(TrackInfo::kVideoTrack
, aError
);
1567 return SamplesPromise::CreateAndReject(aError
, __func__
);
1572 OwnerThread(), __func__
,
1573 [self
, perfRecorder(std::move(perfRecorder
))](
1574 const RefPtr
<MediaTrackDemuxer::SamplesHolder
>& aSamples
) mutable {
1575 perfRecorder
.Record();
1576 self
->OnVideoDemuxCompleted(aSamples
);
1578 [self
](const MediaResult
& aError
) { self
->OnVideoDemuxFailed(aError
); })
1579 ->Track(mVideo
.mDemuxRequest
);
1582 void MediaFormatReader::OnVideoDemuxCompleted(
1583 const RefPtr
<MediaTrackDemuxer::SamplesHolder
>& aSamples
) {
1584 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1586 LOGV("%zu video samples demuxed (sid:%d)", aSamples
->GetSamples().Length(),
1587 aSamples
->GetSamples()[0]->mTrackInfo
1588 ? aSamples
->GetSamples()[0]->mTrackInfo
->GetID()
1590 DDLOG(DDLogCategory::Log
, "video_demuxed_samples",
1591 uint64_t(aSamples
->GetSamples().Length()));
1592 mVideo
.mDemuxRequest
.Complete();
1593 mVideo
.mQueuedSamples
.AppendElements(aSamples
->GetSamples());
1594 ScheduleUpdate(TrackInfo::kVideoTrack
);
1597 RefPtr
<MediaFormatReader::AudioDataPromise
>
1598 MediaFormatReader::RequestAudioData() {
1599 MOZ_ASSERT(OnTaskQueue());
1600 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise(), "No duplicate sample requests");
1601 // Requesting audio can be done independently from video, even during video
1602 // seeking. But it shouldn't happen if we're doing audio seek.
1603 if (!IsVideoOnlySeeking()) {
1604 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty(),
1605 "No sample requests allowed while seeking");
1606 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.mSeekRequest
.Exists() ||
1607 mAudio
.mTimeThreshold
.isSome());
1608 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1613 LOG("called with no audio track");
1614 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR
,
1619 LOG("called mid-seek. Rejecting.");
1620 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1625 NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1626 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1630 RefPtr
<AudioDataPromise
> p
= mAudio
.EnsurePromise(__func__
);
1631 ScheduleUpdate(TrackInfo::kAudioTrack
);
1636 void MediaFormatReader::DoDemuxAudio() {
1637 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK
);
1638 using SamplesPromise
= MediaTrackDemuxer::SamplesPromise
;
1640 DDLOG(DDLogCategory::Log
, "audio_demuxing", DDNoValue
{});
1641 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestDemux
);
1642 auto p
= mAudio
.mTrackDemuxer
->GetSamples(1);
1644 RefPtr
<MediaFormatReader
> self
= this;
1645 if (mAudio
.mFirstDemuxedSampleTime
.isNothing()) {
1647 OwnerThread(), __func__
,
1648 [self
](RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1649 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1651 DDLOGEX(self
.get(), DDLogCategory::Log
, "audio_first_demuxed",
1653 self
->OnFirstDemuxCompleted(TrackInfo::kAudioTrack
, aSamples
);
1654 return SamplesPromise::CreateAndResolve(aSamples
.forget(), __func__
);
1656 [self
](const MediaResult
& aError
) {
1657 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1659 DDLOGEX(self
.get(), DDLogCategory::Log
, "audio_first_demuxing_error",
1661 self
->OnFirstDemuxFailed(TrackInfo::kAudioTrack
, aError
);
1662 return SamplesPromise::CreateAndReject(aError
, __func__
);
1667 OwnerThread(), __func__
,
1668 [self
, perfRecorder(std::move(perfRecorder
))](
1669 const RefPtr
<MediaTrackDemuxer::SamplesHolder
>& aSamples
) mutable {
1670 perfRecorder
.Record();
1671 self
->OnAudioDemuxCompleted(aSamples
);
1673 [self
](const MediaResult
& aError
) { self
->OnAudioDemuxFailed(aError
); })
1674 ->Track(mAudio
.mDemuxRequest
);
1677 void MediaFormatReader::OnAudioDemuxCompleted(
1678 const RefPtr
<MediaTrackDemuxer::SamplesHolder
>& aSamples
) {
1679 LOGV("%zu audio samples demuxed (sid:%d)", aSamples
->GetSamples().Length(),
1680 aSamples
->GetSamples()[0]->mTrackInfo
1681 ? aSamples
->GetSamples()[0]->mTrackInfo
->GetID()
1683 DDLOG(DDLogCategory::Log
, "audio_demuxed_samples",
1684 uint64_t(aSamples
->GetSamples().Length()));
1685 mAudio
.mDemuxRequest
.Complete();
1686 mAudio
.mQueuedSamples
.AppendElements(aSamples
->GetSamples());
1687 ScheduleUpdate(TrackInfo::kAudioTrack
);
1690 void MediaFormatReader::NotifyNewOutput(
1691 TrackType aTrack
, MediaDataDecoder::DecodedData
&& aResults
) {
1692 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK
);
1693 MOZ_ASSERT(OnTaskQueue());
1694 auto& decoder
= GetDecoderData(aTrack
);
1695 if (aResults
.IsEmpty()) {
1696 DDLOG(DDLogCategory::Log
,
1697 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio" : "decoded_video",
1698 "no output samples");
1700 for (auto&& sample
: aResults
) {
1701 if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1702 switch (sample
->mType
) {
1703 case MediaData::Type::AUDIO_DATA
:
1704 DDLOGPR(DDLogCategory::Log
,
1705 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1706 : "decoded_got_audio!?",
1707 "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1708 ", \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1709 ", \"duration_us\":%" PRIi64
", \"frames\":%" PRIu32
1710 ", \"channels\":%" PRIu32
", \"rate\":%" PRIu32
1712 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1713 sample
->mTimecode
.ToMicroseconds(),
1714 sample
->mDuration
.ToMicroseconds(),
1715 sample
->As
<AudioData
>()->Frames(),
1716 sample
->As
<AudioData
>()->mChannels
,
1717 sample
->As
<AudioData
>()->mRate
,
1718 sample
->As
<AudioData
>()->Data().Length());
1720 case MediaData::Type::VIDEO_DATA
:
1721 DDLOGPR(DDLogCategory::Log
,
1722 aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1723 : "decoded_got_video!?",
1724 "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1725 ", \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1726 ", \"duration_us\":%" PRIi64
1727 ", \"kf\":%s, \"size\":[%" PRIi32
",%" PRIi32
"]}",
1728 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1729 sample
->mTimecode
.ToMicroseconds(),
1730 sample
->mDuration
.ToMicroseconds(),
1731 sample
->mKeyframe
? "true" : "false",
1732 sample
->As
<VideoData
>()->mDisplay
.width
,
1733 sample
->As
<VideoData
>()->mDisplay
.height
);
1735 case MediaData::Type::RAW_DATA
:
1736 DDLOGPR(DDLogCategory::Log
,
1737 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1738 : aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1740 "{\"type\":\"RawData\", \"offset\":%" PRIi64
1741 " \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1742 ", \"duration_us\":%" PRIi64
", \"kf\":%s}",
1743 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1744 sample
->mTimecode
.ToMicroseconds(),
1745 sample
->mDuration
.ToMicroseconds(),
1746 sample
->mKeyframe
? "true" : "false");
1748 case MediaData::Type::NULL_DATA
:
1749 DDLOGPR(DDLogCategory::Log
,
1750 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1751 : aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1753 "{\"type\":\"NullData\", \"offset\":%" PRIi64
1754 " \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1755 ", \"duration_us\":%" PRIi64
", \"kf\":%s}",
1756 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1757 sample
->mTimecode
.ToMicroseconds(),
1758 sample
->mDuration
.ToMicroseconds(),
1759 sample
->mKeyframe
? "true" : "false");
1763 LOGV("Received new %s sample time:%" PRId64
" duration:%" PRId64
,
1764 TrackTypeToStr(aTrack
), sample
->mTime
.ToMicroseconds(),
1765 sample
->mDuration
.ToMicroseconds());
1766 decoder
.mOutput
.AppendElement(sample
);
1767 decoder
.mNumSamplesOutput
++;
1768 decoder
.mNumOfConsecutiveDecodingError
= 0;
1769 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
= 0;
1770 if (aTrack
== TrackInfo::kAudioTrack
) {
1771 decoder
.mNumOfConsecutiveUtilityCrashes
= 0;
1775 LOG("Done processing new %s samples", TrackTypeToStr(aTrack
));
1777 if (!aResults
.IsEmpty()) {
1778 // We have decoded our first frame, we can now starts to skip future errors.
1779 decoder
.mFirstFrameTime
.reset();
1781 ScheduleUpdate(aTrack
);
1784 void MediaFormatReader::NotifyError(TrackType aTrack
,
1785 const MediaResult
& aError
) {
1786 MOZ_ASSERT(OnTaskQueue());
1787 NS_WARNING(aError
.Description().get());
1788 LOGV("%s Decoding error", TrackTypeToStr(aTrack
));
1789 auto& decoder
= GetDecoderData(aTrack
);
1790 decoder
.mError
= decoder
.HasFatalError() ? decoder
.mError
: Some(aError
);
1792 ScheduleUpdate(aTrack
);
1795 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack
) {
1796 MOZ_ASSERT(OnTaskQueue());
1797 auto& decoder
= GetDecoderData(aTrack
);
1798 decoder
.mWaitingForDataStartTime
= Some(TimeStamp::Now());
1799 if (decoder
.mTimeThreshold
) {
1800 decoder
.mTimeThreshold
.ref().mWaiting
= true;
1802 ScheduleUpdate(aTrack
);
1805 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack
) {
1806 MOZ_ASSERT(OnTaskQueue());
1807 auto& decoder
= GetDecoderData(aTrack
);
1808 mOnWaitingForKey
.Notify();
1809 if (!decoder
.mDecodeRequest
.Exists()) {
1810 LOGV("WaitingForKey received while no pending decode. Ignoring");
1813 decoder
.mWaitingForKey
= true;
1814 ScheduleUpdate(aTrack
);
1817 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack
) {
1818 MOZ_ASSERT(OnTaskQueue());
1819 auto& decoder
= GetDecoderData(aTrack
);
1820 decoder
.mDemuxEOS
= true;
1821 ScheduleUpdate(aTrack
);
1824 bool MediaFormatReader::NeedInput(DecoderData
& aDecoder
) {
1825 // The decoder will not be fed a new raw sample until the current decoding
1826 // requests has completed.
1827 return (aDecoder
.HasPromise() || aDecoder
.mTimeThreshold
.isSome()) &&
1828 !aDecoder
.HasPendingDrain() && !aDecoder
.HasFatalError() &&
1829 !aDecoder
.mDemuxRequest
.Exists() && !aDecoder
.mOutput
.Length() &&
1830 !aDecoder
.HasInternalSeekPending() &&
1831 !aDecoder
.mDecodeRequest
.Exists();
1834 void MediaFormatReader::ScheduleUpdate(TrackType aTrack
) {
1835 MOZ_ASSERT(OnTaskQueue());
1839 auto& decoder
= GetDecoderData(aTrack
);
1840 MOZ_RELEASE_ASSERT(decoder
.GetCurrentInfo(),
1841 "Can only schedule update when track exists");
1843 if (decoder
.mUpdateScheduled
) {
1846 LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack
));
1847 decoder
.mUpdateScheduled
= true;
1848 RefPtr
<nsIRunnable
> task(NewRunnableMethod
<TrackType
>(
1849 "MediaFormatReader::Update", this, &MediaFormatReader::Update
, aTrack
));
1850 nsresult rv
= OwnerThread()->Dispatch(task
.forget());
1851 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
1855 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack
) {
1856 MOZ_ASSERT(OnTaskQueue());
1857 auto& decoder
= GetDecoderData(aTrack
);
1859 if (!decoder
.mReceivedNewData
) {
1863 // We do not want to clear mWaitingForDataStartTime while there are pending
1864 // demuxing or seeking operations that could affect the value of this flag.
1865 // This is in order to ensure that we will retry once they complete as we may
1866 // now have new data that could potentially allow those operations to
1867 // successfully complete if tried again.
1868 if (decoder
.mSeekRequest
.Exists()) {
1869 // Nothing more to do until this operation complete.
1873 if (aTrack
== TrackType::kVideoTrack
&& mSkipRequest
.Exists()) {
1874 LOGV("Skipping in progress, nothing more to do");
1878 if (decoder
.mDemuxRequest
.Exists()) {
1879 // We may have pending operations to process, so we want to continue
1880 // after UpdateReceivedNewData returns.
1884 if (decoder
.HasPendingDrain()) {
1885 // We do not want to clear mWaitingForDataStartTime or mDemuxEOS while
1886 // a drain is in progress in order to properly complete the operation.
1890 decoder
.mReceivedNewData
= false;
1891 if (decoder
.mTimeThreshold
) {
1892 decoder
.mTimeThreshold
.ref().mWaiting
= false;
1894 if (aTrack
== TrackType::kVideoTrack
&& decoder
.mWaitingForDataStartTime
) {
1895 mTotalWaitingForVideoDataTime
+=
1896 TimeStamp::Now() - *decoder
.mWaitingForDataStartTime
;
1898 decoder
.mWaitingForDataStartTime
.reset();
1900 if (decoder
.HasFatalError()) {
1904 if (!mSeekPromise
.IsEmpty() &&
1905 (!IsVideoOnlySeeking() || aTrack
== TrackInfo::kVideoTrack
)) {
1906 MOZ_ASSERT(!decoder
.HasPromise());
1907 MOZ_DIAGNOSTIC_ASSERT(
1908 (IsVideoOnlySeeking() || !mAudio
.mTimeThreshold
) &&
1909 !mVideo
.mTimeThreshold
,
1910 "InternalSeek must have been aborted when Seek was first called");
1911 MOZ_DIAGNOSTIC_ASSERT(
1912 (IsVideoOnlySeeking() || !mAudio
.HasWaitingPromise()) &&
1913 !mVideo
.HasWaitingPromise(),
1914 "Waiting promises must have been rejected when Seek was first called");
1915 if (mVideo
.mSeekRequest
.Exists() ||
1916 (!IsVideoOnlySeeking() && mAudio
.mSeekRequest
.Exists())) {
1917 // Already waiting for a seek to complete. Nothing more to do.
1920 LOG("Attempting Seek");
1924 if (decoder
.HasInternalSeekPending() || decoder
.HasWaitingPromise()) {
1925 if (decoder
.HasInternalSeekPending()) {
1926 LOG("Attempting Internal Seek");
1927 InternalSeek(aTrack
, decoder
.mTimeThreshold
.ref());
1929 if (decoder
.HasWaitingPromise() && !decoder
.IsWaitingForKey() &&
1930 !decoder
.IsWaitingForData()) {
1931 MOZ_ASSERT(!decoder
.HasPromise());
1932 LOG("We have new data. Resolving WaitingPromise");
1933 decoder
.mWaitingPromise
.Resolve(decoder
.mType
, __func__
);
1940 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack
) {
1941 MOZ_ASSERT(OnTaskQueue());
1942 auto& decoder
= GetDecoderData(aTrack
);
1943 MOZ_ASSERT(!decoder
.mDemuxRequest
.Exists());
1945 if (!decoder
.mQueuedSamples
.IsEmpty()) {
1946 // No need to demux new samples.
1950 if (decoder
.mDemuxEOS
) {
1951 // Nothing left to demux.
1952 // We do not want to attempt to demux while in waiting for data mode
1953 // as it would retrigger an unnecessary drain.
1957 LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack
));
1958 if (aTrack
== TrackInfo::kVideoTrack
) {
1965 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack
,
1966 MediaRawData
* aSample
) {
1967 MOZ_ASSERT(OnTaskQueue());
1968 auto& decoder
= GetDecoderData(aTrack
);
1969 RefPtr
<MediaFormatReader
> self
= this;
1970 decoder
.mFlushed
= false;
1971 DDLOGPR(DDLogCategory::Log
,
1972 aTrack
== TrackInfo::kAudioTrack
? "decode_audio"
1973 : aTrack
== TrackInfo::kVideoTrack
? "decode_video"
1975 "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1976 ", \"bytes\":%zu, \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1977 ", \"duration_us\":%" PRIi64
",%s%s}",
1978 aSample
->mOffset
, aSample
->Size(), aSample
->mTime
.ToMicroseconds(),
1979 aSample
->mTimecode
.ToMicroseconds(),
1980 aSample
->mDuration
.ToMicroseconds(), aSample
->mKeyframe
? " kf" : "",
1981 aSample
->mEOS
? " eos" : "");
1983 const int32_t height
=
1984 aTrack
== TrackInfo::kVideoTrack
1985 ? decoder
.GetCurrentInfo()->GetAsVideoInfo()->mImage
.height
1987 MediaInfoFlag flag
= MediaInfoFlag::None
;
1989 aSample
->mKeyframe
? MediaInfoFlag::KeyFrame
: MediaInfoFlag::NonKeyFrame
;
1990 if (aTrack
== TrackInfo::kVideoTrack
) {
1991 flag
|= VideoIsHardwareAccelerated() ? MediaInfoFlag::HardwareDecoding
1992 : MediaInfoFlag::SoftwareDecoding
;
1993 const nsCString
& mimeType
= decoder
.GetCurrentInfo()->mMimeType
;
1994 if (MP4Decoder::IsH264(mimeType
)) {
1995 flag
|= MediaInfoFlag::VIDEO_H264
;
1996 } else if (VPXDecoder::IsVPX(mimeType
, VPXDecoder::VP8
)) {
1997 flag
|= MediaInfoFlag::VIDEO_VP8
;
1998 } else if (VPXDecoder::IsVPX(mimeType
, VPXDecoder::VP9
)) {
1999 flag
|= MediaInfoFlag::VIDEO_VP9
;
2002 else if (AOMDecoder::IsAV1(mimeType
)) {
2003 flag
|= MediaInfoFlag::VIDEO_AV1
;
2007 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestDecode
,
2009 if (mMediaEngineId
&& aSample
->mCrypto
.IsEncrypted()) {
2010 aSample
->mShouldCopyCryptoToRemoteRawData
= true;
2012 decoder
.mDecoder
->Decode(aSample
)
2014 mTaskQueue
, __func__
,
2015 [self
, aTrack
, &decoder
, perfRecorder(std::move(perfRecorder
))](
2016 MediaDataDecoder::DecodedData
&& aResults
) mutable {
2017 perfRecorder
.Record();
2018 decoder
.mDecodeRequest
.Complete();
2019 self
->NotifyNewOutput(aTrack
, std::move(aResults
));
2021 [self
, aTrack
, &decoder
](const MediaResult
& aError
) {
2022 decoder
.mDecodeRequest
.Complete();
2023 self
->NotifyError(aTrack
, aError
);
2025 ->Track(decoder
.mDecodeRequest
);
2028 void MediaFormatReader::HandleDemuxedSamples(
2029 TrackType aTrack
, FrameStatistics::AutoNotifyDecoded
& aA
) {
2030 MOZ_ASSERT(OnTaskQueue());
2032 auto& decoder
= GetDecoderData(aTrack
);
2034 if (decoder
.mFlushing
) {
2035 LOGV("Decoder operation in progress, let it complete.");
2039 if (decoder
.mQueuedSamples
.IsEmpty()) {
2043 RefPtr
<MediaRawData
> sample
= decoder
.mQueuedSamples
[0];
2044 const RefPtr
<TrackInfoSharedPtr
> info
= sample
->mTrackInfo
;
2046 if (info
&& decoder
.mLastStreamSourceID
!= info
->GetID()) {
2047 nsTArray
<RefPtr
<MediaRawData
>> samples
;
2048 if (decoder
.mDecoder
) {
2050 StaticPrefs::media_decoder_recycle_enabled() &&
2051 decoder
.mDecoder
->SupportDecoderRecycling() &&
2052 (*info
)->mCrypto
.mCryptoScheme
==
2053 decoder
.GetCurrentInfo()->mCrypto
.mCryptoScheme
&&
2054 (*info
)->mMimeType
== decoder
.GetCurrentInfo()->mMimeType
;
2055 if (!recyclable
&& decoder
.mTimeThreshold
.isNothing() &&
2056 (decoder
.mNextStreamSourceID
.isNothing() ||
2057 decoder
.mNextStreamSourceID
.ref() != info
->GetID())) {
2058 LOG("%s stream id has changed from:%d to:%d, draining decoder.",
2059 TrackTypeToStr(aTrack
), decoder
.mLastStreamSourceID
, info
->GetID());
2060 decoder
.RequestDrain();
2061 decoder
.mNextStreamSourceID
= Some(info
->GetID());
2062 ScheduleUpdate(aTrack
);
2066 // If flushing is required, it will clear our array of queued samples.
2067 // So we may need to make a copy.
2068 samples
= decoder
.mQueuedSamples
.Clone();
2070 LOG("Decoder does not support recycling, recreate decoder.");
2071 ShutdownDecoder(aTrack
);
2072 // We're going to be using a new decoder following the change of content
2073 // We can attempt to use hardware decoding again.
2074 decoder
.mHardwareDecodingDisabled
= false;
2075 } else if (decoder
.HasWaitingPromise()) {
2080 nsPrintfCString
markerString(
2081 "%s stream id changed from:%" PRIu32
" to:%" PRIu32
,
2082 TrackTypeToStr(aTrack
), decoder
.mLastStreamSourceID
, info
->GetID());
2083 PROFILER_MARKER_TEXT("StreamID Change", MEDIA_PLAYBACK
, {}, markerString
);
2084 LOG("%s", markerString
.get());
2086 if (aTrack
== TrackInfo::kVideoTrack
) {
2087 // We are about to create a new decoder thus the benchmark,
2088 // up to this point, is stored.
2089 NotifyDecoderBenchmarkStore();
2091 decoder
.mNextStreamSourceID
.reset();
2092 decoder
.mLastStreamSourceID
= info
->GetID();
2093 decoder
.mInfo
= info
;
2095 MutexAutoLock
lock(decoder
.mMutex
);
2096 if (aTrack
== TrackInfo::kAudioTrack
) {
2097 decoder
.mWorkingInfo
= MakeUnique
<AudioInfo
>(*info
->GetAsAudioInfo());
2098 } else if (aTrack
== TrackInfo::kVideoTrack
) {
2099 decoder
.mWorkingInfo
= MakeUnique
<VideoInfo
>(*info
->GetAsVideoInfo());
2101 mWorkingInfoChanged
= true;
2104 decoder
.mMeanRate
.Reset();
2106 if (sample
->mKeyframe
) {
2107 if (samples
.Length()) {
2108 decoder
.mQueuedSamples
= std::move(samples
);
2111 auto time
= TimeInterval(sample
->mTime
, sample
->GetEndTime());
2112 InternalSeekTarget seekTarget
=
2113 decoder
.mTimeThreshold
.refOr(InternalSeekTarget(time
, false));
2114 LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64
,
2115 sample
->mTime
.ToMicroseconds());
2116 InternalSeek(aTrack
, seekTarget
);
2121 // Calculate the average frame rate. The first frame will be accounted
2123 decoder
.mMeanRate
.Update(sample
->mDuration
);
2125 if (!decoder
.mDecoder
) {
2126 // In Clear Lead situation, the `mInfo` could change from unencrypted to
2127 // encrypted so we need to ensure the CDM proxy is ready before creating a
2129 if (decoder
.IsEncrypted() &&
2130 (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack
))) {
2133 mDecoderFactory
->CreateDecoder(aTrack
);
2137 LOGV("Input:%" PRId64
" (dts:%" PRId64
" kf:%d)",
2138 sample
->mTime
.ToMicroseconds(), sample
->mTimecode
.ToMicroseconds(),
2140 decoder
.mNumSamplesInput
++;
2141 decoder
.mSizeOfQueue
++;
2142 if (aTrack
== TrackInfo::kVideoTrack
) {
2143 aA
.mStats
.mParsedFrames
++;
2146 DecodeDemuxedSamples(aTrack
, sample
);
2148 decoder
.mQueuedSamples
.RemoveElementAt(0);
2151 media::TimeUnit
MediaFormatReader::GetInternalSeekTargetEndTime() const {
2152 MOZ_ASSERT(OnTaskQueue());
2153 return mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
->EndTime()
2154 : TimeUnit::FromInfinity();
2157 void MediaFormatReader::InternalSeek(TrackType aTrack
,
2158 const InternalSeekTarget
& aTarget
) {
2159 MOZ_ASSERT(OnTaskQueue());
2160 LOG("%s internal seek to %f", TrackTypeToStr(aTrack
),
2161 aTarget
.Time().ToSeconds());
2163 auto& decoder
= GetDecoderData(aTrack
);
2165 decoder
.ResetDemuxer();
2166 decoder
.mTimeThreshold
= Some(aTarget
);
2167 DDLOG(DDLogCategory::Log
, "seeking", DDNoValue
{});
2168 RefPtr
<MediaFormatReader
> self
= this;
2169 decoder
.mTrackDemuxer
->Seek(decoder
.mTimeThreshold
.ref().Time())
2171 OwnerThread(), __func__
,
2172 [self
, aTrack
](TimeUnit aTime
) {
2173 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeked", DDNoValue
{});
2174 auto& decoder
= self
->GetDecoderData(aTrack
);
2175 decoder
.mSeekRequest
.Complete();
2176 MOZ_ASSERT(decoder
.mTimeThreshold
,
2177 "Seek promise must be disconnected when "
2178 "timethreshold is reset");
2179 decoder
.mTimeThreshold
.ref().mHasSeeked
= true;
2180 self
->SetVideoDecodeThreshold();
2181 self
->ScheduleUpdate(aTrack
);
2183 [self
, aTrack
](const MediaResult
& aError
) {
2184 auto& decoder
= self
->GetDecoderData(aTrack
);
2185 decoder
.mSeekRequest
.Complete();
2186 switch (aError
.Code()) {
2187 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
2188 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2190 self
->NotifyWaitingForData(aTrack
);
2192 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
2193 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2195 decoder
.mTimeThreshold
.reset();
2196 self
->NotifyEndOfStream(aTrack
);
2198 case NS_ERROR_DOM_MEDIA_CANCELED
:
2199 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2201 decoder
.mTimeThreshold
.reset();
2204 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_error",
2206 decoder
.mTimeThreshold
.reset();
2207 self
->NotifyError(aTrack
, aError
);
2211 ->Track(decoder
.mSeekRequest
);
2214 void MediaFormatReader::DrainDecoder(TrackType aTrack
) {
2215 AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK
);
2216 MOZ_ASSERT(OnTaskQueue());
2218 auto& decoder
= GetDecoderData(aTrack
);
2219 if (decoder
.mDrainState
== DrainState::Draining
) {
2222 if (!decoder
.mDecoder
||
2223 (decoder
.mDrainState
!= DrainState::PartialDrainPending
&&
2224 decoder
.mNumSamplesInput
== decoder
.mNumSamplesOutput
)) {
2225 // No frames to drain.
2226 LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack
));
2227 decoder
.mDrainState
= DrainState::DrainAborted
;
2228 ScheduleUpdate(aTrack
);
2232 decoder
.mDrainState
= DrainState::Draining
;
2234 DDLOG(DDLogCategory::Log
, "draining", DDNoValue
{});
2235 RefPtr
<MediaFormatReader
> self
= this;
2236 decoder
.mDecoder
->Drain()
2238 mTaskQueue
, __func__
,
2239 [self
, aTrack
, &decoder
](MediaDataDecoder::DecodedData
&& aResults
) {
2240 decoder
.mDrainRequest
.Complete();
2241 DDLOGEX(self
.get(), DDLogCategory::Log
, "drained", DDNoValue
{});
2242 if (aResults
.IsEmpty()) {
2243 decoder
.mDrainState
= DrainState::DrainCompleted
;
2245 self
->NotifyNewOutput(aTrack
, std::move(aResults
));
2246 // Let's see if we have any more data available to drain.
2247 decoder
.mDrainState
= DrainState::PartialDrainPending
;
2249 self
->ScheduleUpdate(aTrack
);
2251 [self
, aTrack
, &decoder
](const MediaResult
& aError
) {
2252 decoder
.mDrainRequest
.Complete();
2253 DDLOGEX(self
.get(), DDLogCategory::Log
, "draining_error", aError
);
2254 self
->NotifyError(aTrack
, aError
);
2256 ->Track(decoder
.mDrainRequest
);
2257 LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack
));
2260 void MediaFormatReader::Update(TrackType aTrack
) {
2261 AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK
);
2262 MOZ_ASSERT(OnTaskQueue());
2268 LOGV("Processing update for %s", TrackTypeToStr(aTrack
));
2270 bool needOutput
= false;
2271 auto& decoder
= GetDecoderData(aTrack
);
2272 decoder
.mUpdateScheduled
= false;
2278 if (aTrack
== TrackType::kVideoTrack
&& mSkipRequest
.Exists()) {
2279 LOGV("Skipping in progress, nothing more to do");
2283 if (UpdateReceivedNewData(aTrack
)) {
2284 LOGV("Nothing more to do");
2288 if (decoder
.mSeekRequest
.Exists()) {
2289 LOGV("Seeking hasn't completed, nothing more to do");
2293 MOZ_DIAGNOSTIC_ASSERT(
2294 !decoder
.HasInternalSeekPending() ||
2295 (!decoder
.mOutput
.Length() && !decoder
.mQueuedSamples
.Length()),
2296 "No frames can be demuxed or decoded while an internal seek is pending");
2298 // Record number of frames decoded and parsed. Automatically update the
2299 // stats counters using the AutoNotifyDecoded stack-based class.
2300 FrameStatistics::AutoNotifyDecoded
a(mFrameStats
);
2302 // Drop any frames found prior our internal seek target.
2303 while (decoder
.mTimeThreshold
&& decoder
.mOutput
.Length()) {
2304 RefPtr
<MediaData
>& output
= decoder
.mOutput
[0];
2305 InternalSeekTarget target
= decoder
.mTimeThreshold
.ref();
2306 auto time
= output
->mTime
;
2307 if (time
>= target
.Time()) {
2308 // We have reached our internal seek target.
2309 decoder
.mTimeThreshold
.reset();
2310 // We might have dropped some keyframes.
2311 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
2313 if (time
< target
.Time() || (target
.mDropTarget
&& target
.Contains(time
))) {
2314 LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2315 TrackTypeToStr(aTrack
), output
->mTime
.ToSeconds(),
2316 target
.Time().ToSeconds(), output
->mKeyframe
);
2317 decoder
.mOutput
.RemoveElementAt(0);
2318 decoder
.mSizeOfQueue
-= 1;
2322 while (decoder
.mOutput
.Length() &&
2323 decoder
.mOutput
[0]->mType
== MediaData::Type::NULL_DATA
) {
2324 LOGV("Dropping null data. Time: %" PRId64
,
2325 decoder
.mOutput
[0]->mTime
.ToMicroseconds());
2326 decoder
.mOutput
.RemoveElementAt(0);
2327 decoder
.mSizeOfQueue
-= 1;
2330 if (decoder
.HasPromise()) {
2332 if (decoder
.mOutput
.Length()) {
2333 RefPtr
<MediaData
> output
= decoder
.mOutput
[0];
2334 decoder
.mOutput
.RemoveElementAt(0);
2335 decoder
.mSizeOfQueue
-= 1;
2336 decoder
.mLastDecodedSampleTime
=
2337 Some(TimeInterval(output
->mTime
, output
->GetEndTime()));
2338 decoder
.mNumSamplesOutputTotal
++;
2339 ReturnOutput(output
, aTrack
);
2340 // We have a decoded sample ready to be returned.
2341 if (aTrack
== TrackType::kVideoTrack
) {
2343 decoder
.mNumSamplesOutputTotal
- mLastReportedNumDecodedFrames
;
2344 a
.mStats
.mDecodedFrames
= static_cast<uint32_t>(delta
);
2345 mLastReportedNumDecodedFrames
= decoder
.mNumSamplesOutputTotal
;
2346 if (output
->mKeyframe
) {
2347 if (mPreviousDecodedKeyframeTime_us
<
2348 output
->mTime
.ToMicroseconds()) {
2349 // There is a previous keyframe -> Record inter-keyframe stats.
2350 uint64_t segment_us
= output
->mTime
.ToMicroseconds() -
2351 mPreviousDecodedKeyframeTime_us
;
2352 a
.mStats
.mInterKeyframeSum_us
+= segment_us
;
2353 a
.mStats
.mInterKeyframeCount
+= 1;
2354 if (a
.mStats
.mInterKeyFrameMax_us
< segment_us
) {
2355 a
.mStats
.mInterKeyFrameMax_us
= segment_us
;
2358 mPreviousDecodedKeyframeTime_us
= output
->mTime
.ToMicroseconds();
2360 bool wasHardwareAccelerated
= mVideo
.mIsHardwareAccelerated
;
2362 mVideo
.mIsHardwareAccelerated
=
2363 mVideo
.mDecoder
&& mVideo
.mDecoder
->IsHardwareAccelerated(error
);
2364 VideoData
* videoData
= output
->As
<VideoData
>();
2365 if (!mVideo
.mHasReportedVideoHardwareSupportTelemtry
||
2366 wasHardwareAccelerated
!= mVideo
.mIsHardwareAccelerated
) {
2367 mVideo
.mHasReportedVideoHardwareSupportTelemtry
= true;
2368 Telemetry::ScalarSet(
2369 Telemetry::ScalarID::MEDIA_VIDEO_HARDWARE_DECODING_SUPPORT
,
2370 NS_ConvertUTF8toUTF16(mVideo
.GetCurrentInfo()->mMimeType
),
2371 !!mVideo
.mIsHardwareAccelerated
);
2372 static constexpr gfx::IntSize HD_VIDEO_SIZE
{1280, 720};
2373 if (videoData
->mDisplay
.width
>= HD_VIDEO_SIZE
.Width() &&
2374 videoData
->mDisplay
.height
>= HD_VIDEO_SIZE
.Height()) {
2375 Telemetry::ScalarSet(
2376 Telemetry::ScalarID::MEDIA_VIDEO_HD_HARDWARE_DECODING_SUPPORT
,
2377 NS_ConvertUTF8toUTF16(mVideo
.GetCurrentInfo()->mMimeType
),
2378 !!mVideo
.mIsHardwareAccelerated
);
2382 // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2384 mVideo
.mIsHardwareAccelerated
=
2385 mVideo
.mIsHardwareAccelerated
||
2386 (videoData
->mImage
&&
2387 videoData
->mImage
->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE
);
2390 } else if (decoder
.HasFatalError()) {
2391 nsCString mimeType
= decoder
.GetCurrentInfo()->mMimeType
;
2392 if (!mimeType
.IsEmpty()) {
2393 Telemetry::ScalarAdd(
2394 Telemetry::ScalarID::MEDIA_DECODE_ERROR_PER_MIME_TYPE
,
2395 NS_ConvertUTF8toUTF16(mimeType
), 1 /* error count */);
2397 LOG("Rejecting %s promise for %s : DECODE_ERROR", TrackTypeToStr(aTrack
),
2399 decoder
.RejectPromise(decoder
.mError
.ref(), __func__
);
2401 } else if (decoder
.HasCompletedDrain()) {
2402 if (decoder
.mDemuxEOS
) {
2403 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack
));
2404 if (aTrack
== TrackInfo::kVideoTrack
) {
2405 // End of video, store the benchmark of the decoder.
2406 NotifyDecoderBenchmarkStore();
2408 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM
, __func__
);
2409 } else if (decoder
.mWaitingForDataStartTime
) {
2410 if (decoder
.mDrainState
== DrainState::DrainCompleted
&&
2411 decoder
.mLastDecodedSampleTime
&& !decoder
.mNextStreamSourceID
) {
2412 // We have completed draining the decoder following WaitingForData.
2413 // Set up the internal seek machinery to be able to resume from the
2414 // last sample decoded.
2415 LOG("Seeking to last sample time: %" PRId64
,
2416 decoder
.mLastDecodedSampleTime
.ref().mStart
.ToMicroseconds());
2417 InternalSeek(aTrack
, InternalSeekTarget(
2418 decoder
.mLastDecodedSampleTime
.ref(), true));
2420 if (!decoder
.mReceivedNewData
) {
2421 LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack
));
2422 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2426 decoder
.mDrainState
= DrainState::None
;
2428 // Now that draining has completed, we check if we have received
2429 // new data again as the result may now be different from the earlier
2431 if (UpdateReceivedNewData(aTrack
) || decoder
.mSeekRequest
.Exists()) {
2432 LOGV("Nothing more to do");
2435 } else if (decoder
.mDemuxEOS
&& !decoder
.HasPendingDrain() &&
2436 decoder
.mQueuedSamples
.IsEmpty()) {
2437 // It is possible to transition from WAITING_FOR_DATA directly to EOS
2438 // state during the internal seek; in which case no draining would occur.
2439 // There is no more samples left to be decoded and we are already in
2440 // EOS state. We can immediately reject the data promise.
2441 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack
));
2442 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM
, __func__
);
2443 } else if (decoder
.mWaitingForKey
) {
2444 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2445 TrackTypeToStr(aTrack
));
2446 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2447 } else if (IsDecoderWaitingForCDM(aTrack
)) {
2448 // Rejecting the promise could lead to entering buffering state for MDSM,
2449 // once a qualified(with the same key system and sessions created by the
2450 // same InitData) new cdm proxy is set, decoding can be resumed.
2451 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2452 TrackTypeToStr(aTrack
));
2453 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2457 if (decoder
.mDrainState
== DrainState::DrainRequested
||
2458 decoder
.mDrainState
== DrainState::PartialDrainPending
) {
2459 if (decoder
.mOutput
.IsEmpty()) {
2460 DrainDecoder(aTrack
);
2465 if (decoder
.mError
&& !decoder
.HasFatalError()) {
2466 MOZ_RELEASE_ASSERT(!decoder
.HasInternalSeekPending(),
2467 "No error can occur while an internal seek is pending");
2470 bool firstFrameDecodingFailedWithHardware
=
2471 decoder
.mFirstFrameTime
&&
2472 decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR
&&
2473 decoder
.mDecoder
&& decoder
.mDecoder
->IsHardwareAccelerated(error
) &&
2474 !decoder
.mHardwareDecodingDisabled
;
2475 bool needsNewDecoder
=
2476 decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER
||
2477 firstFrameDecodingFailedWithHardware
;
2478 // Limit number of process restarts after crash
2479 if ((decoder
.mError
.ref() ==
2480 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
&&
2481 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
++ <
2482 decoder
.mMaxConsecutiveRDDOrGPUCrashes
) ||
2483 (decoder
.mError
.ref() ==
2484 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR
&&
2485 decoder
.mNumOfConsecutiveUtilityCrashes
++ <
2486 decoder
.mMaxConsecutiveUtilityCrashes
)) {
2487 needsNewDecoder
= true;
2489 // For MF CDM crash, it needs to be handled differently. We need to shutdown
2490 // current decoder and report that error to the state machine in order to
2491 // let it to determine if playback can keep going or not.
2492 if (decoder
.mError
.ref() ==
2493 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR
) {
2494 LOG("Error: notify MF CDM crash and shutdown %s decoder",
2495 TrackTypeToStr(aTrack
));
2496 ShutdownDecoder(aTrack
);
2497 decoder
.RejectPromise(decoder
.mError
.ref(), __func__
);
2498 decoder
.mError
.reset();
2502 // We failed to decode on Linux with HW decoder,
2503 // give it another try without HW decoder.
2504 if (decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR
&&
2505 decoder
.mDecoder
->IsHardwareAccelerated(error
)) {
2506 LOG("Error: %s decode error, disable HW acceleration",
2507 TrackTypeToStr(aTrack
));
2508 needsNewDecoder
= true;
2509 decoder
.mHardwareDecodingDisabled
= true;
2511 // RDD process crashed on Linux, give it another try without HW decoder.
2512 if (decoder
.mError
.ref() ==
2513 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
) {
2514 LOG("Error: %s remote decoder crashed, disable HW acceleration",
2515 TrackTypeToStr(aTrack
));
2516 decoder
.mHardwareDecodingDisabled
= true;
2519 // We don't want to expose crash error so switch to
2520 // NS_ERROR_DOM_MEDIA_DECODE_ERR.
2521 if (decoder
.mError
.ref() ==
2522 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
||
2523 decoder
.mError
.ref() ==
2524 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR
) {
2525 decoder
.mError
= Some(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR
,
2526 RESULT_DETAIL("Unable to decode")));
2528 if (!needsNewDecoder
&& ++decoder
.mNumOfConsecutiveDecodingError
>
2529 decoder
.mMaxConsecutiveDecodingError
) {
2530 DDLOG(DDLogCategory::Log
, "too_many_decode_errors", decoder
.mError
.ref());
2531 NotifyError(aTrack
, decoder
.mError
.ref());
2535 if (firstFrameDecodingFailedWithHardware
) {
2536 decoder
.mHardwareDecodingDisabled
= true;
2538 decoder
.mError
.reset();
2540 LOG("%s decoded error count %d RDD crashes count %d",
2541 TrackTypeToStr(aTrack
), decoder
.mNumOfConsecutiveDecodingError
,
2542 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
);
2544 if (needsNewDecoder
) {
2545 LOG("Error: %s needs a new decoder", TrackTypeToStr(aTrack
));
2546 ShutdownDecoder(aTrack
);
2548 if (decoder
.mFirstFrameTime
) {
2549 TimeInterval seekInterval
= TimeInterval(decoder
.mFirstFrameTime
.ref(),
2550 decoder
.mFirstFrameTime
.ref());
2551 InternalSeek(aTrack
, InternalSeekTarget(seekInterval
, false));
2555 TimeUnit nextKeyframe
;
2556 if (aTrack
== TrackType::kVideoTrack
&&
2558 decoder
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
)) &&
2559 !nextKeyframe
.IsInfinite()) {
2560 SkipVideoDemuxToNextKeyFrame(
2561 decoder
.mLastDecodedSampleTime
.refOr(TimeInterval()).Length());
2562 } else if (aTrack
== TrackType::kAudioTrack
) {
2565 DDLOG(DDLogCategory::Log
, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR
);
2566 // We can't recover from this error.
2567 NotifyError(aTrack
, NS_ERROR_DOM_MEDIA_FATAL_ERR
);
2572 bool needInput
= NeedInput(decoder
);
2574 LOGV("Update(%s) ni=%d no=%d in:%" PRIu64
" out:%" PRIu64
2575 " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2576 "ds:%d sid:%u waitcdm:%d",
2577 TrackTypeToStr(aTrack
), needInput
, needOutput
, decoder
.mNumSamplesInput
,
2578 decoder
.mNumSamplesOutput
, uint32_t(size_t(decoder
.mSizeOfQueue
)),
2579 decoder
.mDecodeRequest
.Exists(), decoder
.mFlushing
,
2580 decoder
.mDescription
.get(), uint32_t(decoder
.mOutput
.Length()),
2581 !!decoder
.mWaitingForDataStartTime
, decoder
.mDemuxEOS
,
2582 int32_t(decoder
.mDrainState
), decoder
.mLastStreamSourceID
,
2583 IsDecoderWaitingForCDM(aTrack
));
2585 if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack
)) {
2586 // If the content is encrypted, MFR won't start to create decoder until
2591 if ((decoder
.IsWaitingForData() &&
2592 (!decoder
.mTimeThreshold
|| decoder
.mTimeThreshold
.ref().mWaiting
)) ||
2593 (decoder
.IsWaitingForKey())) {
2594 // Nothing more we can do at present.
2595 LOGV("Still waiting for data or key. data(%d)/key(%d)",
2596 !!decoder
.mWaitingForDataStartTime
, decoder
.mWaitingForKey
);
2600 if (decoder
.CancelWaitingForKey()) {
2601 LOGV("No longer waiting for key. Resolving waiting promise");
2606 LOGV("No need for additional input (pending:%u)",
2607 uint32_t(decoder
.mOutput
.Length()));
2611 // Demux samples if we don't have some.
2612 RequestDemuxSamples(aTrack
);
2614 HandleDemuxedSamples(aTrack
, a
);
2617 void MediaFormatReader::ReturnOutput(MediaData
* aData
, TrackType aTrack
) {
2618 AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK
);
2619 MOZ_ASSERT(GetDecoderData(aTrack
).HasPromise());
2620 MOZ_DIAGNOSTIC_ASSERT(aData
->mType
!= MediaData::Type::NULL_DATA
);
2621 LOG("Resolved data promise for %s [%" PRId64
", %" PRId64
"]",
2622 TrackTypeToStr(aTrack
), aData
->mTime
.ToMicroseconds(),
2623 aData
->GetEndTime().ToMicroseconds());
2625 if (aTrack
== TrackInfo::kAudioTrack
) {
2626 AudioData
* audioData
= aData
->As
<AudioData
>();
2628 if (audioData
->mChannels
!= mInfo
.mAudio
.mChannels
||
2629 audioData
->mRate
!= mInfo
.mAudio
.mRate
) {
2630 LOG("change of audio format (rate:%d->%d). "
2631 "This is an unsupported configuration",
2632 mInfo
.mAudio
.mRate
, audioData
->mRate
);
2633 mInfo
.mAudio
.mRate
= audioData
->mRate
;
2634 mInfo
.mAudio
.mChannels
= audioData
->mChannels
;
2635 MutexAutoLock
lock(mAudio
.mMutex
);
2636 mAudio
.mWorkingInfo
->GetAsAudioInfo()->mRate
= audioData
->mRate
;
2637 mAudio
.mWorkingInfo
->GetAsAudioInfo()->mChannels
= audioData
->mChannels
;
2638 mWorkingInfoChanged
= true;
2640 mAudio
.ResolvePromise(audioData
, __func__
);
2641 } else if (aTrack
== TrackInfo::kVideoTrack
) {
2642 VideoData
* videoData
= aData
->As
<VideoData
>();
2644 if (videoData
->mDisplay
!= mInfo
.mVideo
.mDisplay
) {
2645 LOG("change of video display size (%dx%d->%dx%d)",
2646 mInfo
.mVideo
.mDisplay
.width
, mInfo
.mVideo
.mDisplay
.height
,
2647 videoData
->mDisplay
.width
, videoData
->mDisplay
.height
);
2648 mInfo
.mVideo
.mDisplay
= videoData
->mDisplay
;
2649 MutexAutoLock
lock(mVideo
.mMutex
);
2650 mVideo
.mWorkingInfo
->GetAsVideoInfo()->mDisplay
= videoData
->mDisplay
;
2651 mWorkingInfoChanged
= true;
2654 mozilla::gfx::ColorDepth colorDepth
= videoData
->GetColorDepth();
2655 if (colorDepth
!= mInfo
.mVideo
.mColorDepth
) {
2656 LOG("change of video color depth (enum %u -> enum %u)",
2657 (unsigned)mInfo
.mVideo
.mColorDepth
, (unsigned)colorDepth
);
2658 mInfo
.mVideo
.mColorDepth
= colorDepth
;
2659 MutexAutoLock
lock(mVideo
.mMutex
);
2660 mVideo
.mWorkingInfo
->GetAsVideoInfo()->mColorDepth
= colorDepth
;
2661 mWorkingInfoChanged
= true;
2664 TimeUnit nextKeyframe
;
2665 if (!mVideo
.HasInternalSeekPending() &&
2667 mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
))) {
2668 videoData
->SetNextKeyFrameTime(nextKeyframe
);
2671 mVideo
.ResolvePromise(videoData
, __func__
);
2675 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2676 return SizeOfQueue(TrackInfo::kVideoTrack
);
2679 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2680 return SizeOfQueue(TrackInfo::kAudioTrack
);
2683 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack
) {
2684 auto& decoder
= GetDecoderData(aTrack
);
2685 return decoder
.mSizeOfQueue
;
2688 RefPtr
<MediaFormatReader::WaitForDataPromise
> MediaFormatReader::WaitForData(
2689 MediaData::Type aType
) {
2690 MOZ_ASSERT(OnTaskQueue());
2691 TrackType trackType
= aType
== MediaData::Type::VIDEO_DATA
2692 ? TrackType::kVideoTrack
2693 : TrackType::kAudioTrack
;
2694 auto& decoder
= GetDecoderData(trackType
);
2695 if (!decoder
.IsWaitingForData() && !decoder
.IsWaitingForKey()) {
2696 // We aren't waiting for anything.
2697 return WaitForDataPromise::CreateAndResolve(decoder
.mType
, __func__
);
2699 RefPtr
<WaitForDataPromise
> p
= decoder
.mWaitingPromise
.Ensure(__func__
);
2700 ScheduleUpdate(trackType
);
2704 nsresult
MediaFormatReader::ResetDecode(const TrackSet
& aTracks
) {
2705 AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK
);
2706 MOZ_ASSERT(OnTaskQueue());
2709 mSeekPromise
.RejectIfExists(NS_OK
, __func__
);
2710 mSkipRequest
.DisconnectIfExists();
2712 // Do the same for any data wait promises.
2713 if (aTracks
.contains(TrackInfo::kAudioTrack
)) {
2714 mAudio
.mWaitingPromise
.RejectIfExists(
2715 WaitForDataRejectValue(MediaData::Type::AUDIO_DATA
,
2716 WaitForDataRejectValue::CANCELED
),
2720 if (aTracks
.contains(TrackInfo::kVideoTrack
)) {
2721 mVideo
.mWaitingPromise
.RejectIfExists(
2722 WaitForDataRejectValue(MediaData::Type::VIDEO_DATA
,
2723 WaitForDataRejectValue::CANCELED
),
2727 // Reset miscellaneous seeking state.
2728 mPendingSeekTime
.reset();
2730 if (HasVideo() && aTracks
.contains(TrackInfo::kVideoTrack
)) {
2731 mVideo
.ResetDemuxer();
2732 mVideo
.mFirstFrameTime
= Some(media::TimeUnit::Zero());
2733 Reset(TrackInfo::kVideoTrack
);
2734 if (mVideo
.HasPromise()) {
2735 mVideo
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
2739 if (HasAudio() && aTracks
.contains(TrackInfo::kAudioTrack
)) {
2740 mAudio
.ResetDemuxer();
2741 mVideo
.mFirstFrameTime
= Some(media::TimeUnit::Zero());
2742 Reset(TrackInfo::kAudioTrack
);
2743 if (mAudio
.HasPromise()) {
2744 mAudio
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
2751 void MediaFormatReader::Reset(TrackType aTrack
) {
2752 MOZ_ASSERT(OnTaskQueue());
2753 LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack
));
2755 auto& decoder
= GetDecoderData(aTrack
);
2757 decoder
.ResetState();
2760 LOG("Reset(%s) END", TrackTypeToStr(aTrack
));
2763 void MediaFormatReader::DropDecodedSamples(TrackType aTrack
) {
2764 MOZ_ASSERT(OnTaskQueue());
2765 auto& decoder
= GetDecoderData(aTrack
);
2766 size_t lengthDecodedQueue
= decoder
.mOutput
.Length();
2767 if (lengthDecodedQueue
&& decoder
.mTimeThreshold
.isSome()) {
2768 auto time
= decoder
.mOutput
.LastElement()->mTime
;
2769 if (time
>= decoder
.mTimeThreshold
.ref().Time()) {
2770 // We would have reached our internal seek target.
2771 decoder
.mTimeThreshold
.reset();
2774 decoder
.mOutput
.Clear();
2775 decoder
.mSizeOfQueue
-= lengthDecodedQueue
;
2776 if (aTrack
== TrackInfo::kVideoTrack
&& mFrameStats
) {
2777 mFrameStats
->Accumulate({0, 0, 0, lengthDecodedQueue
, 0, 0});
2781 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold
) {
2782 AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2784 MOZ_ASSERT(OnTaskQueue());
2785 LOG("Skipping up to %" PRId64
, aTimeThreshold
.ToMicroseconds());
2787 // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2788 // As such we can drop all already decoded samples and discard all pending
2790 DropDecodedSamples(TrackInfo::kVideoTrack
);
2792 mVideo
.mTrackDemuxer
->SkipToNextRandomAccessPoint(aTimeThreshold
)
2793 ->Then(OwnerThread(), __func__
, this,
2794 &MediaFormatReader::OnVideoSkipCompleted
,
2795 &MediaFormatReader::OnVideoSkipFailed
)
2796 ->Track(mSkipRequest
);
2799 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped
) {
2800 PROFILER_MARKER_UNTYPED("SkippedVideoDecode", MEDIA_PLAYBACK
);
2801 MOZ_ASSERT(OnTaskQueue());
2803 // Some frames may have been output by the decoder since we initiated the
2804 // videoskip process and we know they would be late.
2805 DropDecodedSamples(TrackInfo::kVideoTrack
);
2806 // Report the pending frames as dropped.
2808 uint32_t droppedDecoderCount
= SizeOfVideoQueueInFrames();
2809 mFrameStats
->Accumulate({0, 0, 0, droppedDecoderCount
, 0, 0});
2812 // Cancel any pending demux request and pending demuxed samples.
2813 mVideo
.mDemuxRequest
.DisconnectIfExists();
2814 Reset(TrackType::kVideoTrack
);
2817 mFrameStats
->Accumulate({aSkipped
, 0, 0, aSkipped
, 0, 0});
2820 mVideo
.mNumSamplesSkippedTotal
+= aSkipped
;
2823 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped
) {
2824 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2826 MOZ_ASSERT(OnTaskQueue());
2827 LOG("Skipping succeeded, skipped %u frames", aSkipped
);
2828 mSkipRequest
.Complete();
2830 DDLOG(DDLogCategory::Log
, "video_skipped", DDNoValue());
2832 VideoSkipReset(aSkipped
);
2834 ScheduleUpdate(TrackInfo::kVideoTrack
);
2837 void MediaFormatReader::OnVideoSkipFailed(
2838 MediaTrackDemuxer::SkipFailureHolder aFailure
) {
2839 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK
);
2840 MOZ_ASSERT(OnTaskQueue());
2841 LOG("Skipping failed, skipped %u frames", aFailure
.mSkipped
);
2842 mSkipRequest
.Complete();
2844 switch (aFailure
.mFailure
.Code()) {
2845 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
2846 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
2847 DDLOG(DDLogCategory::Log
, "video_skipping_interruption",
2849 // Some frames may have been output by the decoder since we initiated the
2850 // videoskip process and we know they would be late.
2851 DropDecodedSamples(TrackInfo::kVideoTrack
);
2852 // We can't complete the skip operation, will just service a video frame
2854 ScheduleUpdate(TrackInfo::kVideoTrack
);
2856 case NS_ERROR_DOM_MEDIA_CANCELED
:
2857 DDLOG(DDLogCategory::Log
, "video_skipping_interruption",
2859 if (mVideo
.HasPromise()) {
2860 mVideo
.RejectPromise(aFailure
.mFailure
, __func__
);
2864 DDLOG(DDLogCategory::Log
, "video_skipping_error", aFailure
.mFailure
);
2865 NotifyError(TrackType::kVideoTrack
, aFailure
.mFailure
);
2870 RefPtr
<MediaFormatReader::SeekPromise
> MediaFormatReader::Seek(
2871 const SeekTarget
& aTarget
) {
2872 AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK
);
2873 MOZ_ASSERT(OnTaskQueue());
2875 LOG("aTarget=(%" PRId64
"), track=%s", aTarget
.GetTime().ToMicroseconds(),
2876 SeekTarget::TrackToStr(aTarget
.GetTrack()));
2878 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty());
2879 MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime
.isNothing());
2880 // Should reset data request, and no pending internal seek.
2881 if (aTarget
.IsAllTracks()) {
2882 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise());
2883 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise());
2884 MOZ_DIAGNOSTIC_ASSERT(mVideo
.mTimeThreshold
.isNothing());
2885 MOZ_DIAGNOSTIC_ASSERT(mAudio
.mTimeThreshold
.isNothing());
2886 } else if (aTarget
.IsVideoOnly()) {
2887 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise());
2888 MOZ_DIAGNOSTIC_ASSERT(mVideo
.mTimeThreshold
.isNothing());
2889 } else if (aTarget
.IsAudioOnly()) {
2890 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise());
2891 MOZ_DIAGNOSTIC_ASSERT(mAudio
.mTimeThreshold
.isNothing());
2894 if (!mInfo
.mMediaSeekable
&& !mInfo
.mMediaSeekableOnlyInBufferedRanges
) {
2895 LOG("Seek() END (Unseekable)");
2896 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
2900 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
2903 SetSeekTarget(aTarget
);
2905 RefPtr
<SeekPromise
> p
= mSeekPromise
.Ensure(__func__
);
2912 void MediaFormatReader::SetSeekTarget(const SeekTarget
& aTarget
) {
2913 MOZ_ASSERT(OnTaskQueue());
2915 mOriginalSeekTarget
= aTarget
;
2916 mFallbackSeekTime
= mPendingSeekTime
= Some(aTarget
.GetTime());
2919 void MediaFormatReader::ScheduleSeek() {
2920 if (mSeekScheduled
) {
2923 mSeekScheduled
= true;
2924 nsresult rv
= OwnerThread()->Dispatch(NewRunnableMethod(
2925 "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek
));
2926 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
2930 void MediaFormatReader::AttemptSeek() {
2931 AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK
);
2932 MOZ_ASSERT(OnTaskQueue());
2934 mSeekScheduled
= false;
2936 if (mPendingSeekTime
.isNothing()) {
2937 LOGV("AttemptSeek, no pending seek time?");
2941 // Only reset the demuxers targeted by this SeekTarget, to avoid A/V sync
2943 const bool isSeekingAudio
= HasAudio() && !mOriginalSeekTarget
.IsVideoOnly();
2944 const bool isSeekingVideo
= HasVideo() && !mOriginalSeekTarget
.IsAudioOnly();
2945 LOG("AttemptSeek, seekingAudio=%d, seekingVideo=%d", isSeekingAudio
,
2947 if (isSeekingVideo
) {
2948 mVideo
.ResetDemuxer();
2949 mVideo
.ResetState();
2951 if (isSeekingAudio
) {
2952 mAudio
.ResetDemuxer();
2953 mAudio
.ResetState();
2956 // If seeking both tracks, seek the video track, and then the audio track when
2957 // the video track seek has completed. Otherwise, only seek a specific track.
2958 if (isSeekingVideo
) {
2960 } else if (isSeekingAudio
) {
2967 void MediaFormatReader::OnSeekFailed(TrackType aTrack
,
2968 const MediaResult
& aError
) {
2969 AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK
);
2970 MOZ_ASSERT(OnTaskQueue());
2971 LOGV("%s failure:%s", TrackTypeToStr(aTrack
), aError
.ErrorName().get());
2972 if (aTrack
== TrackType::kVideoTrack
) {
2973 mVideo
.mSeekRequest
.Complete();
2975 mAudio
.mSeekRequest
.Complete();
2978 if (aError
== NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
) {
2979 if (HasVideo() && aTrack
== TrackType::kAudioTrack
&&
2980 mFallbackSeekTime
.isSome() &&
2981 mPendingSeekTime
.ref() != mFallbackSeekTime
.ref()) {
2982 // We have failed to seek audio where video seeked to earlier.
2983 // Attempt to seek instead to the closest point that we know we have in
2984 // order to limit A/V sync discrepency.
2986 // Ensure we have the most up to date buffered ranges.
2987 UpdateReceivedNewData(TrackType::kAudioTrack
);
2988 Maybe
<TimeUnit
> nextSeekTime
;
2989 // Find closest buffered time found after video seeked time.
2990 for (const auto& timeRange
: mAudio
.mTimeRanges
) {
2991 if (timeRange
.mStart
>= mPendingSeekTime
.ref()) {
2992 nextSeekTime
.emplace(timeRange
.mStart
);
2996 if (nextSeekTime
.isNothing() ||
2997 nextSeekTime
.ref() > mFallbackSeekTime
.ref()) {
2998 nextSeekTime
= Some(mFallbackSeekTime
.ref());
2999 LOG("Unable to seek audio to video seek time. A/V sync may be broken");
3001 mFallbackSeekTime
.reset();
3003 mPendingSeekTime
= nextSeekTime
;
3007 NotifyWaitingForData(aTrack
);
3009 MOZ_ASSERT(!mVideo
.mSeekRequest
.Exists() && !mAudio
.mSeekRequest
.Exists());
3010 mPendingSeekTime
.reset();
3012 auto type
= aTrack
== TrackType::kAudioTrack
? MediaData::Type::AUDIO_DATA
3013 : MediaData::Type::VIDEO_DATA
;
3014 mSeekPromise
.RejectIfExists(SeekRejectValue(type
, aError
), __func__
);
3017 void MediaFormatReader::DoVideoSeek() {
3018 AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK
);
3019 MOZ_ASSERT(mPendingSeekTime
.isSome());
3020 LOGV("Seeking video to %" PRId64
, mPendingSeekTime
.ref().ToMicroseconds());
3021 MOZ_DIAGNOSTIC_ASSERT(!IsAudioOnlySeeking());
3022 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.mSeekRequest
.Exists());
3023 auto seekTime
= mPendingSeekTime
.ref();
3024 mVideo
.mTrackDemuxer
->Seek(seekTime
)
3025 ->Then(OwnerThread(), __func__
, this,
3026 &MediaFormatReader::OnVideoSeekCompleted
,
3027 &MediaFormatReader::OnVideoSeekFailed
)
3028 ->Track(mVideo
.mSeekRequest
);
3031 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime
) {
3032 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
3034 MOZ_ASSERT(OnTaskQueue());
3035 LOGV("Video seeked to %" PRId64
, aTime
.ToMicroseconds());
3036 mVideo
.mSeekRequest
.Complete();
3038 mVideo
.mFirstFrameTime
= Some(aTime
);
3039 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
3041 SetVideoDecodeThreshold();
3043 if (HasAudio() && !mOriginalSeekTarget
.IsVideoOnly()) {
3044 MOZ_ASSERT(mPendingSeekTime
.isSome());
3045 if (mOriginalSeekTarget
.IsFast()) {
3046 // We are performing a fast seek. We need to seek audio to where the
3047 // video seeked to, to ensure proper A/V sync once playback resume.
3048 mPendingSeekTime
= Some(aTime
);
3052 mPendingSeekTime
.reset();
3053 mSeekPromise
.ResolveIfExists(aTime
, __func__
);
3057 void MediaFormatReader::OnVideoSeekFailed(const MediaResult
& aError
) {
3058 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK
);
3059 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
3060 OnSeekFailed(TrackType::kVideoTrack
, aError
);
3063 void MediaFormatReader::SetVideoDecodeThreshold() {
3064 MOZ_ASSERT(OnTaskQueue());
3066 if (!HasVideo() || !mVideo
.mDecoder
) {
3070 if (!mVideo
.mTimeThreshold
&& !IsSeeking()) {
3075 if (mVideo
.mTimeThreshold
) {
3076 // For internalSeek.
3077 threshold
= mVideo
.mTimeThreshold
.ref().Time();
3078 } else if (IsSeeking()) {
3079 // If IsSeeking() is true, then video seek must have completed already.
3081 if (NS_FAILED(mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&keyframe
))) {
3085 // If the key frame is invalid/infinite, it means the target position is
3086 // closing to end of stream. We don't want to skip any frame at this point.
3087 threshold
= keyframe
.IsValid() && !keyframe
.IsInfinite()
3088 ? mOriginalSeekTarget
.GetTime()
3089 : TimeUnit::Invalid();
3094 if (threshold
.IsValid()) {
3095 LOG("Set seek threshold to %" PRId64
, threshold
.ToMicroseconds());
3097 LOG("Resetting seek threshold");
3099 mVideo
.mDecoder
->SetSeekThreshold(threshold
);
3102 void MediaFormatReader::DoAudioSeek() {
3103 AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK
);
3104 MOZ_ASSERT(mPendingSeekTime
.isSome());
3105 LOGV("Seeking audio to %" PRId64
, mPendingSeekTime
.ref().ToMicroseconds());
3106 MOZ_DIAGNOSTIC_ASSERT(!IsVideoOnlySeeking());
3107 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.mSeekRequest
.Exists());
3108 auto seekTime
= mPendingSeekTime
.ref();
3109 mAudio
.mTrackDemuxer
->Seek(seekTime
)
3110 ->Then(OwnerThread(), __func__
, this,
3111 &MediaFormatReader::OnAudioSeekCompleted
,
3112 &MediaFormatReader::OnAudioSeekFailed
)
3113 ->Track(mAudio
.mSeekRequest
);
3116 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime
) {
3117 MOZ_ASSERT(OnTaskQueue());
3118 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
3120 LOGV("Audio seeked to %" PRId64
, aTime
.ToMicroseconds());
3121 mAudio
.mSeekRequest
.Complete();
3122 mAudio
.mFirstFrameTime
= Some(aTime
);
3123 mPendingSeekTime
.reset();
3124 mSeekPromise
.ResolveIfExists(aTime
, __func__
);
3127 void MediaFormatReader::OnAudioSeekFailed(const MediaResult
& aError
) {
3128 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK
);
3129 OnSeekFailed(TrackType::kAudioTrack
, aError
);
3132 void MediaFormatReader::ReleaseResources() {
3137 ShutdownDecoder(TrackInfo::kAudioTrack
);
3138 ShutdownDecoder(TrackInfo::kVideoTrack
);
3141 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
3142 return mVideo
.mIsHardwareAccelerated
;
3145 void MediaFormatReader::NotifyTrackDemuxers() {
3146 MOZ_ASSERT(OnTaskQueue());
3155 mVideo
.mReceivedNewData
= true;
3156 ScheduleUpdate(TrackType::kVideoTrack
);
3159 mAudio
.mReceivedNewData
= true;
3160 ScheduleUpdate(TrackType::kAudioTrack
);
3164 void MediaFormatReader::NotifyDataArrived() {
3165 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK
);
3166 MOZ_ASSERT(OnTaskQueue());
3168 if (mShutdown
|| !mDemuxer
|| !mDemuxerInitDone
) {
3172 if (mNotifyDataArrivedPromise
.Exists()) {
3173 // Already one in progress. Set the dirty flag so we can process it later.
3174 mPendingNotifyDataArrived
= true;
3178 RefPtr
<MediaFormatReader
> self
= this;
3179 mDemuxer
->NotifyDataArrived()
3181 OwnerThread(), __func__
,
3183 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
3185 self
->mNotifyDataArrivedPromise
.Complete();
3186 self
->UpdateBuffered();
3187 self
->NotifyTrackDemuxers();
3188 if (self
->mPendingNotifyDataArrived
) {
3189 self
->mPendingNotifyDataArrived
= false;
3190 self
->NotifyDataArrived();
3193 [self
]() { self
->mNotifyDataArrivedPromise
.Complete(); })
3194 ->Track(mNotifyDataArrivedPromise
);
3197 void MediaFormatReader::UpdateMediaEngineId(uint64_t aMediaEngineId
) {
3198 LOG("Update external media engine Id %" PRIu64
, aMediaEngineId
);
3199 mMediaEngineId
= Some(aMediaEngineId
);
3202 void MediaFormatReader::UpdateBuffered() {
3203 AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK
);
3204 MOZ_ASSERT(OnTaskQueue());
3210 if (!mInitDone
|| !mHasStartTime
) {
3211 mBuffered
= TimeIntervals();
3216 mVideo
.mTimeRanges
= mVideo
.mTrackDemuxer
->GetBuffered();
3218 auto lastEnd
= mVideo
.mTimeRanges
.GetEnd(&hasLastEnd
);
3220 if (mVideo
.mLastTimeRangesEnd
&&
3221 mVideo
.mLastTimeRangesEnd
.ref() < lastEnd
) {
3222 // New data was added after our previous end, we can clear the EOS flag.
3223 mVideo
.mDemuxEOS
= false;
3224 ScheduleUpdate(TrackInfo::kVideoTrack
);
3226 mVideo
.mLastTimeRangesEnd
= Some(lastEnd
);
3230 mAudio
.mTimeRanges
= mAudio
.mTrackDemuxer
->GetBuffered();
3232 auto lastEnd
= mAudio
.mTimeRanges
.GetEnd(&hasLastEnd
);
3234 if (mAudio
.mLastTimeRangesEnd
&&
3235 mAudio
.mLastTimeRangesEnd
.ref() < lastEnd
) {
3236 // New data was added after our previous end, we can clear the EOS flag.
3237 mAudio
.mDemuxEOS
= false;
3238 ScheduleUpdate(TrackInfo::kAudioTrack
);
3240 mAudio
.mLastTimeRangesEnd
= Some(lastEnd
);
3244 media::TimeIntervals intervals
;
3245 if (HasAudio() && HasVideo()) {
3246 intervals
= media::Intersection(mVideo
.mTimeRanges
, mAudio
.mTimeRanges
);
3247 } else if (HasAudio()) {
3248 intervals
= mAudio
.mTimeRanges
;
3249 } else if (HasVideo()) {
3250 intervals
= mVideo
.mTimeRanges
;
3253 if (intervals
.IsEmpty() || intervals
.GetStart() == TimeUnit::Zero()) {
3254 // IntervalSet already starts at 0 or is empty, nothing to shift.
3255 mBuffered
= intervals
;
3257 mBuffered
= intervals
.Shift(TimeUnit::Zero() - mInfo
.mStartTime
);
3261 layers::ImageContainer
* MediaFormatReader::GetImageContainer() {
3262 return mVideoFrameContainer
? mVideoFrameContainer
->GetImageContainer()
3266 RefPtr
<GenericPromise
> MediaFormatReader::RequestDebugInfo(
3267 dom::MediaFormatReaderDebugInfo
& aInfo
) {
3268 if (!OnTaskQueue()) {
3269 // Run the request on the task queue if it's not already.
3270 return InvokeAsync(mTaskQueue
, __func__
,
3271 [this, self
= RefPtr
{this}, &aInfo
] {
3272 return RequestDebugInfo(aInfo
);
3275 GetDebugInfo(aInfo
);
3276 return GenericPromise::CreateAndResolve(true, __func__
);
3279 Maybe
<nsCString
> MediaFormatReader::GetAudioProcessPerCodec() {
3280 if (mAudio
.mDescription
== "uninitialized"_ns
) {
3284 MOZ_ASSERT(mAudio
.mProcessName
.Length() > 0,
3285 "Should have had a process name");
3286 MOZ_ASSERT(mAudio
.mCodecName
.Length() > 0, "Should have had a codec name");
3288 nsCString processName
= mAudio
.mProcessName
;
3289 nsCString
audioProcessPerCodecName(processName
+ ","_ns
+ mAudio
.mCodecName
);
3290 if (processName
!= "utility"_ns
) {
3291 if (!StaticPrefs::media_rdd_process_enabled()) {
3292 audioProcessPerCodecName
+= ",rdd-disabled"_ns
;
3294 if (!StaticPrefs::media_utility_process_enabled()) {
3295 audioProcessPerCodecName
+= ",utility-disabled"_ns
;
3297 if (StaticPrefs::media_allow_audio_non_utility()) {
3298 audioProcessPerCodecName
+= ",allow-non-utility"_ns
;
3301 return Some(audioProcessPerCodecName
);
3304 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo
& aInfo
) {
3305 MOZ_ASSERT(OnTaskQueue(),
3306 "Don't call this off the task queue, it's going to touch a lot of "
3309 nsAutoCString
audioDecoderName("unavailable");
3310 nsAutoCString videoDecoderName
= audioDecoderName
;
3311 nsAutoCString
audioType("none");
3312 nsAutoCString
videoType("none");
3314 AudioInfo audioInfo
;
3316 audioInfo
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
3317 audioDecoderName
= mAudio
.mDecoder
? mAudio
.mDecoder
->GetDescriptionName()
3318 : mAudio
.mDescription
;
3319 audioType
= audioInfo
.mMimeType
;
3320 aInfo
.mAudioState
.mNeedInput
= NeedInput(mAudio
);
3321 aInfo
.mAudioState
.mHasPromise
= mAudio
.HasPromise();
3322 aInfo
.mAudioState
.mWaitingPromise
= !mAudio
.mWaitingPromise
.IsEmpty();
3323 aInfo
.mAudioState
.mHasDemuxRequest
= mAudio
.mDemuxRequest
.Exists();
3324 aInfo
.mAudioState
.mDemuxQueueSize
=
3325 AssertedCast
<int>(mAudio
.mQueuedSamples
.Length());
3326 aInfo
.mAudioState
.mHasDecoder
= mAudio
.mDecodeRequest
.Exists();
3327 aInfo
.mAudioState
.mTimeTreshold
=
3328 mAudio
.mTimeThreshold
? mAudio
.mTimeThreshold
.ref().Time().ToSeconds()
3330 aInfo
.mAudioState
.mTimeTresholdHasSeeked
=
3331 mAudio
.mTimeThreshold
? mAudio
.mTimeThreshold
.ref().mHasSeeked
: false;
3332 aInfo
.mAudioState
.mNumSamplesInput
=
3333 AssertedCast
<int64_t>(mAudio
.mNumSamplesInput
);
3334 aInfo
.mAudioState
.mNumSamplesOutput
=
3335 AssertedCast
<int64_t>(mAudio
.mNumSamplesOutput
);
3336 aInfo
.mAudioState
.mQueueSize
=
3337 AssertedCast
<int32_t>(size_t(mAudio
.mSizeOfQueue
));
3338 aInfo
.mAudioState
.mPending
= AssertedCast
<int>(mAudio
.mOutput
.Length());
3339 aInfo
.mAudioState
.mWaitingForData
= !!mAudio
.mWaitingForDataStartTime
;
3340 aInfo
.mAudioState
.mDemuxEOS
= mAudio
.mDemuxEOS
;
3341 aInfo
.mAudioState
.mDrainState
= int32_t(mAudio
.mDrainState
);
3342 aInfo
.mAudioState
.mWaitingForKey
= mAudio
.mWaitingForKey
;
3343 aInfo
.mAudioState
.mLastStreamSourceID
=
3344 AssertedCast
<int64_t>(mAudio
.mLastStreamSourceID
);
3347 CopyUTF8toUTF16(audioDecoderName
, aInfo
.mAudioDecoderName
);
3348 CopyUTF8toUTF16(audioType
, aInfo
.mAudioType
);
3349 aInfo
.mAudioChannels
= AssertedCast
<int32_t>(audioInfo
.mChannels
);
3350 aInfo
.mAudioRate
= audioInfo
.mRate
;
3351 aInfo
.mAudioFramesDecoded
=
3352 AssertedCast
<int64_t>(mAudio
.mNumSamplesOutputTotal
);
3354 VideoInfo videoInfo
;
3356 videoInfo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
3357 videoDecoderName
= mVideo
.mDecoder
? mVideo
.mDecoder
->GetDescriptionName()
3358 : mVideo
.mDescription
;
3359 videoType
= videoInfo
.mMimeType
;
3360 aInfo
.mVideoState
.mNeedInput
= NeedInput(mVideo
);
3361 aInfo
.mVideoState
.mHasPromise
= mVideo
.HasPromise();
3362 aInfo
.mVideoState
.mWaitingPromise
= !mVideo
.mWaitingPromise
.IsEmpty();
3363 aInfo
.mVideoState
.mHasDemuxRequest
= mVideo
.mDemuxRequest
.Exists();
3364 aInfo
.mVideoState
.mDemuxQueueSize
=
3365 AssertedCast
<int32_t>(mVideo
.mQueuedSamples
.Length());
3366 aInfo
.mVideoState
.mHasDecoder
= mVideo
.mDecodeRequest
.Exists();
3367 aInfo
.mVideoState
.mTimeTreshold
=
3368 mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
.ref().Time().ToSeconds()
3370 aInfo
.mVideoState
.mTimeTresholdHasSeeked
=
3371 mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
.ref().mHasSeeked
: false;
3372 aInfo
.mVideoState
.mNumSamplesInput
=
3373 AssertedCast
<int64_t>(mVideo
.mNumSamplesInput
);
3374 aInfo
.mVideoState
.mNumSamplesOutput
=
3375 AssertedCast
<int64_t>(mVideo
.mNumSamplesOutput
);
3376 aInfo
.mVideoState
.mQueueSize
=
3377 AssertedCast
<int32_t>(size_t(mVideo
.mSizeOfQueue
));
3378 aInfo
.mVideoState
.mPending
= AssertedCast
<int32_t>(mVideo
.mOutput
.Length());
3379 aInfo
.mVideoState
.mWaitingForData
= !!mVideo
.mWaitingForDataStartTime
;
3380 aInfo
.mVideoState
.mDemuxEOS
= mVideo
.mDemuxEOS
;
3381 aInfo
.mVideoState
.mDrainState
= int32_t(mVideo
.mDrainState
);
3382 aInfo
.mVideoState
.mWaitingForKey
= mVideo
.mWaitingForKey
;
3383 aInfo
.mVideoState
.mLastStreamSourceID
=
3384 AssertedCast
<int64_t>(mVideo
.mLastStreamSourceID
);
3385 aInfo
.mTotalReadMetadataTimeMs
= mReadMetaDataTime
.ToMilliseconds();
3386 aInfo
.mTotalWaitingForVideoDataTimeMs
=
3387 mTotalWaitingForVideoDataTime
.ToMilliseconds();
3390 CopyUTF8toUTF16(videoDecoderName
, aInfo
.mVideoDecoderName
);
3391 CopyUTF8toUTF16(videoType
, aInfo
.mVideoType
);
3393 videoInfo
.mDisplay
.width
< 0 ? 0 : videoInfo
.mDisplay
.width
;
3394 aInfo
.mVideoHeight
=
3395 videoInfo
.mDisplay
.height
< 0 ? 0 : videoInfo
.mDisplay
.height
;
3396 aInfo
.mVideoRate
= mVideo
.mMeanRate
.Mean();
3397 aInfo
.mVideoHardwareAccelerated
= VideoIsHardwareAccelerated();
3398 aInfo
.mVideoNumSamplesOutputTotal
=
3399 AssertedCast
<int64_t>(mVideo
.mNumSamplesOutputTotal
);
3400 aInfo
.mVideoNumSamplesSkippedTotal
=
3401 AssertedCast
<int64_t>(mVideo
.mNumSamplesSkippedTotal
);
3403 // Looking at dropped frames
3404 FrameStatisticsData stats
= mFrameStats
->GetFrameStatisticsData();
3405 aInfo
.mFrameStats
.mDroppedDecodedFrames
=
3406 AssertedCast
<int64_t>(stats
.mDroppedDecodedFrames
);
3407 aInfo
.mFrameStats
.mDroppedSinkFrames
=
3408 AssertedCast
<int64_t>(stats
.mDroppedSinkFrames
);
3409 aInfo
.mFrameStats
.mDroppedCompositorFrames
=
3410 AssertedCast
<int64_t>(stats
.mDroppedCompositorFrames
);
3413 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode
) {
3414 MOZ_ASSERT(OnTaskQueue());
3415 return SetNullDecode(TrackType::kVideoTrack
, aIsNullDecode
);
3418 void MediaFormatReader::UpdateCompositor(
3419 already_AddRefed
<layers::KnowsCompositor
> aCompositor
) {
3420 MOZ_ASSERT(OnTaskQueue());
3421 mKnowsCompositor
= aCompositor
;
3424 void MediaFormatReader::SetNullDecode(TrackType aTrack
, bool aIsNullDecode
) {
3425 MOZ_ASSERT(OnTaskQueue());
3427 auto& decoder
= GetDecoderData(aTrack
);
3428 if (decoder
.mIsNullDecode
== aIsNullDecode
) {
3432 LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3433 TrackTypeToStr(aTrack
), decoder
.mIsNullDecode
, aIsNullDecode
);
3435 decoder
.mIsNullDecode
= aIsNullDecode
;
3436 ShutdownDecoder(aTrack
);
3439 void MediaFormatReader::OnFirstDemuxCompleted(
3440 TrackInfo::TrackType aType
,
3441 const RefPtr
<MediaTrackDemuxer::SamplesHolder
>& aSamples
) {
3442 AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3444 MOZ_ASSERT(OnTaskQueue());
3450 auto& decoder
= GetDecoderData(aType
);
3451 MOZ_ASSERT(decoder
.mFirstDemuxedSampleTime
.isNothing());
3452 decoder
.mFirstDemuxedSampleTime
.emplace(aSamples
->GetSamples()[0]->mTime
);
3453 MaybeResolveMetadataPromise();
3456 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType
,
3457 const MediaResult
& aError
) {
3458 MOZ_ASSERT(OnTaskQueue());
3464 auto& decoder
= GetDecoderData(aType
);
3465 MOZ_ASSERT(decoder
.mFirstDemuxedSampleTime
.isNothing());
3466 decoder
.mFirstDemuxedSampleTime
.emplace(TimeUnit::FromInfinity());
3467 MaybeResolveMetadataPromise();
3470 } // namespace mozilla
3472 #undef NS_DispatchToMainThread