1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "MediaFormatReader.h"
13 #include "AllocationPolicy.h"
15 # include "AOMDecoder.h"
17 #include "DecoderBenchmark.h"
18 #include "MediaData.h"
19 #include "MediaDataDecoderProxy.h"
20 #include "MediaInfo.h"
21 #include "MP4Decoder.h"
22 #include "PDMFactory.h"
23 #include "PerformanceRecorder.h"
24 #include "VideoFrameContainer.h"
25 #include "VideoUtils.h"
26 #include "VPXDecoder.h"
27 #include "mozilla/AbstractThread.h"
28 #include "mozilla/CDMProxy.h"
29 #include "mozilla/ClearOnShutdown.h"
30 #include "mozilla/NotNull.h"
31 #include "mozilla/Preferences.h"
32 #include "mozilla/ProfilerLabels.h"
33 #include "mozilla/ProfilerMarkers.h"
34 #include "mozilla/SharedThreadPool.h"
35 #include "mozilla/StaticPrefs_media.h"
36 #include "mozilla/TaskQueue.h"
37 #include "mozilla/Unused.h"
38 #include "nsContentUtils.h"
39 #include "nsPrintfCString.h"
40 #include "nsTHashSet.h"
42 using namespace mozilla::media
;
44 static mozilla::LazyLogModule
sFormatDecoderLog("MediaFormatReader");
45 mozilla::LazyLogModule
gMediaDemuxerLog("MediaDemuxer");
47 #define LOG(arg, ...) \
48 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
49 __func__, ##__VA_ARGS__)
50 #define LOGV(arg, ...) \
51 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
52 __func__, ##__VA_ARGS__)
54 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
58 typedef void* MediaDataDecoderID
;
61 * This class tracks shutdown promises to ensure all decoders are shut down
62 * completely before MFR continues the rest of the shutdown procedure.
64 class MediaFormatReader::ShutdownPromisePool
{
67 : mOnShutdownComplete(new ShutdownPromise::Private(__func__
)) {}
69 // Return a promise which will be resolved when all the tracking promises
70 // are resolved. Note no more promises should be added for tracking once
71 // this function is called.
72 RefPtr
<ShutdownPromise
> Shutdown();
74 // Track a shutdown promise.
75 void Track(RefPtr
<ShutdownPromise
> aPromise
);
77 // Shut down a decoder and track its shutdown promise.
78 void ShutdownDecoder(already_AddRefed
<MediaDataDecoder
> aDecoder
) {
79 Track(RefPtr
<MediaDataDecoder
>(aDecoder
)->Shutdown());
83 bool mShutdown
= false;
84 const RefPtr
<ShutdownPromise::Private
> mOnShutdownComplete
;
85 nsTHashSet
<RefPtr
<ShutdownPromise
>> mPromises
;
88 RefPtr
<ShutdownPromise
> MediaFormatReader::ShutdownPromisePool::Shutdown() {
89 MOZ_DIAGNOSTIC_ASSERT(!mShutdown
);
91 if (mPromises
.Count() == 0) {
92 mOnShutdownComplete
->Resolve(true, __func__
);
94 return mOnShutdownComplete
;
97 void MediaFormatReader::ShutdownPromisePool::Track(
98 RefPtr
<ShutdownPromise
> aPromise
) {
99 MOZ_DIAGNOSTIC_ASSERT(!mShutdown
);
100 MOZ_DIAGNOSTIC_ASSERT(!mPromises
.Contains(aPromise
));
101 mPromises
.Insert(aPromise
);
102 aPromise
->Then(AbstractThread::GetCurrent(), __func__
, [aPromise
, this]() {
103 MOZ_DIAGNOSTIC_ASSERT(mPromises
.Contains(aPromise
));
104 mPromises
.Remove(aPromise
);
105 if (mShutdown
&& mPromises
.Count() == 0) {
106 mOnShutdownComplete
->Resolve(true, __func__
);
111 void MediaFormatReader::DecoderData::ShutdownDecoder() {
112 MOZ_ASSERT(mOwner
->OnTaskQueue());
114 MutexAutoLock
lock(mMutex
);
117 // No decoder to shut down.
122 // Flush is is in action. Shutdown will be initiated after flush completes.
123 MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise
);
124 mOwner
->mShutdownPromisePool
->Track(mShutdownPromise
->Ensure(__func__
));
125 // The order of decoder creation and shutdown is handled by LocalAllocPolicy
126 // and ShutdownPromisePool. MFR can now reset these members to a fresh state
127 // and be ready to create new decoders again without explicitly waiting for
128 // flush/shutdown to complete.
129 mShutdownPromise
= nullptr;
132 // No flush is in action. We can shut down the decoder now.
133 mOwner
->mShutdownPromisePool
->Track(mDecoder
->Shutdown());
136 // mShutdownPromisePool will handle the order of decoder shutdown so
137 // we can forget mDecoder and be ready to create a new one.
139 mDescription
= "shutdown"_ns
;
140 mHasReportedVideoHardwareSupportTelemtry
= false;
141 mOwner
->ScheduleUpdate(mType
== MediaData::Type::AUDIO_DATA
142 ? TrackType::kAudioTrack
143 : TrackType::kVideoTrack
);
146 void MediaFormatReader::DecoderData::Flush() {
147 AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK
);
148 MOZ_ASSERT(mOwner
->OnTaskQueue());
150 if (mFlushing
|| mFlushed
) {
151 // Flush still pending or already flushed, nothing more to do.
154 mDecodeRequest
.DisconnectIfExists();
155 mDrainRequest
.DisconnectIfExists();
156 mDrainState
= DrainState::None
;
157 CancelWaitingForKey();
159 mNumSamplesInput
= 0;
160 mNumSamplesOutput
= 0;
163 TrackType type
= mType
== MediaData::Type::AUDIO_DATA
164 ? TrackType::kAudioTrack
165 : TrackType::kVideoTrack
;
167 MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise
);
168 mShutdownPromise
= new SharedShutdownPromiseHolder();
169 RefPtr
<SharedShutdownPromiseHolder
> p
= mShutdownPromise
;
170 RefPtr
<MediaDataDecoder
> d
= mDecoder
;
171 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
172 "flushing", DDNoValue
{});
173 mDecoder
->Flush()->Then(
174 mOwner
->OwnerThread(), __func__
,
175 [type
, this, p
, d
]() {
176 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
178 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
179 "flushed", DDNoValue
{});
181 // Shutdown happened before flush completes.
182 // Let's continue to shut down the decoder. Note
183 // we don't access |this| because this decoder
184 // is no longer managed by MFR::DecoderData.
185 d
->Shutdown()->ChainTo(p
->Steal(), __func__
);
189 mShutdownPromise
= nullptr;
190 mOwner
->ScheduleUpdate(type
);
192 [type
, this, p
, d
](const MediaResult
& aError
) {
193 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
195 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log
,
196 "flush_error", aError
);
198 d
->Shutdown()->ChainTo(p
->Steal(), __func__
);
202 mShutdownPromise
= nullptr;
203 mOwner
->NotifyError(type
, aError
);
209 class MediaFormatReader::DecoderFactory
{
210 using InitPromise
= MediaDataDecoder::InitPromise
;
211 using TokenPromise
= AllocPolicy::Promise
;
212 using Token
= AllocPolicy::Token
;
213 using CreateDecoderPromise
= PlatformDecoderModule::CreateDecoderPromise
;
216 explicit DecoderFactory(MediaFormatReader
* aOwner
)
217 : mAudio(aOwner
->mAudio
, TrackInfo::kAudioTrack
, aOwner
->OwnerThread()),
218 mVideo(aOwner
->mVideo
, TrackInfo::kVideoTrack
, aOwner
->OwnerThread()),
219 mOwner(WrapNotNull(aOwner
)) {
220 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
222 DecoderDoctorLogger::LinkParentAndChild(
223 aOwner
, "decoder factory", "MediaFormatReader::DecoderFactory", this);
227 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
231 void CreateDecoder(TrackType aTrack
);
233 // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
234 // pristine state so CreateDecoder() is ready to be called again immediately.
235 void ShutdownDecoder(TrackType aTrack
) {
236 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
237 aTrack
== TrackInfo::kVideoTrack
);
238 auto& data
= aTrack
== TrackInfo::kAudioTrack
? mAudio
: mVideo
;
239 data
.mPolicy
->Cancel();
240 data
.mTokenRequest
.DisconnectIfExists();
241 if (data
.mLiveToken
) {
242 // We haven't completed creation of the decoder, and it hasn't been
244 data
.mLiveToken
= nullptr;
245 // The decoder will be shutdown as soon as it's available and tracked by
246 // the ShutdownPromisePool.
247 mOwner
->mShutdownPromisePool
->Track(data
.mCreateDecoderPromise
->Then(
248 mOwner
->mTaskQueue
, __func__
,
249 [](CreateDecoderPromise::ResolveOrRejectValue
&& aResult
) {
250 if (aResult
.IsReject()) {
251 return ShutdownPromise::CreateAndResolve(true, __func__
);
253 return aResult
.ResolveValue()->Shutdown();
255 // Free the token to leave room for a new decoder.
256 data
.mToken
= nullptr;
258 data
.mInitRequest
.DisconnectIfExists();
260 mOwner
->mShutdownPromisePool
->ShutdownDecoder(data
.mDecoder
.forget());
262 data
.mStage
= Stage::None
;
263 MOZ_ASSERT(!data
.mToken
);
267 enum class Stage
: int8_t { None
, WaitForToken
, CreateDecoder
, WaitForInit
};
270 Data(DecoderData
& aOwnerData
, TrackType aTrack
, TaskQueue
* aThread
)
271 : mOwnerData(aOwnerData
),
273 mPolicy(new SingleAllocPolicy(aTrack
, aThread
)) {}
274 DecoderData
& mOwnerData
;
275 const TrackType mTrack
;
276 RefPtr
<SingleAllocPolicy
> mPolicy
;
277 Stage mStage
= Stage::None
;
278 RefPtr
<Token
> mToken
;
279 RefPtr
<MediaDataDecoder
> mDecoder
;
280 MozPromiseRequestHolder
<TokenPromise
> mTokenRequest
;
281 struct DecoderCancelled
: public SupportsWeakPtr
{
282 NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled
)
284 ~DecoderCancelled() = default;
286 // Set when decoder is about to be created. If cleared before the decoder
287 // creation promise is resolved; it indicates that Shutdown() was called and
288 // further processing such as initialization should stop.
289 RefPtr
<DecoderCancelled
> mLiveToken
;
290 RefPtr
<CreateDecoderPromise
> mCreateDecoderPromise
;
291 MozPromiseRequestHolder
<InitPromise
> mInitRequest
;
294 void RunStage(Data
& aData
);
295 void DoCreateDecoder(Data
& aData
);
296 void DoInitDecoder(Data
& aData
);
298 // guaranteed to be valid by the owner.
299 const NotNull
<MediaFormatReader
*> mOwner
;
302 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack
) {
303 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
304 aTrack
== TrackInfo::kVideoTrack
);
305 RunStage(aTrack
== TrackInfo::kAudioTrack
? mAudio
: mVideo
);
308 void MediaFormatReader::DecoderFactory::RunStage(Data
& aData
) {
309 switch (aData
.mStage
) {
311 MOZ_ASSERT(!aData
.mToken
);
312 aData
.mPolicy
->Alloc()
314 mOwner
->OwnerThread(), __func__
,
315 [this, &aData
](RefPtr
<Token
> aToken
) {
316 aData
.mTokenRequest
.Complete();
317 aData
.mToken
= std::move(aToken
);
318 aData
.mStage
= Stage::CreateDecoder
;
322 aData
.mTokenRequest
.Complete();
323 aData
.mStage
= Stage::None
;
325 ->Track(aData
.mTokenRequest
);
326 aData
.mStage
= Stage::WaitForToken
;
330 case Stage::WaitForToken
: {
331 MOZ_ASSERT(!aData
.mToken
);
332 MOZ_ASSERT(aData
.mTokenRequest
.Exists());
336 case Stage::CreateDecoder
: {
337 MOZ_ASSERT(aData
.mToken
);
338 MOZ_ASSERT(!aData
.mDecoder
);
339 MOZ_ASSERT(!aData
.mInitRequest
.Exists());
341 DoCreateDecoder(aData
);
342 aData
.mStage
= Stage::WaitForInit
;
346 case Stage::WaitForInit
: {
347 MOZ_ASSERT((aData
.mDecoder
&& aData
.mInitRequest
.Exists()) ||
354 void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data
& aData
) {
355 AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK
);
356 auto& ownerData
= aData
.mOwnerData
;
357 auto& decoder
= mOwner
->GetDecoderData(aData
.mTrack
);
359 RefPtr
<PDMFactory
> platform
= new PDMFactory();
360 if (decoder
.IsEncrypted()) {
361 MOZ_ASSERT(mOwner
->mCDMProxy
);
362 platform
->SetCDMProxy(mOwner
->mCDMProxy
);
365 RefPtr
<PlatformDecoderModule::CreateDecoderPromise
> p
;
366 MediaFormatReader
* owner
= mOwner
;
367 auto onWaitingForKeyEvent
=
368 [owner
= ThreadSafeWeakPtr
<MediaFormatReader
>(owner
)]() {
369 RefPtr
<MediaFormatReader
> mfr(owner
);
370 MOZ_DIAGNOSTIC_ASSERT(mfr
, "The MediaFormatReader didn't wait for us");
371 return mfr
? &mfr
->OnTrackWaitingForKeyProducer() : nullptr;
374 switch (aData
.mTrack
) {
375 case TrackInfo::kAudioTrack
: {
376 p
= platform
->CreateDecoder(
377 {*ownerData
.GetCurrentInfo()->GetAsAudioInfo(), mOwner
->mCrashHelper
,
378 CreateDecoderParams::UseNullDecoder(ownerData
.mIsNullDecode
),
379 TrackInfo::kAudioTrack
, std::move(onWaitingForKeyEvent
),
380 mOwner
->mMediaEngineId
, mOwner
->mTrackingId
});
384 case TrackType::kVideoTrack
: {
385 // Decoders use the layers backend to decide if they can use hardware
386 // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
387 using Option
= CreateDecoderParams::Option
;
388 using OptionSet
= CreateDecoderParams::OptionSet
;
390 p
= platform
->CreateDecoder(
391 {*ownerData
.GetCurrentInfo()->GetAsVideoInfo(),
392 mOwner
->mKnowsCompositor
, mOwner
->GetImageContainer(),
393 mOwner
->mCrashHelper
,
394 CreateDecoderParams::UseNullDecoder(ownerData
.mIsNullDecode
),
395 TrackType::kVideoTrack
, std::move(onWaitingForKeyEvent
),
396 CreateDecoderParams::VideoFrameRate(ownerData
.mMeanRate
.Mean()),
397 OptionSet(ownerData
.mHardwareDecodingDisabled
398 ? Option::HardwareDecoderNotAllowed
400 mOwner
->mMediaEngineId
, mOwner
->mTrackingId
});
405 p
= PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(
406 NS_ERROR_DOM_MEDIA_FATAL_ERR
, __func__
);
409 aData
.mLiveToken
= MakeRefPtr
<Data::DecoderCancelled
>();
411 aData
.mCreateDecoderPromise
= p
->Then(
412 mOwner
->OwnerThread(), __func__
,
413 [this, &aData
, &ownerData
, live
= WeakPtr
{aData
.mLiveToken
},
414 owner
= ThreadSafeWeakPtr
<MediaFormatReader
>(owner
)](
415 RefPtr
<MediaDataDecoder
>&& aDecoder
) {
417 return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder
),
420 aData
.mLiveToken
= nullptr;
421 aData
.mDecoder
= new MediaDataDecoderProxy(
422 aDecoder
.forget(), do_AddRef(ownerData
.mTaskQueue
.get()));
423 aData
.mDecoder
= new AllocationWrapper(aData
.mDecoder
.forget(),
424 aData
.mToken
.forget());
425 DecoderDoctorLogger::LinkParentAndChild(
426 aData
.mDecoder
.get(), "decoder",
427 "MediaFormatReader::DecoderFactory", this);
429 DoInitDecoder(aData
);
431 return CreateDecoderPromise::CreateAndResolve(aData
.mDecoder
, __func__
);
434 live
= WeakPtr
{aData
.mLiveToken
}](const MediaResult
& aError
) {
435 NS_WARNING("Error constructing decoders");
437 return CreateDecoderPromise::CreateAndReject(aError
, __func__
);
439 aData
.mLiveToken
= nullptr;
440 aData
.mToken
= nullptr;
441 aData
.mStage
= Stage::None
;
442 aData
.mOwnerData
.mDescription
= aError
.Description();
443 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log
,
444 "create_decoder_error", aError
);
445 mOwner
->NotifyError(aData
.mTrack
, aError
);
447 return CreateDecoderPromise::CreateAndReject(aError
, __func__
);
451 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data
& aData
) {
452 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK
);
453 auto& ownerData
= aData
.mOwnerData
;
455 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log
,
456 "initialize_decoder", DDNoValue
{});
457 aData
.mDecoder
->Init()
459 mOwner
->OwnerThread(), __func__
,
460 [this, &aData
, &ownerData
](TrackType aTrack
) {
461 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
463 aData
.mInitRequest
.Complete();
464 aData
.mStage
= Stage::None
;
465 MutexAutoLock
lock(ownerData
.mMutex
);
466 ownerData
.mDecoder
= std::move(aData
.mDecoder
);
467 ownerData
.mDescription
= ownerData
.mDecoder
->GetDescriptionName();
468 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
469 DDLogCategory::Log
, "decoder_initialized", DDNoValue
{});
470 DecoderDoctorLogger::LinkParentAndChild(
471 "MediaFormatReader::DecoderData", &ownerData
, "decoder",
472 ownerData
.mDecoder
.get());
473 mOwner
->SetVideoDecodeThreshold();
474 mOwner
->ScheduleUpdate(aTrack
);
475 if (aTrack
== TrackInfo::kVideoTrack
) {
476 DecoderBenchmark::CheckVersion(
477 ownerData
.GetCurrentInfo()->mMimeType
);
479 if (aTrack
== TrackInfo::kAudioTrack
) {
480 ownerData
.mProcessName
= ownerData
.mDecoder
->GetProcessName();
481 ownerData
.mCodecName
= ownerData
.mDecoder
->GetCodecName();
484 [this, &aData
, &ownerData
](const MediaResult
& aError
) {
485 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
487 aData
.mInitRequest
.Complete();
488 MOZ_RELEASE_ASSERT(!ownerData
.mDecoder
,
489 "Can't have a decoder already set");
490 aData
.mStage
= Stage::None
;
491 mOwner
->mShutdownPromisePool
->ShutdownDecoder(
492 aData
.mDecoder
.forget());
493 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
494 DDLogCategory::Log
, "initialize_decoder_error", aError
);
495 mOwner
->NotifyError(aData
.mTrack
, aError
);
497 ->Track(aData
.mInitRequest
);
500 // DemuxerProxy ensures that the original main demuxer is only ever accessed
501 // via its own dedicated task queue.
502 // This ensure that the reader's taskqueue will never blocked while a demuxer
503 // is itself blocked attempting to access the MediaCache or the MediaResource.
504 class MediaFormatReader::DemuxerProxy
{
505 using TrackType
= TrackInfo::TrackType
;
509 explicit DemuxerProxy(MediaDataDemuxer
* aDemuxer
)
510 : mTaskQueue(TaskQueue::Create(
511 GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
512 "DemuxerProxy::mTaskQueue")),
513 mData(new Data(aDemuxer
)) {
514 MOZ_COUNT_CTOR(DemuxerProxy
);
517 MOZ_COUNTED_DTOR(DemuxerProxy
)
519 RefPtr
<ShutdownPromise
> Shutdown() {
520 RefPtr
<Data
> data
= std::move(mData
);
521 return InvokeAsync(mTaskQueue
, __func__
, [data
]() {
522 // We need to clear our reference to the demuxer now. So that in the event
523 // the init promise wasn't resolved, such as what can happen with the
524 // mediasource demuxer that is waiting on more data, it will force the
525 // init promise to be rejected.
526 data
->mDemuxer
= nullptr;
527 data
->mAudioDemuxer
= nullptr;
528 data
->mVideoDemuxer
= nullptr;
529 return ShutdownPromise::CreateAndResolve(true, __func__
);
533 RefPtr
<MediaDataDemuxer::InitPromise
> Init();
535 Wrapper
* GetTrackDemuxer(TrackType aTrack
, uint32_t aTrackNumber
) {
536 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
539 case TrackInfo::kAudioTrack
:
540 return mData
->mAudioDemuxer
;
541 case TrackInfo::kVideoTrack
:
542 return mData
->mVideoDemuxer
;
548 uint32_t GetNumberTracks(TrackType aTrack
) const {
549 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
552 case TrackInfo::kAudioTrack
:
553 return mData
->mNumAudioTrack
;
554 case TrackInfo::kVideoTrack
:
555 return mData
->mNumVideoTrack
;
561 bool IsSeekable() const {
562 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
564 return mData
->mSeekable
;
567 bool IsSeekableOnlyInBufferedRanges() const {
568 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
570 return mData
->mSeekableOnlyInBufferedRange
;
573 UniquePtr
<EncryptionInfo
> GetCrypto() const {
574 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
576 if (!mData
->mCrypto
) {
579 auto crypto
= MakeUnique
<EncryptionInfo
>();
580 *crypto
= *mData
->mCrypto
;
584 RefPtr
<NotifyDataArrivedPromise
> NotifyDataArrived();
586 bool ShouldComputeStartTime() const {
587 MOZ_RELEASE_ASSERT(mData
&& mData
->mInitDone
);
589 return mData
->mShouldComputeStartTime
;
593 const RefPtr
<TaskQueue
> mTaskQueue
;
595 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data
)
597 explicit Data(MediaDataDemuxer
* aDemuxer
)
598 : mInitDone(false), mDemuxer(aDemuxer
) {}
600 Atomic
<bool> mInitDone
;
601 // Only ever accessed over mTaskQueue once.
602 RefPtr
<MediaDataDemuxer
> mDemuxer
;
603 // Only accessed once InitPromise has been resolved and immutable after.
604 // So we can safely access them without the use of the mutex.
605 uint32_t mNumAudioTrack
= 0;
606 RefPtr
<Wrapper
> mAudioDemuxer
;
607 uint32_t mNumVideoTrack
= 0;
608 RefPtr
<Wrapper
> mVideoDemuxer
;
609 bool mSeekable
= false;
610 bool mSeekableOnlyInBufferedRange
= false;
611 bool mShouldComputeStartTime
= true;
612 UniquePtr
<EncryptionInfo
> mCrypto
;
620 class MediaFormatReader::DemuxerProxy::Wrapper
: public MediaTrackDemuxer
{
622 Wrapper(MediaTrackDemuxer
* aTrackDemuxer
, TaskQueue
* aTaskQueue
)
623 : mMutex("TrackDemuxer Mutex"),
624 mTaskQueue(aTaskQueue
),
625 mGetSamplesMayBlock(aTrackDemuxer
->GetSamplesMayBlock()),
626 mInfo(aTrackDemuxer
->GetInfo()),
627 mTrackDemuxer(aTrackDemuxer
) {
628 DecoderDoctorLogger::LogConstructionAndBase(
629 "MediaFormatReader::DemuxerProxy::Wrapper", this,
630 static_cast<const MediaTrackDemuxer
*>(this));
631 DecoderDoctorLogger::LinkParentAndChild(
632 "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
636 UniquePtr
<TrackInfo
> GetInfo() const override
{
640 return mInfo
->Clone();
643 RefPtr
<SeekPromise
> Seek(const TimeUnit
& aTime
) override
{
644 RefPtr
<Wrapper
> self
= this;
646 mTaskQueue
, __func__
,
647 [self
, aTime
]() { return self
->mTrackDemuxer
->Seek(aTime
); })
649 mTaskQueue
, __func__
,
650 [self
](const TimeUnit
& aTime
) {
651 self
->UpdateRandomAccessPoint();
652 return SeekPromise::CreateAndResolve(aTime
, __func__
);
654 [self
](const MediaResult
& aError
) {
655 self
->UpdateRandomAccessPoint();
656 return SeekPromise::CreateAndReject(aError
, __func__
);
660 RefPtr
<SamplesPromise
> GetSamples(int32_t aNumSamples
) override
{
661 RefPtr
<Wrapper
> self
= this;
662 return InvokeAsync(mTaskQueue
, __func__
,
663 [self
, aNumSamples
]() {
664 return self
->mTrackDemuxer
->GetSamples(aNumSamples
);
667 mTaskQueue
, __func__
,
668 [self
](RefPtr
<SamplesHolder
> aSamples
) {
669 self
->UpdateRandomAccessPoint();
670 return SamplesPromise::CreateAndResolve(aSamples
.forget(),
673 [self
](const MediaResult
& aError
) {
674 self
->UpdateRandomAccessPoint();
675 return SamplesPromise::CreateAndReject(aError
, __func__
);
679 bool GetSamplesMayBlock() const override
{ return mGetSamplesMayBlock
; }
681 void Reset() override
{
682 RefPtr
<Wrapper
> self
= this;
683 nsresult rv
= mTaskQueue
->Dispatch(NS_NewRunnableFunction(
684 "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
685 [self
]() { self
->mTrackDemuxer
->Reset(); }));
686 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
690 nsresult
GetNextRandomAccessPoint(TimeUnit
* aTime
) override
{
691 MutexAutoLock
lock(mMutex
);
692 if (NS_SUCCEEDED(mNextRandomAccessPointResult
)) {
693 *aTime
= mNextRandomAccessPoint
;
695 return mNextRandomAccessPointResult
;
698 RefPtr
<SkipAccessPointPromise
> SkipToNextRandomAccessPoint(
699 const TimeUnit
& aTimeThreshold
) override
{
700 RefPtr
<Wrapper
> self
= this;
702 mTaskQueue
, __func__
,
703 [self
, aTimeThreshold
]() {
704 return self
->mTrackDemuxer
->SkipToNextRandomAccessPoint(
708 mTaskQueue
, __func__
,
709 [self
](uint32_t aVal
) {
710 self
->UpdateRandomAccessPoint();
711 return SkipAccessPointPromise::CreateAndResolve(aVal
, __func__
);
713 [self
](const SkipFailureHolder
& aError
) {
714 self
->UpdateRandomAccessPoint();
715 return SkipAccessPointPromise::CreateAndReject(aError
, __func__
);
719 TimeIntervals
GetBuffered() override
{
720 MutexAutoLock
lock(mMutex
);
724 void BreakCycles() override
{}
727 Mutex mMutex MOZ_UNANNOTATED
;
728 const RefPtr
<TaskQueue
> mTaskQueue
;
729 const bool mGetSamplesMayBlock
;
730 const UniquePtr
<TrackInfo
> mInfo
;
731 // mTrackDemuxer is only ever accessed on demuxer's task queue.
732 RefPtr
<MediaTrackDemuxer
> mTrackDemuxer
;
733 // All following members are protected by mMutex
734 nsresult mNextRandomAccessPointResult
= NS_OK
;
735 TimeUnit mNextRandomAccessPoint
;
736 TimeIntervals mBuffered
;
737 friend class DemuxerProxy
;
740 RefPtr
<MediaTrackDemuxer
> trackDemuxer
= std::move(mTrackDemuxer
);
741 nsresult rv
= mTaskQueue
->Dispatch(NS_NewRunnableFunction(
742 "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
743 [trackDemuxer
]() { trackDemuxer
->BreakCycles(); }));
744 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
746 DecoderDoctorLogger::LogDestruction(
747 "MediaFormatReader::DemuxerProxy::Wrapper", this);
750 void UpdateRandomAccessPoint() {
751 MOZ_ASSERT(mTaskQueue
->IsCurrentThreadIn());
752 if (!mTrackDemuxer
) {
756 MutexAutoLock
lock(mMutex
);
757 mNextRandomAccessPointResult
=
758 mTrackDemuxer
->GetNextRandomAccessPoint(&mNextRandomAccessPoint
);
761 void UpdateBuffered() {
762 MOZ_ASSERT(mTaskQueue
->IsCurrentThreadIn());
763 if (!mTrackDemuxer
) {
767 MutexAutoLock
lock(mMutex
);
768 mBuffered
= mTrackDemuxer
->GetBuffered();
772 RefPtr
<MediaDataDemuxer::InitPromise
> MediaFormatReader::DemuxerProxy::Init() {
773 AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK
);
774 using InitPromise
= MediaDataDemuxer::InitPromise
;
776 RefPtr
<Data
> data
= mData
;
777 RefPtr
<TaskQueue
> taskQueue
= mTaskQueue
;
778 return InvokeAsync(mTaskQueue
, __func__
,
779 [data
, taskQueue
]() {
780 if (!data
->mDemuxer
) {
781 return InitPromise::CreateAndReject(
782 NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
784 return data
->mDemuxer
->Init();
788 [data
, taskQueue
]() {
789 AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK
);
790 if (!data
->mDemuxer
) { // Was shutdown.
791 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
794 data
->mNumAudioTrack
=
795 data
->mDemuxer
->GetNumberTracks(TrackInfo::kAudioTrack
);
796 if (data
->mNumAudioTrack
) {
797 RefPtr
<MediaTrackDemuxer
> d
=
798 data
->mDemuxer
->GetTrackDemuxer(TrackInfo::kAudioTrack
, 0);
800 RefPtr
<Wrapper
> wrapper
=
801 new DemuxerProxy::Wrapper(d
, taskQueue
);
802 wrapper
->UpdateBuffered();
803 data
->mAudioDemuxer
= wrapper
;
804 DecoderDoctorLogger::LinkParentAndChild(
805 data
->mDemuxer
.get(), "decoder factory wrapper",
806 "MediaFormatReader::DecoderFactory::Wrapper",
810 data
->mNumVideoTrack
=
811 data
->mDemuxer
->GetNumberTracks(TrackInfo::kVideoTrack
);
812 if (data
->mNumVideoTrack
) {
813 RefPtr
<MediaTrackDemuxer
> d
=
814 data
->mDemuxer
->GetTrackDemuxer(TrackInfo::kVideoTrack
, 0);
816 RefPtr
<Wrapper
> wrapper
=
817 new DemuxerProxy::Wrapper(d
, taskQueue
);
818 wrapper
->UpdateBuffered();
819 data
->mVideoDemuxer
= wrapper
;
820 DecoderDoctorLogger::LinkParentAndChild(
821 data
->mDemuxer
.get(), "decoder factory wrapper",
822 "MediaFormatReader::DecoderFactory::Wrapper",
826 data
->mCrypto
= data
->mDemuxer
->GetCrypto();
827 data
->mSeekable
= data
->mDemuxer
->IsSeekable();
828 data
->mSeekableOnlyInBufferedRange
=
829 data
->mDemuxer
->IsSeekableOnlyInBufferedRanges();
830 data
->mShouldComputeStartTime
=
831 data
->mDemuxer
->ShouldComputeStartTime();
832 data
->mInitDone
= true;
833 return InitPromise::CreateAndResolve(NS_OK
, __func__
);
835 [](const MediaResult
& aError
) {
836 return InitPromise::CreateAndReject(aError
, __func__
);
840 RefPtr
<MediaFormatReader::NotifyDataArrivedPromise
>
841 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
842 RefPtr
<Data
> data
= mData
;
843 return InvokeAsync(mTaskQueue
, __func__
, [data
]() {
844 if (!data
->mDemuxer
) {
846 return NotifyDataArrivedPromise::CreateAndReject(
847 NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
849 data
->mDemuxer
->NotifyDataArrived();
850 if (data
->mAudioDemuxer
) {
851 data
->mAudioDemuxer
->UpdateBuffered();
853 if (data
->mVideoDemuxer
) {
854 data
->mVideoDemuxer
->UpdateBuffered();
856 return NotifyDataArrivedPromise::CreateAndResolve(true, __func__
);
860 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit
& aInit
,
861 MediaDataDemuxer
* aDemuxer
)
863 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR
),
864 "MediaFormatReader::mTaskQueue",
865 /* aSupportsTailDispatch = */ true)),
866 mAudio(this, MediaData::Type::AUDIO_DATA
,
867 StaticPrefs::media_audio_max_decode_error()),
868 mVideo(this, MediaData::Type::VIDEO_DATA
,
869 StaticPrefs::media_video_max_decode_error()),
870 mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),
871 mWatchManager(this, OwnerThread()),
872 mIsWatchingWorkingInfo(false),
873 mDemuxer(new DemuxerProxy(aDemuxer
)),
874 mDemuxerInitDone(false),
875 mPendingNotifyDataArrived(false),
876 mLastReportedNumDecodedFrames(0),
877 mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe
),
878 mKnowsCompositor(aInit
.mKnowsCompositor
),
880 mTrackDemuxersMayBlock(false),
881 mSeekScheduled(false),
882 mVideoFrameContainer(aInit
.mVideoFrameContainer
),
883 mCrashHelper(aInit
.mCrashHelper
),
884 mDecoderFactory(new DecoderFactory(this)),
885 mShutdownPromisePool(new ShutdownPromisePool()),
886 mBuffered(mTaskQueue
, TimeIntervals(),
887 "MediaFormatReader::mBuffered (Canonical)"),
888 mFrameStats(aInit
.mFrameStats
),
889 mMediaDecoderOwnerID(aInit
.mMediaDecoderOwnerID
),
890 mTrackingId(std::move(aInit
.mTrackingId
)) {
891 MOZ_ASSERT(aDemuxer
);
892 MOZ_COUNT_CTOR(MediaFormatReader
);
893 DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
895 DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
897 DDLINKCHILD("demuxer", aDemuxer
);
898 mOnTrackWaitingForKeyListener
= OnTrackWaitingForKey().Connect(
899 mTaskQueue
, this, &MediaFormatReader::NotifyWaitingForKey
);
902 MediaFormatReader::~MediaFormatReader() {
903 MOZ_COUNT_DTOR(MediaFormatReader
);
904 MOZ_ASSERT(mShutdown
);
907 RefPtr
<ShutdownPromise
> MediaFormatReader::Shutdown() {
908 MOZ_ASSERT(OnTaskQueue());
911 mDemuxerInitRequest
.DisconnectIfExists();
912 mNotifyDataArrivedPromise
.DisconnectIfExists();
913 mMetadataPromise
.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
914 mSeekPromise
.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
915 mSkipRequest
.DisconnectIfExists();
916 mSetCDMPromise
.RejectIfExists(
917 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
918 "MediaFormatReader is shutting down"),
921 if (mIsWatchingWorkingInfo
) {
922 mWatchManager
.Unwatch(mWorkingInfoChanged
,
923 &MediaFormatReader::NotifyTrackInfoUpdated
);
925 mWatchManager
.Shutdown();
927 if (mAudio
.HasPromise()) {
928 mAudio
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
930 if (mVideo
.HasPromise()) {
931 mVideo
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
935 mAudio
.ResetDemuxer();
936 mAudio
.mTrackDemuxer
->BreakCycles();
938 MutexAutoLock
lock(mAudio
.mMutex
);
939 mAudio
.mTrackDemuxer
= nullptr;
942 ShutdownDecoder(TrackInfo::kAudioTrack
);
946 mVideo
.ResetDemuxer();
947 mVideo
.mTrackDemuxer
->BreakCycles();
949 MutexAutoLock
lock(mVideo
.mMutex
);
950 mVideo
.mTrackDemuxer
= nullptr;
953 ShutdownDecoder(TrackInfo::kVideoTrack
);
956 mShutdownPromisePool
->Track(mDemuxer
->Shutdown());
959 mOnTrackWaitingForKeyListener
.Disconnect();
962 return mShutdownPromisePool
->Shutdown()->Then(
963 OwnerThread(), __func__
, this, &MediaFormatReader::TearDownDecoders
,
964 &MediaFormatReader::TearDownDecoders
);
967 void MediaFormatReader::ShutdownDecoder(TrackType aTrack
) {
968 LOGV("%s", TrackTypeToStr(aTrack
));
970 // Shut down the pending decoder if any.
971 mDecoderFactory
->ShutdownDecoder(aTrack
);
973 auto& decoder
= GetDecoderData(aTrack
);
974 // Flush the decoder if necessary.
977 // Shut down the decoder if any.
978 decoder
.ShutdownDecoder();
981 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
982 MOZ_ASSERT(OnTaskQueue());
983 if (!StaticPrefs::media_mediacapabilities_from_database()) {
986 auto& decoder
= GetDecoderData(TrackInfo::kVideoTrack
);
987 if (decoder
.GetCurrentInfo() && decoder
.GetCurrentInfo()->GetAsVideoInfo()) {
988 VideoInfo info
= *(decoder
.GetCurrentInfo()->GetAsVideoInfo());
989 info
.SetFrameRate(static_cast<int32_t>(ceil(decoder
.mMeanRate
.Mean())));
990 mOnStoreDecoderBenchmark
.Notify(std::move(info
));
994 void MediaFormatReader::NotifyTrackInfoUpdated() {
995 MOZ_ASSERT(OnTaskQueue());
996 if (mWorkingInfoChanged
) {
997 mWorkingInfoChanged
= false;
1000 AudioInfo audioInfo
;
1002 MutexAutoLock
lock(mVideo
.mMutex
);
1004 videoInfo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
1008 MutexAutoLock
lock(mAudio
.mMutex
);
1010 audioInfo
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
1014 mTrackInfoUpdatedEvent
.Notify(videoInfo
, audioInfo
);
1018 RefPtr
<ShutdownPromise
> MediaFormatReader::TearDownDecoders() {
1019 if (mAudio
.mTaskQueue
) {
1020 mAudio
.mTaskQueue
->BeginShutdown();
1021 mAudio
.mTaskQueue
->AwaitShutdownAndIdle();
1022 mAudio
.mTaskQueue
= nullptr;
1024 if (mVideo
.mTaskQueue
) {
1025 mVideo
.mTaskQueue
->BeginShutdown();
1026 mVideo
.mTaskQueue
->AwaitShutdownAndIdle();
1027 mVideo
.mTaskQueue
= nullptr;
1030 mDecoderFactory
= nullptr;
1031 mVideoFrameContainer
= nullptr;
1034 mBuffered
.DisconnectAll();
1035 return mTaskQueue
->BeginShutdown();
1038 nsresult
MediaFormatReader::Init() {
1039 MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1042 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
1043 "MFR::mAudio::mTaskQueue");
1046 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER
),
1047 "MFR::mVideo::mTaskQueue");
1052 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack
) {
1053 // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
1054 // and would create new one for specific track in the next Update.
1055 MOZ_ASSERT(OnTaskQueue());
1057 if (mSetCDMPromise
.IsEmpty()) {
1061 MOZ_ASSERT(mCDMProxy
);
1062 if (mSetCDMForTracks
.contains(aTrack
)) {
1063 mSetCDMForTracks
-= aTrack
;
1066 if (mSetCDMForTracks
.isEmpty()) {
1067 LOGV("%s : Done ", __func__
);
1068 mSetCDMPromise
.Resolve(/* aIgnored = */ true, __func__
);
1070 ScheduleUpdate(TrackInfo::kAudioTrack
);
1073 ScheduleUpdate(TrackInfo::kVideoTrack
);
1077 LOGV("%s : %s track is ready.", __func__
, TrackTypeToStr(aTrack
));
1081 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack
) {
1082 MOZ_ASSERT(OnTaskQueue());
1083 LOGV("%s : %s", __func__
, TrackTypeToStr(aTrack
));
1085 mSetCDMForTracks
+= aTrack
;
1087 // An old cdm proxy exists, so detaching old cdm proxy by shutting down
1088 // MediaDataDecoder.
1089 ShutdownDecoder(aTrack
);
1091 ScheduleUpdate(aTrack
);
1094 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack
) {
1095 MOZ_ASSERT(OnTaskQueue());
1096 return GetDecoderData(aTrack
).IsEncrypted() &&
1097 mSetCDMForTracks
.contains(aTrack
) && !mCDMProxy
;
1100 RefPtr
<SetCDMPromise
> MediaFormatReader::SetCDMProxy(CDMProxy
* aProxy
) {
1101 MOZ_ASSERT(OnTaskQueue());
1102 LOGV("SetCDMProxy (%p)", aProxy
);
1105 return SetCDMPromise::CreateAndReject(
1106 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
1107 "MediaFormatReader is shutting down"),
1111 mSetCDMPromise
.RejectIfExists(
1112 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR
,
1113 "Another new CDM proxy is being set."),
1116 // Shutdown all decoders as switching CDM proxy indicates that it's
1117 // inappropriate for the existing decoders to continue decoding via the old
1120 PrepareToSetCDMForTrack(TrackInfo::kAudioTrack
);
1123 PrepareToSetCDMForTrack(TrackInfo::kVideoTrack
);
1128 if (!mInitDone
|| mSetCDMForTracks
.isEmpty() || !mCDMProxy
) {
1129 // 1) MFR is not initialized yet or
1130 // 2) Demuxer is initialized without active audio and video or
1131 // 3) A null cdm proxy is set
1132 // the promise can be resolved directly.
1133 mSetCDMForTracks
.clear();
1134 return SetCDMPromise::CreateAndResolve(/* aIgnored = */ true, __func__
);
1137 RefPtr
<SetCDMPromise
> p
= mSetCDMPromise
.Ensure(__func__
);
1141 bool MediaFormatReader::IsWaitingOnCDMResource() {
1142 MOZ_ASSERT(OnTaskQueue());
1143 return IsEncrypted() && !mCDMProxy
;
1146 RefPtr
<MediaFormatReader::MetadataPromise
>
1147 MediaFormatReader::AsyncReadMetadata() {
1148 AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK
);
1149 MOZ_ASSERT(OnTaskQueue());
1151 MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise
.IsEmpty());
1154 // We are returning from dormant.
1155 MetadataHolder metadata
;
1156 metadata
.mInfo
= MakeUnique
<MediaInfo
>(mInfo
);
1157 return MetadataPromise::CreateAndResolve(std::move(metadata
), __func__
);
1160 RefPtr
<MetadataPromise
> p
= mMetadataPromise
.Ensure(__func__
);
1163 ->Then(OwnerThread(), __func__
, this,
1164 &MediaFormatReader::OnDemuxerInitDone
,
1165 &MediaFormatReader::OnDemuxerInitFailed
)
1166 ->Track(mDemuxerInitRequest
);
1170 void MediaFormatReader::OnDemuxerInitDone(const MediaResult
& aResult
) {
1171 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK
);
1172 MOZ_ASSERT(OnTaskQueue());
1173 mDemuxerInitRequest
.Complete();
1175 if (NS_FAILED(aResult
) && StaticPrefs::media_playback_warnings_as_errors()) {
1176 mMetadataPromise
.Reject(aResult
, __func__
);
1180 mDemuxerInitDone
= true;
1182 UniquePtr
<MetadataTags
> tags(MakeUnique
<MetadataTags
>());
1184 RefPtr
<PDMFactory
> platform
;
1185 if (!IsWaitingOnCDMResource()) {
1186 platform
= new PDMFactory();
1189 // To decode, we need valid video and a place to put it.
1190 bool videoActive
= !!mDemuxer
->GetNumberTracks(TrackInfo::kVideoTrack
) &&
1191 GetImageContainer();
1194 // We currently only handle the first video track.
1195 MutexAutoLock
lock(mVideo
.mMutex
);
1196 mVideo
.mTrackDemuxer
= mDemuxer
->GetTrackDemuxer(TrackInfo::kVideoTrack
, 0);
1197 if (!mVideo
.mTrackDemuxer
) {
1198 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1202 UniquePtr
<TrackInfo
> videoInfo
= mVideo
.mTrackDemuxer
->GetInfo();
1203 videoActive
= videoInfo
&& videoInfo
->IsValid();
1205 if (platform
&& platform
->SupportsMimeType(videoInfo
->mMimeType
) ==
1206 media::DecodeSupport::Unsupported
) {
1207 // We have no decoder for this track. Error.
1208 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1211 mInfo
.mVideo
= *videoInfo
->GetAsVideoInfo();
1212 mVideo
.mWorkingInfo
= MakeUnique
<VideoInfo
>(mInfo
.mVideo
);
1213 for (const MetadataTag
& tag
: videoInfo
->mTags
) {
1214 tags
->InsertOrUpdate(tag
.mKey
, tag
.mValue
);
1216 mWorkingInfoChanged
= true;
1217 mVideo
.mOriginalInfo
= std::move(videoInfo
);
1218 mTrackDemuxersMayBlock
|= mVideo
.mTrackDemuxer
->GetSamplesMayBlock();
1220 mVideo
.mTrackDemuxer
->BreakCycles();
1221 mVideo
.mTrackDemuxer
= nullptr;
1225 bool audioActive
= !!mDemuxer
->GetNumberTracks(TrackInfo::kAudioTrack
);
1227 MutexAutoLock
lock(mAudio
.mMutex
);
1228 mAudio
.mTrackDemuxer
= mDemuxer
->GetTrackDemuxer(TrackInfo::kAudioTrack
, 0);
1229 if (!mAudio
.mTrackDemuxer
) {
1230 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1234 UniquePtr
<TrackInfo
> audioInfo
= mAudio
.mTrackDemuxer
->GetInfo();
1235 // We actively ignore audio tracks that we know we can't play.
1237 audioInfo
&& audioInfo
->IsValid() &&
1238 (!platform
|| platform
->SupportsMimeType(audioInfo
->mMimeType
) !=
1239 media::DecodeSupport::Unsupported
);
1242 mInfo
.mAudio
= *audioInfo
->GetAsAudioInfo();
1243 mAudio
.mWorkingInfo
= MakeUnique
<AudioInfo
>(mInfo
.mAudio
);
1244 for (const MetadataTag
& tag
: audioInfo
->mTags
) {
1245 tags
->InsertOrUpdate(tag
.mKey
, tag
.mValue
);
1247 mWorkingInfoChanged
= true;
1248 mAudio
.mOriginalInfo
= std::move(audioInfo
);
1249 mTrackDemuxersMayBlock
|= mAudio
.mTrackDemuxer
->GetSamplesMayBlock();
1251 mAudio
.mTrackDemuxer
->BreakCycles();
1252 mAudio
.mTrackDemuxer
= nullptr;
1256 UniquePtr
<EncryptionInfo
> crypto
= mDemuxer
->GetCrypto();
1257 if (crypto
&& crypto
->IsEncrypted()) {
1258 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1259 for (uint32_t i
= 0; i
< crypto
->mInitDatas
.Length(); i
++) {
1260 mOnEncrypted
.Notify(crypto
->mInitDatas
[i
].mInitData
,
1261 crypto
->mInitDatas
[i
].mType
);
1263 mInfo
.mCrypto
= *crypto
;
1266 auto videoDuration
= HasVideo() ? mInfo
.mVideo
.mDuration
: TimeUnit::Zero();
1267 auto audioDuration
= HasAudio() ? mInfo
.mAudio
.mDuration
: TimeUnit::Zero();
1269 auto duration
= std::max(videoDuration
, audioDuration
);
1270 if (duration
.IsPositive()) {
1271 mInfo
.mMetadataDuration
= Some(duration
);
1274 mInfo
.mMediaSeekable
= mDemuxer
->IsSeekable();
1275 mInfo
.mMediaSeekableOnlyInBufferedRanges
=
1276 mDemuxer
->IsSeekableOnlyInBufferedRanges();
1278 if (!videoActive
&& !audioActive
) {
1279 mMetadataPromise
.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR
, __func__
);
1283 mTags
= std::move(tags
);
1286 // Try to get the start time.
1287 // For MSE case, the start time of each track is assumed to be 0.
1288 // For others, we must demux the first sample to know the start time for each
1290 if (!mDemuxer
->ShouldComputeStartTime()) {
1291 mAudio
.mFirstDemuxedSampleTime
.emplace(TimeUnit::Zero());
1292 mVideo
.mFirstDemuxedSampleTime
.emplace(TimeUnit::Zero());
1295 RequestDemuxSamples(TrackInfo::kAudioTrack
);
1299 RequestDemuxSamples(TrackInfo::kVideoTrack
);
1303 if (aResult
!= NS_OK
) {
1304 mOnDecodeWarning
.Notify(aResult
);
1307 MaybeResolveMetadataPromise();
1310 void MediaFormatReader::MaybeResolveMetadataPromise() {
1311 MOZ_ASSERT(OnTaskQueue());
1313 if ((HasAudio() && mAudio
.mFirstDemuxedSampleTime
.isNothing()) ||
1314 (HasVideo() && mVideo
.mFirstDemuxedSampleTime
.isNothing())) {
1318 TimeUnit startTime
=
1319 std::min(mAudio
.mFirstDemuxedSampleTime
.refOr(TimeUnit::FromInfinity()),
1320 mVideo
.mFirstDemuxedSampleTime
.refOr(TimeUnit::FromInfinity()));
1322 if (!startTime
.IsInfinite()) {
1323 mInfo
.mStartTime
= startTime
; // mInfo.mStartTime is initialized to 0.
1326 MetadataHolder metadata
;
1327 metadata
.mInfo
= MakeUnique
<MediaInfo
>(mInfo
);
1328 metadata
.mTags
= mTags
->Count() ? std::move(mTags
) : nullptr;
1330 // We now have all the informations required to calculate the initial buffered
1332 mHasStartTime
= true;
1335 mWatchManager
.Watch(mWorkingInfoChanged
,
1336 &MediaFormatReader::NotifyTrackInfoUpdated
);
1337 mIsWatchingWorkingInfo
= true;
1339 mMetadataPromise
.Resolve(std::move(metadata
), __func__
);
1342 bool MediaFormatReader::IsEncrypted() const {
1343 return (HasAudio() && mAudio
.GetCurrentInfo()->mCrypto
.IsEncrypted()) ||
1344 (HasVideo() && mVideo
.GetCurrentInfo()->mCrypto
.IsEncrypted());
1347 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult
& aError
) {
1348 mDemuxerInitRequest
.Complete();
1349 mMetadataPromise
.Reject(aError
, __func__
);
1352 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo
* aInfo
) {
1353 // Called on the MDSM's TaskQueue.
1355 MutexAutoLock
lock(mVideo
.mMutex
);
1357 aInfo
->mVideo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
1361 MutexAutoLock
lock(mAudio
.mMutex
);
1363 aInfo
->mAudio
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
1364 Maybe
<nsCString
> audioProcessPerCodecName
= GetAudioProcessPerCodec();
1365 if (audioProcessPerCodecName
.isSome()) {
1366 Telemetry::ScalarAdd(
1367 Telemetry::ScalarID::MEDIA_AUDIO_PROCESS_PER_CODEC_NAME
,
1368 NS_ConvertUTF8toUTF16(*audioProcessPerCodecName
), 1);
1374 MediaFormatReader::DecoderData
& MediaFormatReader::GetDecoderData(
1376 MOZ_ASSERT(aTrack
== TrackInfo::kAudioTrack
||
1377 aTrack
== TrackInfo::kVideoTrack
);
1378 if (aTrack
== TrackInfo::kAudioTrack
) {
1384 Maybe
<TimeUnit
> MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold
,
1385 bool aRequestNextVideoKeyFrame
) {
1386 MOZ_ASSERT(OnTaskQueue());
1387 MOZ_ASSERT(HasVideo());
1389 if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1393 // Ensure we have no pending seek going as skip-to-keyframe could return out
1394 // of date information.
1395 if (mVideo
.HasInternalSeekPending()) {
1399 TimeUnit nextKeyframe
;
1400 nsresult rv
= mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
);
1401 if (NS_FAILED(rv
)) {
1402 // Only OggTrackDemuxer with video type gets into here.
1403 // We don't support skip-to-next-frame for this case.
1407 const bool isNextKeyframeValid
=
1408 nextKeyframe
.ToMicroseconds() >= 0 && !nextKeyframe
.IsInfinite();
1409 // If we request the next keyframe, only return times greater than
1410 // aTimeThreshold. Otherwise, data will be already behind the threshold and
1411 // will be eventually discarded somewhere in the media pipeline.
1412 if (aRequestNextVideoKeyFrame
&& isNextKeyframeValid
&&
1413 nextKeyframe
> aTimeThreshold
) {
1414 return Some(nextKeyframe
);
1417 const bool isNextVideoBehindTheThreshold
=
1418 (isNextKeyframeValid
&& nextKeyframe
<= aTimeThreshold
) ||
1419 GetInternalSeekTargetEndTime() < aTimeThreshold
;
1420 return isNextVideoBehindTheThreshold
? Some(aTimeThreshold
) : Nothing();
1423 RefPtr
<MediaFormatReader::VideoDataPromise
> MediaFormatReader::RequestVideoData(
1424 const TimeUnit
& aTimeThreshold
, bool aRequestNextVideoKeyFrame
) {
1425 MOZ_ASSERT(OnTaskQueue());
1426 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise(), "No duplicate sample requests");
1427 // Requesting video can be done independently from audio, even during audio
1428 // seeking. But it shouldn't happen if we're doing video seek.
1429 if (!IsAudioOnlySeeking()) {
1430 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty(),
1431 "No sample requests allowed while seeking");
1432 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.mSeekRequest
.Exists() ||
1433 mVideo
.mTimeThreshold
.isSome());
1434 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1436 LOGV("RequestVideoData(%" PRId64
"), requestNextKeyFrame=%d",
1437 aTimeThreshold
.ToMicroseconds(), aRequestNextVideoKeyFrame
);
1440 LOG("called with no video track");
1441 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR
,
1446 LOG("called mid-seek. Rejecting.");
1447 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1452 NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1453 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1457 if (Maybe
<TimeUnit
> target
=
1458 ShouldSkip(aTimeThreshold
, aRequestNextVideoKeyFrame
)) {
1459 PROFILER_MARKER_UNTYPED("RequestVideoData SkipVideoDemuxToNextKeyFrame",
1461 RefPtr
<VideoDataPromise
> p
= mVideo
.EnsurePromise(__func__
);
1462 SkipVideoDemuxToNextKeyFrame(*target
);
1466 RefPtr
<VideoDataPromise
> p
= mVideo
.EnsurePromise(__func__
);
1467 ScheduleUpdate(TrackInfo::kVideoTrack
);
1472 void MediaFormatReader::OnDemuxFailed(TrackType aTrack
,
1473 const MediaResult
& aError
) {
1474 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK
);
1475 MOZ_ASSERT(OnTaskQueue());
1476 LOG("Failed to demux %s, failure:%s",
1477 aTrack
== TrackType::kVideoTrack
? "video" : "audio",
1478 aError
.ErrorName().get());
1479 auto& decoder
= GetDecoderData(aTrack
);
1480 decoder
.mDemuxRequest
.Complete();
1481 switch (aError
.Code()) {
1482 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
1483 DDLOG(DDLogCategory::Log
,
1484 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1485 : "audio_demux_interruption",
1487 if (!decoder
.mWaitingForData
) {
1488 decoder
.RequestDrain();
1490 NotifyEndOfStream(aTrack
);
1492 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
1493 DDLOG(DDLogCategory::Log
,
1494 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1495 : "audio_demux_interruption",
1497 if (!decoder
.mWaitingForData
) {
1498 decoder
.RequestDrain();
1500 NotifyWaitingForData(aTrack
);
1502 case NS_ERROR_DOM_MEDIA_CANCELED
:
1503 DDLOG(DDLogCategory::Log
,
1504 aTrack
== TrackType::kVideoTrack
? "video_demux_interruption"
1505 : "audio_demux_interruption",
1507 if (decoder
.HasPromise()) {
1508 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
1512 DDLOG(DDLogCategory::Log
,
1513 aTrack
== TrackType::kVideoTrack
? "video_demux_error"
1514 : "audio_demux_error",
1516 NotifyError(aTrack
, aError
);
1521 void MediaFormatReader::DoDemuxVideo() {
1522 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK
);
1523 using SamplesPromise
= MediaTrackDemuxer::SamplesPromise
;
1525 DDLOG(DDLogCategory::Log
, "video_demuxing", DDNoValue
{});
1526 PerformanceRecorder
<PlaybackStage
> perfRecorder(
1527 MediaStage::RequestDemux
,
1528 mVideo
.GetCurrentInfo()->GetAsVideoInfo()->mImage
.height
);
1529 auto p
= mVideo
.mTrackDemuxer
->GetSamples(1);
1531 RefPtr
<MediaFormatReader
> self
= this;
1532 if (mVideo
.mFirstDemuxedSampleTime
.isNothing()) {
1534 OwnerThread(), __func__
,
1535 [self
](RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1536 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1538 DDLOGEX(self
.get(), DDLogCategory::Log
, "video_first_demuxed",
1540 self
->OnFirstDemuxCompleted(TrackInfo::kVideoTrack
, aSamples
);
1541 return SamplesPromise::CreateAndResolve(aSamples
.forget(), __func__
);
1543 [self
](const MediaResult
& aError
) {
1544 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1546 DDLOGEX(self
.get(), DDLogCategory::Log
, "video_first_demuxing_error",
1548 self
->OnFirstDemuxFailed(TrackInfo::kVideoTrack
, aError
);
1549 return SamplesPromise::CreateAndReject(aError
, __func__
);
1554 OwnerThread(), __func__
,
1555 [self
, perfRecorder(std::move(perfRecorder
))](
1556 RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) mutable {
1557 perfRecorder
.Record();
1558 self
->OnVideoDemuxCompleted(std::move(aSamples
));
1560 [self
](const MediaResult
& aError
) { self
->OnVideoDemuxFailed(aError
); })
1561 ->Track(mVideo
.mDemuxRequest
);
1564 void MediaFormatReader::OnVideoDemuxCompleted(
1565 RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1566 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1568 LOGV("%zu video samples demuxed (sid:%d)", aSamples
->GetSamples().Length(),
1569 aSamples
->GetSamples()[0]->mTrackInfo
1570 ? aSamples
->GetSamples()[0]->mTrackInfo
->GetID()
1572 DDLOG(DDLogCategory::Log
, "video_demuxed_samples",
1573 uint64_t(aSamples
->GetSamples().Length()));
1574 mVideo
.mDemuxRequest
.Complete();
1575 mVideo
.mQueuedSamples
.AppendElements(aSamples
->GetSamples());
1576 ScheduleUpdate(TrackInfo::kVideoTrack
);
1579 RefPtr
<MediaFormatReader::AudioDataPromise
>
1580 MediaFormatReader::RequestAudioData() {
1581 MOZ_ASSERT(OnTaskQueue());
1582 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise(), "No duplicate sample requests");
1583 // Requesting audio can be done independently from video, even during video
1584 // seeking. But it shouldn't happen if we're doing audio seek.
1585 if (!IsVideoOnlySeeking()) {
1586 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty(),
1587 "No sample requests allowed while seeking");
1588 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.mSeekRequest
.Exists() ||
1589 mAudio
.mTimeThreshold
.isSome());
1590 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1595 LOG("called with no audio track");
1596 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR
,
1601 LOG("called mid-seek. Rejecting.");
1602 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1607 NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1608 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED
,
1612 RefPtr
<AudioDataPromise
> p
= mAudio
.EnsurePromise(__func__
);
1613 ScheduleUpdate(TrackInfo::kAudioTrack
);
1618 void MediaFormatReader::DoDemuxAudio() {
1619 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK
);
1620 using SamplesPromise
= MediaTrackDemuxer::SamplesPromise
;
1622 DDLOG(DDLogCategory::Log
, "audio_demuxing", DDNoValue
{});
1623 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestDemux
);
1624 auto p
= mAudio
.mTrackDemuxer
->GetSamples(1);
1626 RefPtr
<MediaFormatReader
> self
= this;
1627 if (mAudio
.mFirstDemuxedSampleTime
.isNothing()) {
1629 OwnerThread(), __func__
,
1630 [self
](RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1631 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1633 DDLOGEX(self
.get(), DDLogCategory::Log
, "audio_first_demuxed",
1635 self
->OnFirstDemuxCompleted(TrackInfo::kAudioTrack
, aSamples
);
1636 return SamplesPromise::CreateAndResolve(aSamples
.forget(), __func__
);
1638 [self
](const MediaResult
& aError
) {
1639 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1641 DDLOGEX(self
.get(), DDLogCategory::Log
, "audio_first_demuxing_error",
1643 self
->OnFirstDemuxFailed(TrackInfo::kAudioTrack
, aError
);
1644 return SamplesPromise::CreateAndReject(aError
, __func__
);
1649 OwnerThread(), __func__
,
1650 [self
, perfRecorder(std::move(perfRecorder
))](
1651 RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) mutable {
1652 perfRecorder
.Record();
1653 self
->OnAudioDemuxCompleted(std::move(aSamples
));
1655 [self
](const MediaResult
& aError
) { self
->OnAudioDemuxFailed(aError
); })
1656 ->Track(mAudio
.mDemuxRequest
);
1659 void MediaFormatReader::OnAudioDemuxCompleted(
1660 RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
1661 LOGV("%zu audio samples demuxed (sid:%d)", aSamples
->GetSamples().Length(),
1662 aSamples
->GetSamples()[0]->mTrackInfo
1663 ? aSamples
->GetSamples()[0]->mTrackInfo
->GetID()
1665 DDLOG(DDLogCategory::Log
, "audio_demuxed_samples",
1666 uint64_t(aSamples
->GetSamples().Length()));
1667 mAudio
.mDemuxRequest
.Complete();
1668 mAudio
.mQueuedSamples
.AppendElements(aSamples
->GetSamples());
1669 ScheduleUpdate(TrackInfo::kAudioTrack
);
1672 void MediaFormatReader::NotifyNewOutput(
1673 TrackType aTrack
, MediaDataDecoder::DecodedData
&& aResults
) {
1674 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK
);
1675 MOZ_ASSERT(OnTaskQueue());
1676 auto& decoder
= GetDecoderData(aTrack
);
1677 if (aResults
.IsEmpty()) {
1678 DDLOG(DDLogCategory::Log
,
1679 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio" : "decoded_video",
1680 "no output samples");
1682 for (auto&& sample
: aResults
) {
1683 if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1684 switch (sample
->mType
) {
1685 case MediaData::Type::AUDIO_DATA
:
1686 DDLOGPR(DDLogCategory::Log
,
1687 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1688 : "decoded_got_audio!?",
1689 "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1690 ", \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1691 ", \"duration_us\":%" PRIi64
", \"frames\":%" PRIu32
1692 ", \"channels\":%" PRIu32
", \"rate\":%" PRIu32
1694 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1695 sample
->mTimecode
.ToMicroseconds(),
1696 sample
->mDuration
.ToMicroseconds(),
1697 sample
->As
<AudioData
>()->Frames(),
1698 sample
->As
<AudioData
>()->mChannels
,
1699 sample
->As
<AudioData
>()->mRate
,
1700 sample
->As
<AudioData
>()->Data().Length());
1702 case MediaData::Type::VIDEO_DATA
:
1703 DDLOGPR(DDLogCategory::Log
,
1704 aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1705 : "decoded_got_video!?",
1706 "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1707 ", \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1708 ", \"duration_us\":%" PRIi64
1709 ", \"kf\":%s, \"size\":[%" PRIi32
",%" PRIi32
"]}",
1710 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1711 sample
->mTimecode
.ToMicroseconds(),
1712 sample
->mDuration
.ToMicroseconds(),
1713 sample
->mKeyframe
? "true" : "false",
1714 sample
->As
<VideoData
>()->mDisplay
.width
,
1715 sample
->As
<VideoData
>()->mDisplay
.height
);
1717 case MediaData::Type::RAW_DATA
:
1718 DDLOGPR(DDLogCategory::Log
,
1719 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1720 : aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1722 "{\"type\":\"RawData\", \"offset\":%" PRIi64
1723 " \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1724 ", \"duration_us\":%" PRIi64
", \"kf\":%s}",
1725 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1726 sample
->mTimecode
.ToMicroseconds(),
1727 sample
->mDuration
.ToMicroseconds(),
1728 sample
->mKeyframe
? "true" : "false");
1730 case MediaData::Type::NULL_DATA
:
1731 DDLOGPR(DDLogCategory::Log
,
1732 aTrack
== TrackInfo::kAudioTrack
? "decoded_audio"
1733 : aTrack
== TrackInfo::kVideoTrack
? "decoded_video"
1735 "{\"type\":\"NullData\", \"offset\":%" PRIi64
1736 " \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1737 ", \"duration_us\":%" PRIi64
", \"kf\":%s}",
1738 sample
->mOffset
, sample
->mTime
.ToMicroseconds(),
1739 sample
->mTimecode
.ToMicroseconds(),
1740 sample
->mDuration
.ToMicroseconds(),
1741 sample
->mKeyframe
? "true" : "false");
1745 LOGV("Received new %s sample time:%" PRId64
" duration:%" PRId64
,
1746 TrackTypeToStr(aTrack
), sample
->mTime
.ToMicroseconds(),
1747 sample
->mDuration
.ToMicroseconds());
1748 decoder
.mOutput
.AppendElement(sample
);
1749 decoder
.mNumSamplesOutput
++;
1750 decoder
.mNumOfConsecutiveDecodingError
= 0;
1751 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
= 0;
1752 if (aTrack
== TrackInfo::kAudioTrack
) {
1753 decoder
.mNumOfConsecutiveUtilityCrashes
= 0;
1756 LOG("Done processing new %s samples", TrackTypeToStr(aTrack
));
1758 if (!aResults
.IsEmpty()) {
1759 // We have decoded our first frame, we can now starts to skip future errors.
1760 decoder
.mFirstFrameTime
.reset();
1762 ScheduleUpdate(aTrack
);
1765 void MediaFormatReader::NotifyError(TrackType aTrack
,
1766 const MediaResult
& aError
) {
1767 MOZ_ASSERT(OnTaskQueue());
1768 NS_WARNING(aError
.Description().get());
1769 LOGV("%s Decoding error", TrackTypeToStr(aTrack
));
1770 auto& decoder
= GetDecoderData(aTrack
);
1771 decoder
.mError
= decoder
.HasFatalError() ? decoder
.mError
: Some(aError
);
1773 ScheduleUpdate(aTrack
);
1776 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack
) {
1777 MOZ_ASSERT(OnTaskQueue());
1778 auto& decoder
= GetDecoderData(aTrack
);
1779 decoder
.mWaitingForData
= true;
1780 if (decoder
.mTimeThreshold
) {
1781 decoder
.mTimeThreshold
.ref().mWaiting
= true;
1783 ScheduleUpdate(aTrack
);
1786 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack
) {
1787 MOZ_ASSERT(OnTaskQueue());
1788 auto& decoder
= GetDecoderData(aTrack
);
1789 mOnWaitingForKey
.Notify();
1790 if (!decoder
.mDecodeRequest
.Exists()) {
1791 LOGV("WaitingForKey received while no pending decode. Ignoring");
1794 decoder
.mWaitingForKey
= true;
1795 ScheduleUpdate(aTrack
);
1798 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack
) {
1799 MOZ_ASSERT(OnTaskQueue());
1800 auto& decoder
= GetDecoderData(aTrack
);
1801 decoder
.mDemuxEOS
= true;
1802 ScheduleUpdate(aTrack
);
1805 bool MediaFormatReader::NeedInput(DecoderData
& aDecoder
) {
1806 // The decoder will not be fed a new raw sample until the current decoding
1807 // requests has completed.
1808 return (aDecoder
.HasPromise() || aDecoder
.mTimeThreshold
.isSome()) &&
1809 !aDecoder
.HasPendingDrain() && !aDecoder
.HasFatalError() &&
1810 !aDecoder
.mDemuxRequest
.Exists() && !aDecoder
.mOutput
.Length() &&
1811 !aDecoder
.HasInternalSeekPending() &&
1812 !aDecoder
.mDecodeRequest
.Exists();
1815 void MediaFormatReader::ScheduleUpdate(TrackType aTrack
) {
1816 MOZ_ASSERT(OnTaskQueue());
1820 auto& decoder
= GetDecoderData(aTrack
);
1821 MOZ_RELEASE_ASSERT(decoder
.GetCurrentInfo(),
1822 "Can only schedule update when track exists");
1824 if (decoder
.mUpdateScheduled
) {
1827 LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack
));
1828 decoder
.mUpdateScheduled
= true;
1829 RefPtr
<nsIRunnable
> task(NewRunnableMethod
<TrackType
>(
1830 "MediaFormatReader::Update", this, &MediaFormatReader::Update
, aTrack
));
1831 nsresult rv
= OwnerThread()->Dispatch(task
.forget());
1832 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
1836 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack
) {
1837 MOZ_ASSERT(OnTaskQueue());
1838 auto& decoder
= GetDecoderData(aTrack
);
1840 if (!decoder
.mReceivedNewData
) {
1844 // We do not want to clear mWaitingForData while there are pending
1845 // demuxing or seeking operations that could affect the value of this flag.
1846 // This is in order to ensure that we will retry once they complete as we may
1847 // now have new data that could potentially allow those operations to
1848 // successfully complete if tried again.
1849 if (decoder
.mSeekRequest
.Exists()) {
1850 // Nothing more to do until this operation complete.
1854 if (aTrack
== TrackType::kVideoTrack
&& mSkipRequest
.Exists()) {
1855 LOGV("Skipping in progress, nothing more to do");
1859 if (decoder
.mDemuxRequest
.Exists()) {
1860 // We may have pending operations to process, so we want to continue
1861 // after UpdateReceivedNewData returns.
1865 if (decoder
.HasPendingDrain()) {
1866 // We do not want to clear mWaitingForData or mDemuxEOS while
1867 // a drain is in progress in order to properly complete the operation.
1871 decoder
.mReceivedNewData
= false;
1872 if (decoder
.mTimeThreshold
) {
1873 decoder
.mTimeThreshold
.ref().mWaiting
= false;
1875 decoder
.mWaitingForData
= false;
1877 if (decoder
.HasFatalError()) {
1881 if (!mSeekPromise
.IsEmpty() &&
1882 (!IsVideoOnlySeeking() || aTrack
== TrackInfo::kVideoTrack
)) {
1883 MOZ_ASSERT(!decoder
.HasPromise());
1884 MOZ_DIAGNOSTIC_ASSERT(
1885 (IsVideoOnlySeeking() || !mAudio
.mTimeThreshold
) &&
1886 !mVideo
.mTimeThreshold
,
1887 "InternalSeek must have been aborted when Seek was first called");
1888 MOZ_DIAGNOSTIC_ASSERT(
1889 (IsVideoOnlySeeking() || !mAudio
.HasWaitingPromise()) &&
1890 !mVideo
.HasWaitingPromise(),
1891 "Waiting promises must have been rejected when Seek was first called");
1892 if (mVideo
.mSeekRequest
.Exists() ||
1893 (!IsVideoOnlySeeking() && mAudio
.mSeekRequest
.Exists())) {
1894 // Already waiting for a seek to complete. Nothing more to do.
1897 LOG("Attempting Seek");
1901 if (decoder
.HasInternalSeekPending() || decoder
.HasWaitingPromise()) {
1902 if (decoder
.HasInternalSeekPending()) {
1903 LOG("Attempting Internal Seek");
1904 InternalSeek(aTrack
, decoder
.mTimeThreshold
.ref());
1906 if (decoder
.HasWaitingPromise() && !decoder
.IsWaitingForKey() &&
1907 !decoder
.IsWaitingForData()) {
1908 MOZ_ASSERT(!decoder
.HasPromise());
1909 LOG("We have new data. Resolving WaitingPromise");
1910 decoder
.mWaitingPromise
.Resolve(decoder
.mType
, __func__
);
1917 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack
) {
1918 MOZ_ASSERT(OnTaskQueue());
1919 auto& decoder
= GetDecoderData(aTrack
);
1920 MOZ_ASSERT(!decoder
.mDemuxRequest
.Exists());
1922 if (!decoder
.mQueuedSamples
.IsEmpty()) {
1923 // No need to demux new samples.
1927 if (decoder
.mDemuxEOS
) {
1928 // Nothing left to demux.
1929 // We do not want to attempt to demux while in waiting for data mode
1930 // as it would retrigger an unnecessary drain.
1934 LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack
));
1935 if (aTrack
== TrackInfo::kVideoTrack
) {
1942 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack
,
1943 MediaRawData
* aSample
) {
1944 MOZ_ASSERT(OnTaskQueue());
1945 auto& decoder
= GetDecoderData(aTrack
);
1946 RefPtr
<MediaFormatReader
> self
= this;
1947 decoder
.mFlushed
= false;
1948 DDLOGPR(DDLogCategory::Log
,
1949 aTrack
== TrackInfo::kAudioTrack
? "decode_audio"
1950 : aTrack
== TrackInfo::kVideoTrack
? "decode_video"
1952 "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1953 ", \"bytes\":%zu, \"time_us\":%" PRIi64
", \"timecode_us\":%" PRIi64
1954 ", \"duration_us\":%" PRIi64
",%s%s}",
1955 aSample
->mOffset
, aSample
->Size(), aSample
->mTime
.ToMicroseconds(),
1956 aSample
->mTimecode
.ToMicroseconds(),
1957 aSample
->mDuration
.ToMicroseconds(), aSample
->mKeyframe
? " kf" : "",
1958 aSample
->mEOS
? " eos" : "");
1960 const int32_t height
=
1961 aTrack
== TrackInfo::kVideoTrack
1962 ? decoder
.GetCurrentInfo()->GetAsVideoInfo()->mImage
.height
1964 MediaInfoFlag flag
= MediaInfoFlag::None
;
1966 aSample
->mKeyframe
? MediaInfoFlag::KeyFrame
: MediaInfoFlag::NonKeyFrame
;
1967 if (aTrack
== TrackInfo::kVideoTrack
) {
1968 flag
|= VideoIsHardwareAccelerated() ? MediaInfoFlag::HardwareDecoding
1969 : MediaInfoFlag::SoftwareDecoding
;
1970 const nsCString
& mimeType
= decoder
.GetCurrentInfo()->mMimeType
;
1971 if (MP4Decoder::IsH264(mimeType
)) {
1972 flag
|= MediaInfoFlag::VIDEO_H264
;
1973 } else if (VPXDecoder::IsVPX(mimeType
, VPXDecoder::VP8
)) {
1974 flag
|= MediaInfoFlag::VIDEO_VP8
;
1975 } else if (VPXDecoder::IsVPX(mimeType
, VPXDecoder::VP9
)) {
1976 flag
|= MediaInfoFlag::VIDEO_VP9
;
1979 else if (AOMDecoder::IsAV1(mimeType
)) {
1980 flag
|= MediaInfoFlag::VIDEO_AV1
;
1984 PerformanceRecorder
<PlaybackStage
> perfRecorder(MediaStage::RequestDecode
,
1986 if (mMediaEngineId
&& aSample
->mCrypto
.IsEncrypted()) {
1987 aSample
->mShouldCopyCryptoToRemoteRawData
= true;
1989 decoder
.mDecoder
->Decode(aSample
)
1991 mTaskQueue
, __func__
,
1992 [self
, aTrack
, &decoder
, perfRecorder(std::move(perfRecorder
))](
1993 MediaDataDecoder::DecodedData
&& aResults
) mutable {
1994 perfRecorder
.Record();
1995 decoder
.mDecodeRequest
.Complete();
1996 self
->NotifyNewOutput(aTrack
, std::move(aResults
));
1998 [self
, aTrack
, &decoder
](const MediaResult
& aError
) {
1999 decoder
.mDecodeRequest
.Complete();
2000 self
->NotifyError(aTrack
, aError
);
2002 ->Track(decoder
.mDecodeRequest
);
2005 void MediaFormatReader::HandleDemuxedSamples(
2006 TrackType aTrack
, FrameStatistics::AutoNotifyDecoded
& aA
) {
2007 MOZ_ASSERT(OnTaskQueue());
2009 auto& decoder
= GetDecoderData(aTrack
);
2011 if (decoder
.mFlushing
) {
2012 LOGV("Decoder operation in progress, let it complete.");
2016 if (decoder
.mQueuedSamples
.IsEmpty()) {
2020 RefPtr
<MediaRawData
> sample
= decoder
.mQueuedSamples
[0];
2021 const RefPtr
<TrackInfoSharedPtr
> info
= sample
->mTrackInfo
;
2023 if (info
&& decoder
.mLastStreamSourceID
!= info
->GetID()) {
2024 nsTArray
<RefPtr
<MediaRawData
>> samples
;
2025 if (decoder
.mDecoder
) {
2027 StaticPrefs::media_decoder_recycle_enabled() &&
2028 decoder
.mDecoder
->SupportDecoderRecycling() &&
2029 (*info
)->mCrypto
.mCryptoScheme
==
2030 decoder
.GetCurrentInfo()->mCrypto
.mCryptoScheme
&&
2031 (*info
)->mMimeType
== decoder
.GetCurrentInfo()->mMimeType
;
2032 if (!recyclable
&& decoder
.mTimeThreshold
.isNothing() &&
2033 (decoder
.mNextStreamSourceID
.isNothing() ||
2034 decoder
.mNextStreamSourceID
.ref() != info
->GetID())) {
2035 LOG("%s stream id has changed from:%d to:%d, draining decoder.",
2036 TrackTypeToStr(aTrack
), decoder
.mLastStreamSourceID
, info
->GetID());
2037 decoder
.RequestDrain();
2038 decoder
.mNextStreamSourceID
= Some(info
->GetID());
2039 ScheduleUpdate(aTrack
);
2043 // If flushing is required, it will clear our array of queued samples.
2044 // So we may need to make a copy.
2045 samples
= decoder
.mQueuedSamples
.Clone();
2047 LOG("Decoder does not support recycling, recreate decoder.");
2048 ShutdownDecoder(aTrack
);
2049 // We're going to be using a new decoder following the change of content
2050 // We can attempt to use hardware decoding again.
2051 decoder
.mHardwareDecodingDisabled
= false;
2052 } else if (decoder
.HasWaitingPromise()) {
2057 nsPrintfCString
markerString(
2058 "%s stream id changed from:%" PRIu32
" to:%" PRIu32
,
2059 TrackTypeToStr(aTrack
), decoder
.mLastStreamSourceID
, info
->GetID());
2060 PROFILER_MARKER_TEXT("StreamID Change", MEDIA_PLAYBACK
, {}, markerString
);
2061 LOG("%s", markerString
.get());
2063 if (aTrack
== TrackInfo::kVideoTrack
) {
2064 // We are about to create a new decoder thus the benchmark,
2065 // up to this point, is stored.
2066 NotifyDecoderBenchmarkStore();
2068 decoder
.mNextStreamSourceID
.reset();
2069 decoder
.mLastStreamSourceID
= info
->GetID();
2070 decoder
.mInfo
= info
;
2072 MutexAutoLock
lock(decoder
.mMutex
);
2073 if (aTrack
== TrackInfo::kAudioTrack
) {
2074 decoder
.mWorkingInfo
= MakeUnique
<AudioInfo
>(*info
->GetAsAudioInfo());
2075 } else if (aTrack
== TrackInfo::kVideoTrack
) {
2076 decoder
.mWorkingInfo
= MakeUnique
<VideoInfo
>(*info
->GetAsVideoInfo());
2078 mWorkingInfoChanged
= true;
2081 decoder
.mMeanRate
.Reset();
2083 if (sample
->mKeyframe
) {
2084 if (samples
.Length()) {
2085 decoder
.mQueuedSamples
= std::move(samples
);
2088 auto time
= TimeInterval(sample
->mTime
, sample
->GetEndTime());
2089 InternalSeekTarget seekTarget
=
2090 decoder
.mTimeThreshold
.refOr(InternalSeekTarget(time
, false));
2091 LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64
,
2092 sample
->mTime
.ToMicroseconds());
2093 InternalSeek(aTrack
, seekTarget
);
2098 // Calculate the average frame rate. The first frame will be accounted
2100 decoder
.mMeanRate
.Update(sample
->mDuration
);
2102 if (!decoder
.mDecoder
) {
2103 mDecoderFactory
->CreateDecoder(aTrack
);
2107 LOGV("Input:%" PRId64
" (dts:%" PRId64
" kf:%d)",
2108 sample
->mTime
.ToMicroseconds(), sample
->mTimecode
.ToMicroseconds(),
2110 decoder
.mNumSamplesInput
++;
2111 decoder
.mSizeOfQueue
++;
2112 if (aTrack
== TrackInfo::kVideoTrack
) {
2113 aA
.mStats
.mParsedFrames
++;
2116 DecodeDemuxedSamples(aTrack
, sample
);
2118 decoder
.mQueuedSamples
.RemoveElementAt(0);
2121 media::TimeUnit
MediaFormatReader::GetInternalSeekTargetEndTime() const {
2122 MOZ_ASSERT(OnTaskQueue());
2123 return mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
->EndTime()
2124 : TimeUnit::FromInfinity();
2127 void MediaFormatReader::InternalSeek(TrackType aTrack
,
2128 const InternalSeekTarget
& aTarget
) {
2129 MOZ_ASSERT(OnTaskQueue());
2130 LOG("%s internal seek to %f", TrackTypeToStr(aTrack
),
2131 aTarget
.Time().ToSeconds());
2133 auto& decoder
= GetDecoderData(aTrack
);
2135 decoder
.ResetDemuxer();
2136 decoder
.mTimeThreshold
= Some(aTarget
);
2137 DDLOG(DDLogCategory::Log
, "seeking", DDNoValue
{});
2138 RefPtr
<MediaFormatReader
> self
= this;
2139 decoder
.mTrackDemuxer
->Seek(decoder
.mTimeThreshold
.ref().Time())
2141 OwnerThread(), __func__
,
2142 [self
, aTrack
](TimeUnit aTime
) {
2143 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeked", DDNoValue
{});
2144 auto& decoder
= self
->GetDecoderData(aTrack
);
2145 decoder
.mSeekRequest
.Complete();
2146 MOZ_ASSERT(decoder
.mTimeThreshold
,
2147 "Seek promise must be disconnected when "
2148 "timethreshold is reset");
2149 decoder
.mTimeThreshold
.ref().mHasSeeked
= true;
2150 self
->SetVideoDecodeThreshold();
2151 self
->ScheduleUpdate(aTrack
);
2153 [self
, aTrack
](const MediaResult
& aError
) {
2154 auto& decoder
= self
->GetDecoderData(aTrack
);
2155 decoder
.mSeekRequest
.Complete();
2156 switch (aError
.Code()) {
2157 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
2158 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2160 self
->NotifyWaitingForData(aTrack
);
2162 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
2163 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2165 decoder
.mTimeThreshold
.reset();
2166 self
->NotifyEndOfStream(aTrack
);
2168 case NS_ERROR_DOM_MEDIA_CANCELED
:
2169 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_interrupted",
2171 decoder
.mTimeThreshold
.reset();
2174 DDLOGEX(self
.get(), DDLogCategory::Log
, "seeking_error",
2176 decoder
.mTimeThreshold
.reset();
2177 self
->NotifyError(aTrack
, aError
);
2181 ->Track(decoder
.mSeekRequest
);
2184 void MediaFormatReader::DrainDecoder(TrackType aTrack
) {
2185 AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK
);
2186 MOZ_ASSERT(OnTaskQueue());
2188 auto& decoder
= GetDecoderData(aTrack
);
2189 if (decoder
.mDrainState
== DrainState::Draining
) {
2192 if (!decoder
.mDecoder
||
2193 (decoder
.mDrainState
!= DrainState::PartialDrainPending
&&
2194 decoder
.mNumSamplesInput
== decoder
.mNumSamplesOutput
)) {
2195 // No frames to drain.
2196 LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack
));
2197 decoder
.mDrainState
= DrainState::DrainAborted
;
2198 ScheduleUpdate(aTrack
);
2202 decoder
.mDrainState
= DrainState::Draining
;
2204 DDLOG(DDLogCategory::Log
, "draining", DDNoValue
{});
2205 RefPtr
<MediaFormatReader
> self
= this;
2206 decoder
.mDecoder
->Drain()
2208 mTaskQueue
, __func__
,
2209 [self
, aTrack
, &decoder
](MediaDataDecoder::DecodedData
&& aResults
) {
2210 decoder
.mDrainRequest
.Complete();
2211 DDLOGEX(self
.get(), DDLogCategory::Log
, "drained", DDNoValue
{});
2212 if (aResults
.IsEmpty()) {
2213 decoder
.mDrainState
= DrainState::DrainCompleted
;
2215 self
->NotifyNewOutput(aTrack
, std::move(aResults
));
2216 // Let's see if we have any more data available to drain.
2217 decoder
.mDrainState
= DrainState::PartialDrainPending
;
2219 self
->ScheduleUpdate(aTrack
);
2221 [self
, aTrack
, &decoder
](const MediaResult
& aError
) {
2222 decoder
.mDrainRequest
.Complete();
2223 DDLOGEX(self
.get(), DDLogCategory::Log
, "draining_error", aError
);
2224 self
->NotifyError(aTrack
, aError
);
2226 ->Track(decoder
.mDrainRequest
);
2227 LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack
));
2230 void MediaFormatReader::Update(TrackType aTrack
) {
2231 AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK
);
2232 MOZ_ASSERT(OnTaskQueue());
2238 LOGV("Processing update for %s", TrackTypeToStr(aTrack
));
2240 bool needOutput
= false;
2241 auto& decoder
= GetDecoderData(aTrack
);
2242 decoder
.mUpdateScheduled
= false;
2248 if (aTrack
== TrackType::kVideoTrack
&& mSkipRequest
.Exists()) {
2249 LOGV("Skipping in progress, nothing more to do");
2253 if (UpdateReceivedNewData(aTrack
)) {
2254 LOGV("Nothing more to do");
2258 if (decoder
.mSeekRequest
.Exists()) {
2259 LOGV("Seeking hasn't completed, nothing more to do");
2263 MOZ_DIAGNOSTIC_ASSERT(
2264 !decoder
.HasInternalSeekPending() ||
2265 (!decoder
.mOutput
.Length() && !decoder
.mQueuedSamples
.Length()),
2266 "No frames can be demuxed or decoded while an internal seek is pending");
2268 // Record number of frames decoded and parsed. Automatically update the
2269 // stats counters using the AutoNotifyDecoded stack-based class.
2270 FrameStatistics::AutoNotifyDecoded
a(mFrameStats
);
2272 // Drop any frames found prior our internal seek target.
2273 while (decoder
.mTimeThreshold
&& decoder
.mOutput
.Length()) {
2274 RefPtr
<MediaData
>& output
= decoder
.mOutput
[0];
2275 InternalSeekTarget target
= decoder
.mTimeThreshold
.ref();
2276 auto time
= output
->mTime
;
2277 if (time
>= target
.Time()) {
2278 // We have reached our internal seek target.
2279 decoder
.mTimeThreshold
.reset();
2280 // We might have dropped some keyframes.
2281 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
2283 if (time
< target
.Time() || (target
.mDropTarget
&& target
.Contains(time
))) {
2284 LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2285 TrackTypeToStr(aTrack
), output
->mTime
.ToSeconds(),
2286 target
.Time().ToSeconds(), output
->mKeyframe
);
2287 decoder
.mOutput
.RemoveElementAt(0);
2288 decoder
.mSizeOfQueue
-= 1;
2292 while (decoder
.mOutput
.Length() &&
2293 decoder
.mOutput
[0]->mType
== MediaData::Type::NULL_DATA
) {
2294 LOGV("Dropping null data. Time: %" PRId64
,
2295 decoder
.mOutput
[0]->mTime
.ToMicroseconds());
2296 decoder
.mOutput
.RemoveElementAt(0);
2297 decoder
.mSizeOfQueue
-= 1;
2300 if (decoder
.HasPromise()) {
2302 if (decoder
.mOutput
.Length()) {
2303 RefPtr
<MediaData
> output
= decoder
.mOutput
[0];
2304 decoder
.mOutput
.RemoveElementAt(0);
2305 decoder
.mSizeOfQueue
-= 1;
2306 decoder
.mLastDecodedSampleTime
=
2307 Some(TimeInterval(output
->mTime
, output
->GetEndTime()));
2308 decoder
.mNumSamplesOutputTotal
++;
2309 ReturnOutput(output
, aTrack
);
2310 // We have a decoded sample ready to be returned.
2311 if (aTrack
== TrackType::kVideoTrack
) {
2313 decoder
.mNumSamplesOutputTotal
- mLastReportedNumDecodedFrames
;
2314 a
.mStats
.mDecodedFrames
= static_cast<uint32_t>(delta
);
2315 mLastReportedNumDecodedFrames
= decoder
.mNumSamplesOutputTotal
;
2316 if (output
->mKeyframe
) {
2317 if (mPreviousDecodedKeyframeTime_us
<
2318 output
->mTime
.ToMicroseconds()) {
2319 // There is a previous keyframe -> Record inter-keyframe stats.
2320 uint64_t segment_us
= output
->mTime
.ToMicroseconds() -
2321 mPreviousDecodedKeyframeTime_us
;
2322 a
.mStats
.mInterKeyframeSum_us
+= segment_us
;
2323 a
.mStats
.mInterKeyframeCount
+= 1;
2324 if (a
.mStats
.mInterKeyFrameMax_us
< segment_us
) {
2325 a
.mStats
.mInterKeyFrameMax_us
= segment_us
;
2328 mPreviousDecodedKeyframeTime_us
= output
->mTime
.ToMicroseconds();
2330 bool wasHardwareAccelerated
= mVideo
.mIsHardwareAccelerated
;
2332 mVideo
.mIsHardwareAccelerated
=
2333 mVideo
.mDecoder
&& mVideo
.mDecoder
->IsHardwareAccelerated(error
);
2334 VideoData
* videoData
= output
->As
<VideoData
>();
2335 if (!mVideo
.mHasReportedVideoHardwareSupportTelemtry
||
2336 wasHardwareAccelerated
!= mVideo
.mIsHardwareAccelerated
) {
2337 mVideo
.mHasReportedVideoHardwareSupportTelemtry
= true;
2338 Telemetry::ScalarSet(
2339 Telemetry::ScalarID::MEDIA_VIDEO_HARDWARE_DECODING_SUPPORT
,
2340 NS_ConvertUTF8toUTF16(mVideo
.GetCurrentInfo()->mMimeType
),
2341 !!mVideo
.mIsHardwareAccelerated
);
2342 static constexpr gfx::IntSize HD_VIDEO_SIZE
{1280, 720};
2343 if (videoData
->mDisplay
.width
>= HD_VIDEO_SIZE
.Width() &&
2344 videoData
->mDisplay
.height
>= HD_VIDEO_SIZE
.Height()) {
2345 Telemetry::ScalarSet(
2346 Telemetry::ScalarID::MEDIA_VIDEO_HD_HARDWARE_DECODING_SUPPORT
,
2347 NS_ConvertUTF8toUTF16(mVideo
.GetCurrentInfo()->mMimeType
),
2348 !!mVideo
.mIsHardwareAccelerated
);
2352 // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2354 mVideo
.mIsHardwareAccelerated
=
2355 mVideo
.mIsHardwareAccelerated
||
2356 (videoData
->mImage
&&
2357 videoData
->mImage
->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE
);
2360 } else if (decoder
.HasFatalError()) {
2361 nsCString mimeType
= decoder
.GetCurrentInfo()->mMimeType
;
2362 if (!mimeType
.IsEmpty()) {
2363 Telemetry::ScalarAdd(
2364 Telemetry::ScalarID::MEDIA_DECODE_ERROR_PER_MIME_TYPE
,
2365 NS_ConvertUTF8toUTF16(mimeType
), 1 /* error count */);
2367 LOG("Rejecting %s promise for %s : DECODE_ERROR", TrackTypeToStr(aTrack
),
2369 decoder
.RejectPromise(decoder
.mError
.ref(), __func__
);
2371 } else if (decoder
.HasCompletedDrain()) {
2372 if (decoder
.mDemuxEOS
) {
2373 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack
));
2374 if (aTrack
== TrackInfo::kVideoTrack
) {
2375 // End of video, store the benchmark of the decoder.
2376 NotifyDecoderBenchmarkStore();
2378 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM
, __func__
);
2379 } else if (decoder
.mWaitingForData
) {
2380 if (decoder
.mDrainState
== DrainState::DrainCompleted
&&
2381 decoder
.mLastDecodedSampleTime
&& !decoder
.mNextStreamSourceID
) {
2382 // We have completed draining the decoder following WaitingForData.
2383 // Set up the internal seek machinery to be able to resume from the
2384 // last sample decoded.
2385 LOG("Seeking to last sample time: %" PRId64
,
2386 decoder
.mLastDecodedSampleTime
.ref().mStart
.ToMicroseconds());
2387 InternalSeek(aTrack
, InternalSeekTarget(
2388 decoder
.mLastDecodedSampleTime
.ref(), true));
2390 if (!decoder
.mReceivedNewData
) {
2391 LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack
));
2392 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2396 decoder
.mDrainState
= DrainState::None
;
2398 // Now that draining has completed, we check if we have received
2399 // new data again as the result may now be different from the earlier
2401 if (UpdateReceivedNewData(aTrack
) || decoder
.mSeekRequest
.Exists()) {
2402 LOGV("Nothing more to do");
2405 } else if (decoder
.mDemuxEOS
&& !decoder
.HasPendingDrain() &&
2406 decoder
.mQueuedSamples
.IsEmpty()) {
2407 // It is possible to transition from WAITING_FOR_DATA directly to EOS
2408 // state during the internal seek; in which case no draining would occur.
2409 // There is no more samples left to be decoded and we are already in
2410 // EOS state. We can immediately reject the data promise.
2411 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack
));
2412 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM
, __func__
);
2413 } else if (decoder
.mWaitingForKey
) {
2414 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2415 TrackTypeToStr(aTrack
));
2416 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2417 } else if (IsDecoderWaitingForCDM(aTrack
)) {
2418 // Rejecting the promise could lead to entering buffering state for MDSM,
2419 // once a qualified(with the same key system and sessions created by the
2420 // same InitData) new cdm proxy is set, decoding can be resumed.
2421 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2422 TrackTypeToStr(aTrack
));
2423 decoder
.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
, __func__
);
2427 if (decoder
.mDrainState
== DrainState::DrainRequested
||
2428 decoder
.mDrainState
== DrainState::PartialDrainPending
) {
2429 if (decoder
.mOutput
.IsEmpty()) {
2430 DrainDecoder(aTrack
);
2435 if (decoder
.mError
&& !decoder
.HasFatalError()) {
2436 MOZ_RELEASE_ASSERT(!decoder
.HasInternalSeekPending(),
2437 "No error can occur while an internal seek is pending");
2440 bool firstFrameDecodingFailedWithHardware
=
2441 decoder
.mFirstFrameTime
&&
2442 decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR
&&
2443 decoder
.mDecoder
&& decoder
.mDecoder
->IsHardwareAccelerated(error
) &&
2444 !decoder
.mHardwareDecodingDisabled
;
2445 bool needsNewDecoder
=
2446 decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER
||
2447 firstFrameDecodingFailedWithHardware
;
2448 // Limit number of process restarts after crash
2449 if ((decoder
.mError
.ref() ==
2450 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
&&
2451 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
++ <
2452 decoder
.mMaxConsecutiveRDDOrGPUCrashes
) ||
2453 (decoder
.mError
.ref() ==
2454 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR
&&
2455 decoder
.mNumOfConsecutiveUtilityCrashes
++ <
2456 decoder
.mMaxConsecutiveUtilityCrashes
)) {
2457 needsNewDecoder
= true;
2459 // For MF CDM crash, it needs to be handled differently. We need to shutdown
2460 // current decoder and report that error to the state machine in order to
2461 // let it to determine if playback can keep going or not.
2462 if (decoder
.mError
.ref() ==
2463 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR
) {
2464 LOG("Error: notify MF CDM crash and shutdown %s decoder",
2465 TrackTypeToStr(aTrack
));
2466 ShutdownDecoder(aTrack
);
2467 decoder
.RejectPromise(decoder
.mError
.ref(), __func__
);
2468 decoder
.mError
.reset();
2472 // We failed to decode on Linux with HW decoder,
2473 // give it another try without HW decoder.
2474 if (decoder
.mError
.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR
&&
2475 decoder
.mDecoder
->IsHardwareAccelerated(error
)) {
2476 LOG("Error: %s decode error, disable HW acceleration",
2477 TrackTypeToStr(aTrack
));
2478 needsNewDecoder
= true;
2479 decoder
.mHardwareDecodingDisabled
= true;
2481 // RDD process crashed on Linux, give it another try without HW decoder.
2482 if (decoder
.mError
.ref() ==
2483 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
) {
2484 LOG("Error: %s remote decoder crashed, disable HW acceleration",
2485 TrackTypeToStr(aTrack
));
2486 decoder
.mHardwareDecodingDisabled
= true;
2489 // We don't want to expose crash error so switch to
2490 // NS_ERROR_DOM_MEDIA_DECODE_ERR.
2491 if (decoder
.mError
.ref() ==
2492 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR
||
2493 decoder
.mError
.ref() ==
2494 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR
) {
2495 decoder
.mError
= Some(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR
,
2496 RESULT_DETAIL("Unable to decode")));
2498 if (!needsNewDecoder
&& ++decoder
.mNumOfConsecutiveDecodingError
>
2499 decoder
.mMaxConsecutiveDecodingError
) {
2500 DDLOG(DDLogCategory::Log
, "too_many_decode_errors", decoder
.mError
.ref());
2501 NotifyError(aTrack
, decoder
.mError
.ref());
2505 if (firstFrameDecodingFailedWithHardware
) {
2506 decoder
.mHardwareDecodingDisabled
= true;
2508 decoder
.mError
.reset();
2510 LOG("%s decoded error count %d RDD crashes count %d",
2511 TrackTypeToStr(aTrack
), decoder
.mNumOfConsecutiveDecodingError
,
2512 decoder
.mNumOfConsecutiveRDDOrGPUCrashes
);
2514 if (needsNewDecoder
) {
2515 LOG("Error: %s needs a new decoder", TrackTypeToStr(aTrack
));
2516 ShutdownDecoder(aTrack
);
2518 if (decoder
.mFirstFrameTime
) {
2519 TimeInterval seekInterval
= TimeInterval(decoder
.mFirstFrameTime
.ref(),
2520 decoder
.mFirstFrameTime
.ref());
2521 InternalSeek(aTrack
, InternalSeekTarget(seekInterval
, false));
2525 TimeUnit nextKeyframe
;
2526 if (aTrack
== TrackType::kVideoTrack
&&
2528 decoder
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
)) &&
2529 !nextKeyframe
.IsInfinite()) {
2530 SkipVideoDemuxToNextKeyFrame(
2531 decoder
.mLastDecodedSampleTime
.refOr(TimeInterval()).Length());
2532 } else if (aTrack
== TrackType::kAudioTrack
) {
2535 DDLOG(DDLogCategory::Log
, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR
);
2536 // We can't recover from this error.
2537 NotifyError(aTrack
, NS_ERROR_DOM_MEDIA_FATAL_ERR
);
2542 bool needInput
= NeedInput(decoder
);
2544 LOGV("Update(%s) ni=%d no=%d in:%" PRIu64
" out:%" PRIu64
2545 " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2546 "ds:%d sid:%u waitcdm:%d",
2547 TrackTypeToStr(aTrack
), needInput
, needOutput
, decoder
.mNumSamplesInput
,
2548 decoder
.mNumSamplesOutput
, uint32_t(size_t(decoder
.mSizeOfQueue
)),
2549 decoder
.mDecodeRequest
.Exists(), decoder
.mFlushing
,
2550 decoder
.mDescription
.get(), uint32_t(decoder
.mOutput
.Length()),
2551 decoder
.mWaitingForData
, decoder
.mDemuxEOS
, int32_t(decoder
.mDrainState
),
2552 decoder
.mLastStreamSourceID
, IsDecoderWaitingForCDM(aTrack
));
2554 if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack
)) {
2555 // If the content is encrypted, MFR won't start to create decoder until
2560 if ((decoder
.IsWaitingForData() &&
2561 (!decoder
.mTimeThreshold
|| decoder
.mTimeThreshold
.ref().mWaiting
)) ||
2562 (decoder
.IsWaitingForKey())) {
2563 // Nothing more we can do at present.
2564 LOGV("Still waiting for data or key. data(%d)/key(%d)",
2565 decoder
.mWaitingForData
, decoder
.mWaitingForKey
);
2569 if (decoder
.CancelWaitingForKey()) {
2570 LOGV("No longer waiting for key. Resolving waiting promise");
2575 LOGV("No need for additional input (pending:%u)",
2576 uint32_t(decoder
.mOutput
.Length()));
2580 // Demux samples if we don't have some.
2581 RequestDemuxSamples(aTrack
);
2583 HandleDemuxedSamples(aTrack
, a
);
2586 void MediaFormatReader::ReturnOutput(MediaData
* aData
, TrackType aTrack
) {
2587 AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK
);
2588 MOZ_ASSERT(GetDecoderData(aTrack
).HasPromise());
2589 MOZ_DIAGNOSTIC_ASSERT(aData
->mType
!= MediaData::Type::NULL_DATA
);
2590 LOG("Resolved data promise for %s [%" PRId64
", %" PRId64
"]",
2591 TrackTypeToStr(aTrack
), aData
->mTime
.ToMicroseconds(),
2592 aData
->GetEndTime().ToMicroseconds());
2594 if (aTrack
== TrackInfo::kAudioTrack
) {
2595 AudioData
* audioData
= aData
->As
<AudioData
>();
2597 if (audioData
->mChannels
!= mInfo
.mAudio
.mChannels
||
2598 audioData
->mRate
!= mInfo
.mAudio
.mRate
) {
2599 LOG("change of audio format (rate:%d->%d). "
2600 "This is an unsupported configuration",
2601 mInfo
.mAudio
.mRate
, audioData
->mRate
);
2602 mInfo
.mAudio
.mRate
= audioData
->mRate
;
2603 mInfo
.mAudio
.mChannels
= audioData
->mChannels
;
2604 MutexAutoLock
lock(mAudio
.mMutex
);
2605 mAudio
.mWorkingInfo
->GetAsAudioInfo()->mRate
= audioData
->mRate
;
2606 mAudio
.mWorkingInfo
->GetAsAudioInfo()->mChannels
= audioData
->mChannels
;
2607 mWorkingInfoChanged
= true;
2609 mAudio
.ResolvePromise(audioData
, __func__
);
2610 } else if (aTrack
== TrackInfo::kVideoTrack
) {
2611 VideoData
* videoData
= aData
->As
<VideoData
>();
2613 if (videoData
->mDisplay
!= mInfo
.mVideo
.mDisplay
) {
2614 LOG("change of video display size (%dx%d->%dx%d)",
2615 mInfo
.mVideo
.mDisplay
.width
, mInfo
.mVideo
.mDisplay
.height
,
2616 videoData
->mDisplay
.width
, videoData
->mDisplay
.height
);
2617 mInfo
.mVideo
.mDisplay
= videoData
->mDisplay
;
2618 MutexAutoLock
lock(mVideo
.mMutex
);
2619 mVideo
.mWorkingInfo
->GetAsVideoInfo()->mDisplay
= videoData
->mDisplay
;
2620 mWorkingInfoChanged
= true;
2623 mozilla::gfx::ColorDepth colorDepth
= videoData
->GetColorDepth();
2624 if (colorDepth
!= mInfo
.mVideo
.mColorDepth
) {
2625 LOG("change of video color depth (enum %u -> enum %u)",
2626 (unsigned)mInfo
.mVideo
.mColorDepth
, (unsigned)colorDepth
);
2627 mInfo
.mVideo
.mColorDepth
= colorDepth
;
2628 MutexAutoLock
lock(mVideo
.mMutex
);
2629 mVideo
.mWorkingInfo
->GetAsVideoInfo()->mColorDepth
= colorDepth
;
2630 mWorkingInfoChanged
= true;
2633 TimeUnit nextKeyframe
;
2634 if (!mVideo
.HasInternalSeekPending() &&
2636 mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&nextKeyframe
))) {
2637 videoData
->SetNextKeyFrameTime(nextKeyframe
);
2640 mVideo
.ResolvePromise(videoData
, __func__
);
2644 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2645 return SizeOfQueue(TrackInfo::kVideoTrack
);
2648 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2649 return SizeOfQueue(TrackInfo::kAudioTrack
);
2652 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack
) {
2653 auto& decoder
= GetDecoderData(aTrack
);
2654 return decoder
.mSizeOfQueue
;
2657 RefPtr
<MediaFormatReader::WaitForDataPromise
> MediaFormatReader::WaitForData(
2658 MediaData::Type aType
) {
2659 MOZ_ASSERT(OnTaskQueue());
2660 TrackType trackType
= aType
== MediaData::Type::VIDEO_DATA
2661 ? TrackType::kVideoTrack
2662 : TrackType::kAudioTrack
;
2663 auto& decoder
= GetDecoderData(trackType
);
2664 if (!decoder
.IsWaitingForData() && !decoder
.IsWaitingForKey()) {
2665 // We aren't waiting for anything.
2666 return WaitForDataPromise::CreateAndResolve(decoder
.mType
, __func__
);
2668 RefPtr
<WaitForDataPromise
> p
= decoder
.mWaitingPromise
.Ensure(__func__
);
2669 ScheduleUpdate(trackType
);
2673 nsresult
MediaFormatReader::ResetDecode(TrackSet aTracks
) {
2674 AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK
);
2675 MOZ_ASSERT(OnTaskQueue());
2678 mSeekPromise
.RejectIfExists(NS_OK
, __func__
);
2679 mSkipRequest
.DisconnectIfExists();
2681 // Do the same for any data wait promises.
2682 if (aTracks
.contains(TrackInfo::kAudioTrack
)) {
2683 mAudio
.mWaitingPromise
.RejectIfExists(
2684 WaitForDataRejectValue(MediaData::Type::AUDIO_DATA
,
2685 WaitForDataRejectValue::CANCELED
),
2689 if (aTracks
.contains(TrackInfo::kVideoTrack
)) {
2690 mVideo
.mWaitingPromise
.RejectIfExists(
2691 WaitForDataRejectValue(MediaData::Type::VIDEO_DATA
,
2692 WaitForDataRejectValue::CANCELED
),
2696 // Reset miscellaneous seeking state.
2697 mPendingSeekTime
.reset();
2699 if (HasVideo() && aTracks
.contains(TrackInfo::kVideoTrack
)) {
2700 mVideo
.ResetDemuxer();
2701 mVideo
.mFirstFrameTime
= Some(media::TimeUnit::Zero());
2702 Reset(TrackInfo::kVideoTrack
);
2703 if (mVideo
.HasPromise()) {
2704 mVideo
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
2708 if (HasAudio() && aTracks
.contains(TrackInfo::kAudioTrack
)) {
2709 mAudio
.ResetDemuxer();
2710 mVideo
.mFirstFrameTime
= Some(media::TimeUnit::Zero());
2711 Reset(TrackInfo::kAudioTrack
);
2712 if (mAudio
.HasPromise()) {
2713 mAudio
.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED
, __func__
);
2720 void MediaFormatReader::Reset(TrackType aTrack
) {
2721 MOZ_ASSERT(OnTaskQueue());
2722 LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack
));
2724 auto& decoder
= GetDecoderData(aTrack
);
2726 decoder
.ResetState();
2729 LOG("Reset(%s) END", TrackTypeToStr(aTrack
));
2732 void MediaFormatReader::DropDecodedSamples(TrackType aTrack
) {
2733 MOZ_ASSERT(OnTaskQueue());
2734 auto& decoder
= GetDecoderData(aTrack
);
2735 size_t lengthDecodedQueue
= decoder
.mOutput
.Length();
2736 if (lengthDecodedQueue
&& decoder
.mTimeThreshold
.isSome()) {
2737 auto time
= decoder
.mOutput
.LastElement()->mTime
;
2738 if (time
>= decoder
.mTimeThreshold
.ref().Time()) {
2739 // We would have reached our internal seek target.
2740 decoder
.mTimeThreshold
.reset();
2743 decoder
.mOutput
.Clear();
2744 decoder
.mSizeOfQueue
-= lengthDecodedQueue
;
2745 if (aTrack
== TrackInfo::kVideoTrack
&& mFrameStats
) {
2746 mFrameStats
->Accumulate({0, 0, 0, lengthDecodedQueue
, 0, 0});
2750 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold
) {
2751 AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2753 MOZ_ASSERT(OnTaskQueue());
2754 LOG("Skipping up to %" PRId64
, aTimeThreshold
.ToMicroseconds());
2756 // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2757 // As such we can drop all already decoded samples and discard all pending
2759 DropDecodedSamples(TrackInfo::kVideoTrack
);
2761 mVideo
.mTrackDemuxer
->SkipToNextRandomAccessPoint(aTimeThreshold
)
2762 ->Then(OwnerThread(), __func__
, this,
2763 &MediaFormatReader::OnVideoSkipCompleted
,
2764 &MediaFormatReader::OnVideoSkipFailed
)
2765 ->Track(mSkipRequest
);
2768 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped
) {
2769 PROFILER_MARKER_UNTYPED("SkippedVideoDecode", MEDIA_PLAYBACK
);
2770 MOZ_ASSERT(OnTaskQueue());
2772 // Some frames may have been output by the decoder since we initiated the
2773 // videoskip process and we know they would be late.
2774 DropDecodedSamples(TrackInfo::kVideoTrack
);
2775 // Report the pending frames as dropped.
2777 uint32_t droppedDecoderCount
= SizeOfVideoQueueInFrames();
2778 mFrameStats
->Accumulate({0, 0, 0, droppedDecoderCount
, 0, 0});
2781 // Cancel any pending demux request and pending demuxed samples.
2782 mVideo
.mDemuxRequest
.DisconnectIfExists();
2783 Reset(TrackType::kVideoTrack
);
2786 mFrameStats
->Accumulate({aSkipped
, 0, 0, aSkipped
, 0, 0});
2789 mVideo
.mNumSamplesSkippedTotal
+= aSkipped
;
2792 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped
) {
2793 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2795 MOZ_ASSERT(OnTaskQueue());
2796 LOG("Skipping succeeded, skipped %u frames", aSkipped
);
2797 mSkipRequest
.Complete();
2799 DDLOG(DDLogCategory::Log
, "video_skipped", DDNoValue());
2801 VideoSkipReset(aSkipped
);
2803 ScheduleUpdate(TrackInfo::kVideoTrack
);
2806 void MediaFormatReader::OnVideoSkipFailed(
2807 MediaTrackDemuxer::SkipFailureHolder aFailure
) {
2808 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK
);
2809 MOZ_ASSERT(OnTaskQueue());
2810 LOG("Skipping failed, skipped %u frames", aFailure
.mSkipped
);
2811 mSkipRequest
.Complete();
2813 switch (aFailure
.mFailure
.Code()) {
2814 case NS_ERROR_DOM_MEDIA_END_OF_STREAM
:
2815 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
:
2816 DDLOG(DDLogCategory::Log
, "video_skipping_interruption",
2818 // Some frames may have been output by the decoder since we initiated the
2819 // videoskip process and we know they would be late.
2820 DropDecodedSamples(TrackInfo::kVideoTrack
);
2821 // We can't complete the skip operation, will just service a video frame
2823 ScheduleUpdate(TrackInfo::kVideoTrack
);
2825 case NS_ERROR_DOM_MEDIA_CANCELED
:
2826 DDLOG(DDLogCategory::Log
, "video_skipping_interruption",
2828 if (mVideo
.HasPromise()) {
2829 mVideo
.RejectPromise(aFailure
.mFailure
, __func__
);
2833 DDLOG(DDLogCategory::Log
, "video_skipping_error", aFailure
.mFailure
);
2834 NotifyError(TrackType::kVideoTrack
, aFailure
.mFailure
);
2839 RefPtr
<MediaFormatReader::SeekPromise
> MediaFormatReader::Seek(
2840 const SeekTarget
& aTarget
) {
2841 AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK
);
2842 MOZ_ASSERT(OnTaskQueue());
2844 LOG("aTarget=(%" PRId64
"), track=%s", aTarget
.GetTime().ToMicroseconds(),
2845 SeekTarget::TrackToStr(aTarget
.GetTrack()));
2847 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise
.IsEmpty());
2848 MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime
.isNothing());
2849 // Should reset data request, and no pending internal seek.
2850 if (aTarget
.IsAllTracks()) {
2851 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise());
2852 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise());
2853 MOZ_DIAGNOSTIC_ASSERT(mVideo
.mTimeThreshold
.isNothing());
2854 MOZ_DIAGNOSTIC_ASSERT(mAudio
.mTimeThreshold
.isNothing());
2855 } else if (aTarget
.IsVideoOnly()) {
2856 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.HasPromise());
2857 MOZ_DIAGNOSTIC_ASSERT(mVideo
.mTimeThreshold
.isNothing());
2858 } else if (aTarget
.IsAudioOnly()) {
2859 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.HasPromise());
2860 MOZ_DIAGNOSTIC_ASSERT(mAudio
.mTimeThreshold
.isNothing());
2863 if (!mInfo
.mMediaSeekable
&& !mInfo
.mMediaSeekableOnlyInBufferedRanges
) {
2864 LOG("Seek() END (Unseekable)");
2865 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
2869 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE
, __func__
);
2872 SetSeekTarget(aTarget
);
2874 RefPtr
<SeekPromise
> p
= mSeekPromise
.Ensure(__func__
);
2881 void MediaFormatReader::SetSeekTarget(const SeekTarget
& aTarget
) {
2882 MOZ_ASSERT(OnTaskQueue());
2884 mOriginalSeekTarget
= aTarget
;
2885 mFallbackSeekTime
= mPendingSeekTime
= Some(aTarget
.GetTime());
2888 void MediaFormatReader::ScheduleSeek() {
2889 if (mSeekScheduled
) {
2892 mSeekScheduled
= true;
2893 nsresult rv
= OwnerThread()->Dispatch(NewRunnableMethod(
2894 "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek
));
2895 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv
));
2899 void MediaFormatReader::AttemptSeek() {
2900 AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK
);
2901 MOZ_ASSERT(OnTaskQueue());
2903 mSeekScheduled
= false;
2905 if (mPendingSeekTime
.isNothing()) {
2906 LOGV("AttemptSeek, no pending seek time?");
2910 // Only reset the demuxers targeted by this SeekTarget, to avoid A/V sync
2912 const bool isSeekingAudio
= HasAudio() && !mOriginalSeekTarget
.IsVideoOnly();
2913 const bool isSeekingVideo
= HasVideo() && !mOriginalSeekTarget
.IsAudioOnly();
2914 LOG("AttemptSeek, seekingAudio=%d, seekingVideo=%d", isSeekingAudio
,
2916 if (isSeekingVideo
) {
2917 mVideo
.ResetDemuxer();
2918 mVideo
.ResetState();
2920 if (isSeekingAudio
) {
2921 mAudio
.ResetDemuxer();
2922 mAudio
.ResetState();
2925 // If seeking both tracks, seek the video track, and then the audio track when
2926 // the video track seek has completed. Otherwise, only seek a specific track.
2927 if (isSeekingVideo
) {
2929 } else if (isSeekingAudio
) {
2936 void MediaFormatReader::OnSeekFailed(TrackType aTrack
,
2937 const MediaResult
& aError
) {
2938 AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK
);
2939 MOZ_ASSERT(OnTaskQueue());
2940 LOGV("%s failure:%s", TrackTypeToStr(aTrack
), aError
.ErrorName().get());
2941 if (aTrack
== TrackType::kVideoTrack
) {
2942 mVideo
.mSeekRequest
.Complete();
2944 mAudio
.mSeekRequest
.Complete();
2947 if (aError
== NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
) {
2948 if (HasVideo() && aTrack
== TrackType::kAudioTrack
&&
2949 mFallbackSeekTime
.isSome() &&
2950 mPendingSeekTime
.ref() != mFallbackSeekTime
.ref()) {
2951 // We have failed to seek audio where video seeked to earlier.
2952 // Attempt to seek instead to the closest point that we know we have in
2953 // order to limit A/V sync discrepency.
2955 // Ensure we have the most up to date buffered ranges.
2956 UpdateReceivedNewData(TrackType::kAudioTrack
);
2957 Maybe
<TimeUnit
> nextSeekTime
;
2958 // Find closest buffered time found after video seeked time.
2959 for (const auto& timeRange
: mAudio
.mTimeRanges
) {
2960 if (timeRange
.mStart
>= mPendingSeekTime
.ref()) {
2961 nextSeekTime
.emplace(timeRange
.mStart
);
2965 if (nextSeekTime
.isNothing() ||
2966 nextSeekTime
.ref() > mFallbackSeekTime
.ref()) {
2967 nextSeekTime
= Some(mFallbackSeekTime
.ref());
2968 LOG("Unable to seek audio to video seek time. A/V sync may be broken");
2970 mFallbackSeekTime
.reset();
2972 mPendingSeekTime
= nextSeekTime
;
2976 NotifyWaitingForData(aTrack
);
2978 MOZ_ASSERT(!mVideo
.mSeekRequest
.Exists() && !mAudio
.mSeekRequest
.Exists());
2979 mPendingSeekTime
.reset();
2981 auto type
= aTrack
== TrackType::kAudioTrack
? MediaData::Type::AUDIO_DATA
2982 : MediaData::Type::VIDEO_DATA
;
2983 mSeekPromise
.RejectIfExists(SeekRejectValue(type
, aError
), __func__
);
2986 void MediaFormatReader::DoVideoSeek() {
2987 AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK
);
2988 MOZ_ASSERT(mPendingSeekTime
.isSome());
2989 LOGV("Seeking video to %" PRId64
, mPendingSeekTime
.ref().ToMicroseconds());
2990 MOZ_DIAGNOSTIC_ASSERT(!IsAudioOnlySeeking());
2991 MOZ_DIAGNOSTIC_ASSERT(!mVideo
.mSeekRequest
.Exists());
2992 auto seekTime
= mPendingSeekTime
.ref();
2993 mVideo
.mTrackDemuxer
->Seek(seekTime
)
2994 ->Then(OwnerThread(), __func__
, this,
2995 &MediaFormatReader::OnVideoSeekCompleted
,
2996 &MediaFormatReader::OnVideoSeekFailed
)
2997 ->Track(mVideo
.mSeekRequest
);
3000 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime
) {
3001 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
3003 MOZ_ASSERT(OnTaskQueue());
3004 LOGV("Video seeked to %" PRId64
, aTime
.ToMicroseconds());
3005 mVideo
.mSeekRequest
.Complete();
3007 mVideo
.mFirstFrameTime
= Some(aTime
);
3008 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
3010 SetVideoDecodeThreshold();
3012 if (HasAudio() && !mOriginalSeekTarget
.IsVideoOnly()) {
3013 MOZ_ASSERT(mPendingSeekTime
.isSome());
3014 if (mOriginalSeekTarget
.IsFast()) {
3015 // We are performing a fast seek. We need to seek audio to where the
3016 // video seeked to, to ensure proper A/V sync once playback resume.
3017 mPendingSeekTime
= Some(aTime
);
3021 mPendingSeekTime
.reset();
3022 mSeekPromise
.ResolveIfExists(aTime
, __func__
);
3026 void MediaFormatReader::OnVideoSeekFailed(const MediaResult
& aError
) {
3027 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK
);
3028 mPreviousDecodedKeyframeTime_us
= sNoPreviousDecodedKeyframe
;
3029 OnSeekFailed(TrackType::kVideoTrack
, aError
);
3032 void MediaFormatReader::SetVideoDecodeThreshold() {
3033 MOZ_ASSERT(OnTaskQueue());
3035 if (!HasVideo() || !mVideo
.mDecoder
) {
3039 if (!mVideo
.mTimeThreshold
&& !IsSeeking()) {
3044 if (mVideo
.mTimeThreshold
) {
3045 // For internalSeek.
3046 threshold
= mVideo
.mTimeThreshold
.ref().Time();
3047 } else if (IsSeeking()) {
3048 // If IsSeeking() is true, then video seek must have completed already.
3050 if (NS_FAILED(mVideo
.mTrackDemuxer
->GetNextRandomAccessPoint(&keyframe
))) {
3054 // If the key frame is invalid/infinite, it means the target position is
3055 // closing to end of stream. We don't want to skip any frame at this point.
3056 threshold
= keyframe
.IsValid() && !keyframe
.IsInfinite()
3057 ? mOriginalSeekTarget
.GetTime()
3058 : TimeUnit::Invalid();
3063 if (threshold
.IsValid()) {
3064 LOG("Set seek threshold to %" PRId64
, threshold
.ToMicroseconds());
3066 LOG("Resetting seek threshold");
3068 mVideo
.mDecoder
->SetSeekThreshold(threshold
);
3071 void MediaFormatReader::DoAudioSeek() {
3072 AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK
);
3073 MOZ_ASSERT(mPendingSeekTime
.isSome());
3074 LOGV("Seeking audio to %" PRId64
, mPendingSeekTime
.ref().ToMicroseconds());
3075 MOZ_DIAGNOSTIC_ASSERT(!IsVideoOnlySeeking());
3076 MOZ_DIAGNOSTIC_ASSERT(!mAudio
.mSeekRequest
.Exists());
3077 auto seekTime
= mPendingSeekTime
.ref();
3078 mAudio
.mTrackDemuxer
->Seek(seekTime
)
3079 ->Then(OwnerThread(), __func__
, this,
3080 &MediaFormatReader::OnAudioSeekCompleted
,
3081 &MediaFormatReader::OnAudioSeekFailed
)
3082 ->Track(mAudio
.mSeekRequest
);
3085 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime
) {
3086 MOZ_ASSERT(OnTaskQueue());
3087 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
3089 LOGV("Audio seeked to %" PRId64
, aTime
.ToMicroseconds());
3090 mAudio
.mSeekRequest
.Complete();
3091 mAudio
.mFirstFrameTime
= Some(aTime
);
3092 mPendingSeekTime
.reset();
3093 mSeekPromise
.ResolveIfExists(aTime
, __func__
);
3096 void MediaFormatReader::OnAudioSeekFailed(const MediaResult
& aError
) {
3097 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK
);
3098 OnSeekFailed(TrackType::kAudioTrack
, aError
);
3101 void MediaFormatReader::ReleaseResources() {
3106 ShutdownDecoder(TrackInfo::kAudioTrack
);
3107 ShutdownDecoder(TrackInfo::kVideoTrack
);
3110 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
3111 return mVideo
.mIsHardwareAccelerated
;
3114 void MediaFormatReader::NotifyTrackDemuxers() {
3115 MOZ_ASSERT(OnTaskQueue());
3124 mVideo
.mReceivedNewData
= true;
3125 ScheduleUpdate(TrackType::kVideoTrack
);
3128 mAudio
.mReceivedNewData
= true;
3129 ScheduleUpdate(TrackType::kAudioTrack
);
3133 void MediaFormatReader::NotifyDataArrived() {
3134 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK
);
3135 MOZ_ASSERT(OnTaskQueue());
3137 if (mShutdown
|| !mDemuxer
|| !mDemuxerInitDone
) {
3141 if (mNotifyDataArrivedPromise
.Exists()) {
3142 // Already one in progress. Set the dirty flag so we can process it later.
3143 mPendingNotifyDataArrived
= true;
3147 RefPtr
<MediaFormatReader
> self
= this;
3148 mDemuxer
->NotifyDataArrived()
3150 OwnerThread(), __func__
,
3152 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
3154 self
->mNotifyDataArrivedPromise
.Complete();
3155 self
->UpdateBuffered();
3156 self
->NotifyTrackDemuxers();
3157 if (self
->mPendingNotifyDataArrived
) {
3158 self
->mPendingNotifyDataArrived
= false;
3159 self
->NotifyDataArrived();
3162 [self
]() { self
->mNotifyDataArrivedPromise
.Complete(); })
3163 ->Track(mNotifyDataArrivedPromise
);
3166 void MediaFormatReader::UpdateMediaEngineId(uint64_t aMediaEngineId
) {
3167 LOG("Update external media engine Id %" PRIu64
, aMediaEngineId
);
3168 mMediaEngineId
= Some(aMediaEngineId
);
3171 void MediaFormatReader::UpdateBuffered() {
3172 AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK
);
3173 MOZ_ASSERT(OnTaskQueue());
3179 if (!mInitDone
|| !mHasStartTime
) {
3180 mBuffered
= TimeIntervals();
3185 mVideo
.mTimeRanges
= mVideo
.mTrackDemuxer
->GetBuffered();
3187 auto lastEnd
= mVideo
.mTimeRanges
.GetEnd(&hasLastEnd
);
3189 if (mVideo
.mLastTimeRangesEnd
&&
3190 mVideo
.mLastTimeRangesEnd
.ref() < lastEnd
) {
3191 // New data was added after our previous end, we can clear the EOS flag.
3192 mVideo
.mDemuxEOS
= false;
3193 ScheduleUpdate(TrackInfo::kVideoTrack
);
3195 mVideo
.mLastTimeRangesEnd
= Some(lastEnd
);
3199 mAudio
.mTimeRanges
= mAudio
.mTrackDemuxer
->GetBuffered();
3201 auto lastEnd
= mAudio
.mTimeRanges
.GetEnd(&hasLastEnd
);
3203 if (mAudio
.mLastTimeRangesEnd
&&
3204 mAudio
.mLastTimeRangesEnd
.ref() < lastEnd
) {
3205 // New data was added after our previous end, we can clear the EOS flag.
3206 mAudio
.mDemuxEOS
= false;
3207 ScheduleUpdate(TrackInfo::kAudioTrack
);
3209 mAudio
.mLastTimeRangesEnd
= Some(lastEnd
);
3213 media::TimeIntervals intervals
;
3214 if (HasAudio() && HasVideo()) {
3215 intervals
= media::Intersection(mVideo
.mTimeRanges
, mAudio
.mTimeRanges
);
3216 } else if (HasAudio()) {
3217 intervals
= mAudio
.mTimeRanges
;
3218 } else if (HasVideo()) {
3219 intervals
= mVideo
.mTimeRanges
;
3222 if (intervals
.IsEmpty() || intervals
.GetStart() == TimeUnit::Zero()) {
3223 // IntervalSet already starts at 0 or is empty, nothing to shift.
3224 mBuffered
= intervals
;
3226 mBuffered
= intervals
.Shift(TimeUnit::Zero() - mInfo
.mStartTime
);
3230 layers::ImageContainer
* MediaFormatReader::GetImageContainer() {
3231 return mVideoFrameContainer
? mVideoFrameContainer
->GetImageContainer()
3235 RefPtr
<GenericPromise
> MediaFormatReader::RequestDebugInfo(
3236 dom::MediaFormatReaderDebugInfo
& aInfo
) {
3237 if (!OnTaskQueue()) {
3238 // Run the request on the task queue if it's not already.
3239 return InvokeAsync(mTaskQueue
, __func__
,
3240 [this, self
= RefPtr
{this}, &aInfo
] {
3241 return RequestDebugInfo(aInfo
);
3244 GetDebugInfo(aInfo
);
3245 return GenericPromise::CreateAndResolve(true, __func__
);
3248 Maybe
<nsCString
> MediaFormatReader::GetAudioProcessPerCodec() {
3249 if (mAudio
.mDescription
== "uninitialized"_ns
) {
3253 MOZ_ASSERT(mAudio
.mProcessName
.Length() > 0,
3254 "Should have had a process name");
3255 MOZ_ASSERT(mAudio
.mCodecName
.Length() > 0, "Should have had a codec name");
3257 nsCString processName
= mAudio
.mProcessName
;
3258 nsCString
audioProcessPerCodecName(processName
+ ","_ns
+ mAudio
.mCodecName
);
3259 if (processName
!= "utility"_ns
) {
3260 if (!StaticPrefs::media_rdd_process_enabled()) {
3261 audioProcessPerCodecName
+= ",rdd-disabled"_ns
;
3263 if (!StaticPrefs::media_utility_process_enabled()) {
3264 audioProcessPerCodecName
+= ",utility-disabled"_ns
;
3267 return Some(audioProcessPerCodecName
);
3270 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo
& aInfo
) {
3271 MOZ_ASSERT(OnTaskQueue(),
3272 "Don't call this off the task queue, it's going to touch a lot of "
3275 nsAutoCString
audioDecoderName("unavailable");
3276 nsAutoCString videoDecoderName
= audioDecoderName
;
3277 nsAutoCString
audioType("none");
3278 nsAutoCString
videoType("none");
3280 AudioInfo audioInfo
;
3282 audioInfo
= *mAudio
.GetWorkingInfo()->GetAsAudioInfo();
3283 audioDecoderName
= mAudio
.mDecoder
? mAudio
.mDecoder
->GetDescriptionName()
3284 : mAudio
.mDescription
;
3285 audioType
= audioInfo
.mMimeType
;
3286 aInfo
.mAudioState
.mNeedInput
= NeedInput(mAudio
);
3287 aInfo
.mAudioState
.mHasPromise
= mAudio
.HasPromise();
3288 aInfo
.mAudioState
.mWaitingPromise
= !mAudio
.mWaitingPromise
.IsEmpty();
3289 aInfo
.mAudioState
.mHasDemuxRequest
= mAudio
.mDemuxRequest
.Exists();
3290 aInfo
.mAudioState
.mDemuxQueueSize
=
3291 uint32_t(mAudio
.mQueuedSamples
.Length());
3292 aInfo
.mAudioState
.mHasDecoder
= mAudio
.mDecodeRequest
.Exists();
3293 aInfo
.mAudioState
.mTimeTreshold
=
3294 mAudio
.mTimeThreshold
? mAudio
.mTimeThreshold
.ref().Time().ToSeconds()
3296 aInfo
.mAudioState
.mTimeTresholdHasSeeked
=
3297 mAudio
.mTimeThreshold
? mAudio
.mTimeThreshold
.ref().mHasSeeked
: false;
3298 aInfo
.mAudioState
.mNumSamplesInput
= mAudio
.mNumSamplesInput
;
3299 aInfo
.mAudioState
.mNumSamplesOutput
= mAudio
.mNumSamplesOutput
;
3300 aInfo
.mAudioState
.mQueueSize
= size_t(mAudio
.mSizeOfQueue
);
3301 aInfo
.mAudioState
.mPending
= mAudio
.mOutput
.Length();
3302 aInfo
.mAudioState
.mWaitingForData
= mAudio
.mWaitingForData
;
3303 aInfo
.mAudioState
.mDemuxEOS
= mAudio
.mDemuxEOS
;
3304 aInfo
.mAudioState
.mDrainState
= int32_t(mAudio
.mDrainState
);
3305 aInfo
.mAudioState
.mWaitingForKey
= mAudio
.mWaitingForKey
;
3306 aInfo
.mAudioState
.mLastStreamSourceID
= mAudio
.mLastStreamSourceID
;
3309 CopyUTF8toUTF16(audioDecoderName
, aInfo
.mAudioDecoderName
);
3310 CopyUTF8toUTF16(audioType
, aInfo
.mAudioType
);
3311 aInfo
.mAudioChannels
= audioInfo
.mChannels
;
3312 aInfo
.mAudioRate
= audioInfo
.mRate
;
3313 aInfo
.mAudioFramesDecoded
= mAudio
.mNumSamplesOutputTotal
;
3315 VideoInfo videoInfo
;
3317 videoInfo
= *mVideo
.GetWorkingInfo()->GetAsVideoInfo();
3318 videoDecoderName
= mVideo
.mDecoder
? mVideo
.mDecoder
->GetDescriptionName()
3319 : mVideo
.mDescription
;
3320 videoType
= videoInfo
.mMimeType
;
3321 aInfo
.mVideoState
.mNeedInput
= NeedInput(mVideo
);
3322 aInfo
.mVideoState
.mHasPromise
= mVideo
.HasPromise();
3323 aInfo
.mVideoState
.mWaitingPromise
= !mVideo
.mWaitingPromise
.IsEmpty();
3324 aInfo
.mVideoState
.mHasDemuxRequest
= mVideo
.mDemuxRequest
.Exists();
3325 aInfo
.mVideoState
.mDemuxQueueSize
=
3326 uint32_t(mVideo
.mQueuedSamples
.Length());
3327 aInfo
.mVideoState
.mHasDecoder
= mVideo
.mDecodeRequest
.Exists();
3328 aInfo
.mVideoState
.mTimeTreshold
=
3329 mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
.ref().Time().ToSeconds()
3331 aInfo
.mVideoState
.mTimeTresholdHasSeeked
=
3332 mVideo
.mTimeThreshold
? mVideo
.mTimeThreshold
.ref().mHasSeeked
: false;
3333 aInfo
.mVideoState
.mNumSamplesInput
= mVideo
.mNumSamplesInput
;
3334 aInfo
.mVideoState
.mNumSamplesOutput
= mVideo
.mNumSamplesOutput
;
3335 aInfo
.mVideoState
.mQueueSize
= size_t(mVideo
.mSizeOfQueue
);
3336 aInfo
.mVideoState
.mPending
= mVideo
.mOutput
.Length();
3337 aInfo
.mVideoState
.mWaitingForData
= mVideo
.mWaitingForData
;
3338 aInfo
.mVideoState
.mDemuxEOS
= mVideo
.mDemuxEOS
;
3339 aInfo
.mVideoState
.mDrainState
= int32_t(mVideo
.mDrainState
);
3340 aInfo
.mVideoState
.mWaitingForKey
= mVideo
.mWaitingForKey
;
3341 aInfo
.mVideoState
.mLastStreamSourceID
= mVideo
.mLastStreamSourceID
;
3344 CopyUTF8toUTF16(videoDecoderName
, aInfo
.mVideoDecoderName
);
3345 CopyUTF8toUTF16(videoType
, aInfo
.mVideoType
);
3347 videoInfo
.mDisplay
.width
< 0 ? 0 : videoInfo
.mDisplay
.width
;
3348 aInfo
.mVideoHeight
=
3349 videoInfo
.mDisplay
.height
< 0 ? 0 : videoInfo
.mDisplay
.height
;
3350 aInfo
.mVideoRate
= mVideo
.mMeanRate
.Mean();
3351 aInfo
.mVideoHardwareAccelerated
= VideoIsHardwareAccelerated();
3352 aInfo
.mVideoNumSamplesOutputTotal
= mVideo
.mNumSamplesOutputTotal
;
3353 aInfo
.mVideoNumSamplesSkippedTotal
= mVideo
.mNumSamplesSkippedTotal
;
3355 // Looking at dropped frames
3356 FrameStatisticsData stats
= mFrameStats
->GetFrameStatisticsData();
3357 aInfo
.mFrameStats
.mDroppedDecodedFrames
= stats
.mDroppedDecodedFrames
;
3358 aInfo
.mFrameStats
.mDroppedSinkFrames
= stats
.mDroppedSinkFrames
;
3359 aInfo
.mFrameStats
.mDroppedCompositorFrames
= stats
.mDroppedCompositorFrames
;
3362 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode
) {
3363 MOZ_ASSERT(OnTaskQueue());
3364 return SetNullDecode(TrackType::kVideoTrack
, aIsNullDecode
);
3367 void MediaFormatReader::UpdateCompositor(
3368 already_AddRefed
<layers::KnowsCompositor
> aCompositor
) {
3369 MOZ_ASSERT(OnTaskQueue());
3370 mKnowsCompositor
= aCompositor
;
3373 void MediaFormatReader::SetNullDecode(TrackType aTrack
, bool aIsNullDecode
) {
3374 MOZ_ASSERT(OnTaskQueue());
3376 auto& decoder
= GetDecoderData(aTrack
);
3377 if (decoder
.mIsNullDecode
== aIsNullDecode
) {
3381 LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3382 TrackTypeToStr(aTrack
), decoder
.mIsNullDecode
, aIsNullDecode
);
3384 decoder
.mIsNullDecode
= aIsNullDecode
;
3385 ShutdownDecoder(aTrack
);
3388 void MediaFormatReader::OnFirstDemuxCompleted(
3389 TrackInfo::TrackType aType
,
3390 RefPtr
<MediaTrackDemuxer::SamplesHolder
> aSamples
) {
3391 AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3393 MOZ_ASSERT(OnTaskQueue());
3399 auto& decoder
= GetDecoderData(aType
);
3400 MOZ_ASSERT(decoder
.mFirstDemuxedSampleTime
.isNothing());
3401 decoder
.mFirstDemuxedSampleTime
.emplace(aSamples
->GetSamples()[0]->mTime
);
3402 MaybeResolveMetadataPromise();
3405 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType
,
3406 const MediaResult
& aError
) {
3407 MOZ_ASSERT(OnTaskQueue());
3413 auto& decoder
= GetDecoderData(aType
);
3414 MOZ_ASSERT(decoder
.mFirstDemuxedSampleTime
.isNothing());
3415 decoder
.mFirstDemuxedSampleTime
.emplace(TimeUnit::FromInfinity());
3416 MaybeResolveMetadataPromise();
3419 } // namespace mozilla
3421 #undef NS_DispatchToMainThread