Bug 1839170 - Refactor Snap pulling, Add Firefox Snap Core22 and GNOME 42 SDK symbols...
[gecko.git] / dom / media / MediaFormatReader.cpp
blob45e4f3e7a96203b32555d8de3325222ef8c54383
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "MediaFormatReader.h"
9 #include <algorithm>
10 #include <map>
11 #include <queue>
13 #include "AllocationPolicy.h"
14 #ifdef MOZ_AV1
15 # include "AOMDecoder.h"
16 #endif
17 #include "DecoderBenchmark.h"
18 #include "MediaData.h"
19 #include "MediaDataDecoderProxy.h"
20 #include "MediaInfo.h"
21 #include "MP4Decoder.h"
22 #include "PDMFactory.h"
23 #include "PerformanceRecorder.h"
24 #include "VideoFrameContainer.h"
25 #include "VideoUtils.h"
26 #include "VPXDecoder.h"
27 #include "mozilla/AbstractThread.h"
28 #include "mozilla/CDMProxy.h"
29 #include "mozilla/ClearOnShutdown.h"
30 #include "mozilla/NotNull.h"
31 #include "mozilla/Preferences.h"
32 #include "mozilla/ProfilerLabels.h"
33 #include "mozilla/ProfilerMarkers.h"
34 #include "mozilla/SharedThreadPool.h"
35 #include "mozilla/StaticPrefs_media.h"
36 #include "mozilla/TaskQueue.h"
37 #include "mozilla/Unused.h"
38 #include "nsContentUtils.h"
39 #include "nsPrintfCString.h"
40 #include "nsTHashSet.h"
42 using namespace mozilla::media;
44 static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
45 mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
47 #define LOG(arg, ...) \
48 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
49 __func__, ##__VA_ARGS__)
50 #define LOGV(arg, ...) \
51 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
52 __func__, ##__VA_ARGS__)
54 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
56 namespace mozilla {
58 typedef void* MediaDataDecoderID;
60 /**
61 * This class tracks shutdown promises to ensure all decoders are shut down
62 * completely before MFR continues the rest of the shutdown procedure.
64 class MediaFormatReader::ShutdownPromisePool {
65 public:
66 ShutdownPromisePool()
67 : mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}
69 // Return a promise which will be resolved when all the tracking promises
70 // are resolved. Note no more promises should be added for tracking once
71 // this function is called.
72 RefPtr<ShutdownPromise> Shutdown();
74 // Track a shutdown promise.
75 void Track(RefPtr<ShutdownPromise> aPromise);
77 // Shut down a decoder and track its shutdown promise.
78 void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {
79 Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());
82 private:
83 bool mShutdown = false;
84 const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;
85 nsTHashSet<RefPtr<ShutdownPromise>> mPromises;
88 RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {
89 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
90 mShutdown = true;
91 if (mPromises.Count() == 0) {
92 mOnShutdownComplete->Resolve(true, __func__);
94 return mOnShutdownComplete;
97 void MediaFormatReader::ShutdownPromisePool::Track(
98 RefPtr<ShutdownPromise> aPromise) {
99 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
100 MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));
101 mPromises.Insert(aPromise);
102 aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {
103 MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));
104 mPromises.Remove(aPromise);
105 if (mShutdown && mPromises.Count() == 0) {
106 mOnShutdownComplete->Resolve(true, __func__);
111 void MediaFormatReader::DecoderData::ShutdownDecoder() {
112 MOZ_ASSERT(mOwner->OnTaskQueue());
114 MutexAutoLock lock(mMutex);
116 if (!mDecoder) {
117 // No decoder to shut down.
118 return;
121 if (mFlushing) {
122 // Flush is is in action. Shutdown will be initiated after flush completes.
123 MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);
124 mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));
125 // The order of decoder creation and shutdown is handled by LocalAllocPolicy
126 // and ShutdownPromisePool. MFR can now reset these members to a fresh state
127 // and be ready to create new decoders again without explicitly waiting for
128 // flush/shutdown to complete.
129 mShutdownPromise = nullptr;
130 mFlushing = false;
131 } else {
132 // No flush is in action. We can shut down the decoder now.
133 mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());
136 // mShutdownPromisePool will handle the order of decoder shutdown so
137 // we can forget mDecoder and be ready to create a new one.
138 mDecoder = nullptr;
139 mDescription = "shutdown"_ns;
140 mHasReportedVideoHardwareSupportTelemtry = false;
141 mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA
142 ? TrackType::kAudioTrack
143 : TrackType::kVideoTrack);
146 void MediaFormatReader::DecoderData::Flush() {
147 AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);
148 MOZ_ASSERT(mOwner->OnTaskQueue());
150 if (mFlushing || mFlushed) {
151 // Flush still pending or already flushed, nothing more to do.
152 return;
154 mDecodeRequest.DisconnectIfExists();
155 mDrainRequest.DisconnectIfExists();
156 mDrainState = DrainState::None;
157 CancelWaitingForKey();
158 mOutput.Clear();
159 mNumSamplesInput = 0;
160 mNumSamplesOutput = 0;
161 mSizeOfQueue = 0;
162 if (mDecoder) {
163 TrackType type = mType == MediaData::Type::AUDIO_DATA
164 ? TrackType::kAudioTrack
165 : TrackType::kVideoTrack;
166 mFlushing = true;
167 MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);
168 mShutdownPromise = new SharedShutdownPromiseHolder();
169 RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;
170 RefPtr<MediaDataDecoder> d = mDecoder;
171 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
172 "flushing", DDNoValue{});
173 mDecoder->Flush()->Then(
174 mOwner->OwnerThread(), __func__,
175 [type, this, p, d]() {
176 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
177 MEDIA_PLAYBACK);
178 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
179 "flushed", DDNoValue{});
180 if (!p->IsEmpty()) {
181 // Shutdown happened before flush completes.
182 // Let's continue to shut down the decoder. Note
183 // we don't access |this| because this decoder
184 // is no longer managed by MFR::DecoderData.
185 d->Shutdown()->ChainTo(p->Steal(), __func__);
186 return;
188 mFlushing = false;
189 mShutdownPromise = nullptr;
190 mOwner->ScheduleUpdate(type);
192 [type, this, p, d](const MediaResult& aError) {
193 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
194 MEDIA_PLAYBACK);
195 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
196 "flush_error", aError);
197 if (!p->IsEmpty()) {
198 d->Shutdown()->ChainTo(p->Steal(), __func__);
199 return;
201 mFlushing = false;
202 mShutdownPromise = nullptr;
203 mOwner->NotifyError(type, aError);
206 mFlushed = true;
209 class MediaFormatReader::DecoderFactory {
210 using InitPromise = MediaDataDecoder::InitPromise;
211 using TokenPromise = AllocPolicy::Promise;
212 using Token = AllocPolicy::Token;
213 using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise;
215 public:
216 explicit DecoderFactory(MediaFormatReader* aOwner)
217 : mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),
218 mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),
219 mOwner(WrapNotNull(aOwner)) {
220 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
221 this);
222 DecoderDoctorLogger::LinkParentAndChild(
223 aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);
226 ~DecoderFactory() {
227 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
228 this);
231 void CreateDecoder(TrackType aTrack);
233 // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
234 // pristine state so CreateDecoder() is ready to be called again immediately.
235 void ShutdownDecoder(TrackType aTrack) {
236 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
237 aTrack == TrackInfo::kVideoTrack);
238 auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
239 data.mPolicy->Cancel();
240 data.mTokenRequest.DisconnectIfExists();
241 if (data.mLiveToken) {
242 // We haven't completed creation of the decoder, and it hasn't been
243 // initialised yet.
244 data.mLiveToken = nullptr;
245 // The decoder will be shutdown as soon as it's available and tracked by
246 // the ShutdownPromisePool.
247 mOwner->mShutdownPromisePool->Track(data.mCreateDecoderPromise->Then(
248 mOwner->mTaskQueue, __func__,
249 [](CreateDecoderPromise::ResolveOrRejectValue&& aResult) {
250 if (aResult.IsReject()) {
251 return ShutdownPromise::CreateAndResolve(true, __func__);
253 return aResult.ResolveValue()->Shutdown();
254 }));
255 // Free the token to leave room for a new decoder.
256 data.mToken = nullptr;
258 data.mInitRequest.DisconnectIfExists();
259 if (data.mDecoder) {
260 mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());
262 data.mStage = Stage::None;
263 MOZ_ASSERT(!data.mToken);
266 private:
267 enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };
269 struct Data {
270 Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)
271 : mOwnerData(aOwnerData),
272 mTrack(aTrack),
273 mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}
274 DecoderData& mOwnerData;
275 const TrackType mTrack;
276 RefPtr<SingleAllocPolicy> mPolicy;
277 Stage mStage = Stage::None;
278 RefPtr<Token> mToken;
279 RefPtr<MediaDataDecoder> mDecoder;
280 MozPromiseRequestHolder<TokenPromise> mTokenRequest;
281 struct DecoderCancelled : public SupportsWeakPtr {
282 NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled)
283 private:
284 ~DecoderCancelled() = default;
286 // Set when decoder is about to be created. If cleared before the decoder
287 // creation promise is resolved; it indicates that Shutdown() was called and
288 // further processing such as initialization should stop.
289 RefPtr<DecoderCancelled> mLiveToken;
290 RefPtr<CreateDecoderPromise> mCreateDecoderPromise;
291 MozPromiseRequestHolder<InitPromise> mInitRequest;
292 } mAudio, mVideo;
294 void RunStage(Data& aData);
295 void DoCreateDecoder(Data& aData);
296 void DoInitDecoder(Data& aData);
298 // guaranteed to be valid by the owner.
299 const NotNull<MediaFormatReader*> mOwner;
302 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {
303 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
304 aTrack == TrackInfo::kVideoTrack);
305 RunStage(aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo);
308 void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {
309 switch (aData.mStage) {
310 case Stage::None: {
311 MOZ_ASSERT(!aData.mToken);
312 aData.mPolicy->Alloc()
313 ->Then(
314 mOwner->OwnerThread(), __func__,
315 [this, &aData](RefPtr<Token> aToken) {
316 aData.mTokenRequest.Complete();
317 aData.mToken = std::move(aToken);
318 aData.mStage = Stage::CreateDecoder;
319 RunStage(aData);
321 [&aData]() {
322 aData.mTokenRequest.Complete();
323 aData.mStage = Stage::None;
325 ->Track(aData.mTokenRequest);
326 aData.mStage = Stage::WaitForToken;
327 break;
330 case Stage::WaitForToken: {
331 MOZ_ASSERT(!aData.mToken);
332 MOZ_ASSERT(aData.mTokenRequest.Exists());
333 break;
336 case Stage::CreateDecoder: {
337 MOZ_ASSERT(aData.mToken);
338 MOZ_ASSERT(!aData.mDecoder);
339 MOZ_ASSERT(!aData.mInitRequest.Exists());
341 DoCreateDecoder(aData);
342 aData.mStage = Stage::WaitForInit;
343 break;
346 case Stage::WaitForInit: {
347 MOZ_ASSERT((aData.mDecoder && aData.mInitRequest.Exists()) ||
348 aData.mLiveToken);
349 break;
354 void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {
355 AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);
356 auto& ownerData = aData.mOwnerData;
357 auto& decoder = mOwner->GetDecoderData(aData.mTrack);
359 RefPtr<PDMFactory> platform = new PDMFactory();
360 if (decoder.IsEncrypted()) {
361 MOZ_ASSERT(mOwner->mCDMProxy);
362 platform->SetCDMProxy(mOwner->mCDMProxy);
365 RefPtr<PlatformDecoderModule::CreateDecoderPromise> p;
366 MediaFormatReader* owner = mOwner;
367 auto onWaitingForKeyEvent =
368 [owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)]() {
369 RefPtr<MediaFormatReader> mfr(owner);
370 MOZ_DIAGNOSTIC_ASSERT(mfr, "The MediaFormatReader didn't wait for us");
371 return mfr ? &mfr->OnTrackWaitingForKeyProducer() : nullptr;
374 switch (aData.mTrack) {
375 case TrackInfo::kAudioTrack: {
376 p = platform->CreateDecoder(
377 {*ownerData.GetCurrentInfo()->GetAsAudioInfo(), mOwner->mCrashHelper,
378 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
379 TrackInfo::kAudioTrack, std::move(onWaitingForKeyEvent),
380 mOwner->mMediaEngineId, mOwner->mTrackingId});
381 break;
384 case TrackType::kVideoTrack: {
385 // Decoders use the layers backend to decide if they can use hardware
386 // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
387 using Option = CreateDecoderParams::Option;
388 using OptionSet = CreateDecoderParams::OptionSet;
390 p = platform->CreateDecoder(
391 {*ownerData.GetCurrentInfo()->GetAsVideoInfo(),
392 mOwner->mKnowsCompositor, mOwner->GetImageContainer(),
393 mOwner->mCrashHelper,
394 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
395 TrackType::kVideoTrack, std::move(onWaitingForKeyEvent),
396 CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),
397 OptionSet(ownerData.mHardwareDecodingDisabled
398 ? Option::HardwareDecoderNotAllowed
399 : Option::Default),
400 mOwner->mMediaEngineId, mOwner->mTrackingId});
401 break;
404 default:
405 p = PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(
406 NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
409 aData.mLiveToken = MakeRefPtr<Data::DecoderCancelled>();
411 aData.mCreateDecoderPromise = p->Then(
412 mOwner->OwnerThread(), __func__,
413 [this, &aData, &ownerData, live = WeakPtr{aData.mLiveToken},
414 owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)](
415 RefPtr<MediaDataDecoder>&& aDecoder) {
416 if (!live) {
417 return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder),
418 __func__);
420 aData.mLiveToken = nullptr;
421 aData.mDecoder = new MediaDataDecoderProxy(
422 aDecoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));
423 aData.mDecoder = new AllocationWrapper(aData.mDecoder.forget(),
424 aData.mToken.forget());
425 DecoderDoctorLogger::LinkParentAndChild(
426 aData.mDecoder.get(), "decoder",
427 "MediaFormatReader::DecoderFactory", this);
429 DoInitDecoder(aData);
431 return CreateDecoderPromise::CreateAndResolve(aData.mDecoder, __func__);
433 [this, &aData,
434 live = WeakPtr{aData.mLiveToken}](const MediaResult& aError) {
435 NS_WARNING("Error constructing decoders");
436 if (!live) {
437 return CreateDecoderPromise::CreateAndReject(aError, __func__);
439 aData.mLiveToken = nullptr;
440 aData.mToken = nullptr;
441 aData.mStage = Stage::None;
442 aData.mOwnerData.mDescription = aError.Description();
443 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
444 "create_decoder_error", aError);
445 mOwner->NotifyError(aData.mTrack, aError);
447 return CreateDecoderPromise::CreateAndReject(aError, __func__);
451 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {
452 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);
453 auto& ownerData = aData.mOwnerData;
455 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
456 "initialize_decoder", DDNoValue{});
457 aData.mDecoder->Init()
458 ->Then(
459 mOwner->OwnerThread(), __func__,
460 [this, &aData, &ownerData](TrackType aTrack) {
461 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
462 MEDIA_PLAYBACK);
463 aData.mInitRequest.Complete();
464 aData.mStage = Stage::None;
465 MutexAutoLock lock(ownerData.mMutex);
466 ownerData.mDecoder = std::move(aData.mDecoder);
467 ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
468 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
469 DDLogCategory::Log, "decoder_initialized", DDNoValue{});
470 DecoderDoctorLogger::LinkParentAndChild(
471 "MediaFormatReader::DecoderData", &ownerData, "decoder",
472 ownerData.mDecoder.get());
473 mOwner->SetVideoDecodeThreshold();
474 mOwner->ScheduleUpdate(aTrack);
475 if (aTrack == TrackInfo::kVideoTrack) {
476 DecoderBenchmark::CheckVersion(
477 ownerData.GetCurrentInfo()->mMimeType);
479 if (aTrack == TrackInfo::kAudioTrack) {
480 ownerData.mProcessName = ownerData.mDecoder->GetProcessName();
481 ownerData.mCodecName = ownerData.mDecoder->GetCodecName();
484 [this, &aData, &ownerData](const MediaResult& aError) {
485 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
486 MEDIA_PLAYBACK);
487 aData.mInitRequest.Complete();
488 MOZ_RELEASE_ASSERT(!ownerData.mDecoder,
489 "Can't have a decoder already set");
490 aData.mStage = Stage::None;
491 mOwner->mShutdownPromisePool->ShutdownDecoder(
492 aData.mDecoder.forget());
493 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
494 DDLogCategory::Log, "initialize_decoder_error", aError);
495 mOwner->NotifyError(aData.mTrack, aError);
497 ->Track(aData.mInitRequest);
500 // DemuxerProxy ensures that the original main demuxer is only ever accessed
501 // via its own dedicated task queue.
502 // This ensure that the reader's taskqueue will never blocked while a demuxer
503 // is itself blocked attempting to access the MediaCache or the MediaResource.
504 class MediaFormatReader::DemuxerProxy {
505 using TrackType = TrackInfo::TrackType;
506 class Wrapper;
508 public:
509 explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)
510 : mTaskQueue(TaskQueue::Create(
511 GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
512 "DemuxerProxy::mTaskQueue")),
513 mData(new Data(aDemuxer)) {
514 MOZ_COUNT_CTOR(DemuxerProxy);
517 MOZ_COUNTED_DTOR(DemuxerProxy)
519 RefPtr<ShutdownPromise> Shutdown() {
520 RefPtr<Data> data = std::move(mData);
521 return InvokeAsync(mTaskQueue, __func__, [data]() {
522 // We need to clear our reference to the demuxer now. So that in the event
523 // the init promise wasn't resolved, such as what can happen with the
524 // mediasource demuxer that is waiting on more data, it will force the
525 // init promise to be rejected.
526 data->mDemuxer = nullptr;
527 data->mAudioDemuxer = nullptr;
528 data->mVideoDemuxer = nullptr;
529 return ShutdownPromise::CreateAndResolve(true, __func__);
533 RefPtr<MediaDataDemuxer::InitPromise> Init();
535 Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {
536 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
538 switch (aTrack) {
539 case TrackInfo::kAudioTrack:
540 return mData->mAudioDemuxer;
541 case TrackInfo::kVideoTrack:
542 return mData->mVideoDemuxer;
543 default:
544 return nullptr;
548 uint32_t GetNumberTracks(TrackType aTrack) const {
549 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
551 switch (aTrack) {
552 case TrackInfo::kAudioTrack:
553 return mData->mNumAudioTrack;
554 case TrackInfo::kVideoTrack:
555 return mData->mNumVideoTrack;
556 default:
557 return 0;
561 bool IsSeekable() const {
562 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
564 return mData->mSeekable;
567 bool IsSeekableOnlyInBufferedRanges() const {
568 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
570 return mData->mSeekableOnlyInBufferedRange;
573 UniquePtr<EncryptionInfo> GetCrypto() const {
574 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
576 if (!mData->mCrypto) {
577 return nullptr;
579 auto crypto = MakeUnique<EncryptionInfo>();
580 *crypto = *mData->mCrypto;
581 return crypto;
584 RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();
586 bool ShouldComputeStartTime() const {
587 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
589 return mData->mShouldComputeStartTime;
592 private:
593 const RefPtr<TaskQueue> mTaskQueue;
594 struct Data {
595 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)
597 explicit Data(MediaDataDemuxer* aDemuxer)
598 : mInitDone(false), mDemuxer(aDemuxer) {}
600 Atomic<bool> mInitDone;
601 // Only ever accessed over mTaskQueue once.
602 RefPtr<MediaDataDemuxer> mDemuxer;
603 // Only accessed once InitPromise has been resolved and immutable after.
604 // So we can safely access them without the use of the mutex.
605 uint32_t mNumAudioTrack = 0;
606 RefPtr<Wrapper> mAudioDemuxer;
607 uint32_t mNumVideoTrack = 0;
608 RefPtr<Wrapper> mVideoDemuxer;
609 bool mSeekable = false;
610 bool mSeekableOnlyInBufferedRange = false;
611 bool mShouldComputeStartTime = true;
612 UniquePtr<EncryptionInfo> mCrypto;
614 private:
615 ~Data() = default;
617 RefPtr<Data> mData;
620 class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {
621 public:
622 Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)
623 : mMutex("TrackDemuxer Mutex"),
624 mTaskQueue(aTaskQueue),
625 mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),
626 mInfo(aTrackDemuxer->GetInfo()),
627 mTrackDemuxer(aTrackDemuxer) {
628 DecoderDoctorLogger::LogConstructionAndBase(
629 "MediaFormatReader::DemuxerProxy::Wrapper", this,
630 static_cast<const MediaTrackDemuxer*>(this));
631 DecoderDoctorLogger::LinkParentAndChild(
632 "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
633 aTrackDemuxer);
636 UniquePtr<TrackInfo> GetInfo() const override {
637 if (!mInfo) {
638 return nullptr;
640 return mInfo->Clone();
643 RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {
644 RefPtr<Wrapper> self = this;
645 return InvokeAsync(
646 mTaskQueue, __func__,
647 [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })
648 ->Then(
649 mTaskQueue, __func__,
650 [self](const TimeUnit& aTime) {
651 self->UpdateRandomAccessPoint();
652 return SeekPromise::CreateAndResolve(aTime, __func__);
654 [self](const MediaResult& aError) {
655 self->UpdateRandomAccessPoint();
656 return SeekPromise::CreateAndReject(aError, __func__);
660 RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {
661 RefPtr<Wrapper> self = this;
662 return InvokeAsync(mTaskQueue, __func__,
663 [self, aNumSamples]() {
664 return self->mTrackDemuxer->GetSamples(aNumSamples);
666 ->Then(
667 mTaskQueue, __func__,
668 [self](RefPtr<SamplesHolder> aSamples) {
669 self->UpdateRandomAccessPoint();
670 return SamplesPromise::CreateAndResolve(aSamples.forget(),
671 __func__);
673 [self](const MediaResult& aError) {
674 self->UpdateRandomAccessPoint();
675 return SamplesPromise::CreateAndReject(aError, __func__);
679 bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }
681 void Reset() override {
682 RefPtr<Wrapper> self = this;
683 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
684 "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
685 [self]() { self->mTrackDemuxer->Reset(); }));
686 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
687 Unused << rv;
690 nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {
691 MutexAutoLock lock(mMutex);
692 if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {
693 *aTime = mNextRandomAccessPoint;
695 return mNextRandomAccessPointResult;
698 RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
699 const TimeUnit& aTimeThreshold) override {
700 RefPtr<Wrapper> self = this;
701 return InvokeAsync(
702 mTaskQueue, __func__,
703 [self, aTimeThreshold]() {
704 return self->mTrackDemuxer->SkipToNextRandomAccessPoint(
705 aTimeThreshold);
707 ->Then(
708 mTaskQueue, __func__,
709 [self](uint32_t aVal) {
710 self->UpdateRandomAccessPoint();
711 return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);
713 [self](const SkipFailureHolder& aError) {
714 self->UpdateRandomAccessPoint();
715 return SkipAccessPointPromise::CreateAndReject(aError, __func__);
719 TimeIntervals GetBuffered() override {
720 MutexAutoLock lock(mMutex);
721 return mBuffered;
724 void BreakCycles() override {}
726 private:
727 Mutex mMutex MOZ_UNANNOTATED;
728 const RefPtr<TaskQueue> mTaskQueue;
729 const bool mGetSamplesMayBlock;
730 const UniquePtr<TrackInfo> mInfo;
731 // mTrackDemuxer is only ever accessed on demuxer's task queue.
732 RefPtr<MediaTrackDemuxer> mTrackDemuxer;
733 // All following members are protected by mMutex
734 nsresult mNextRandomAccessPointResult = NS_OK;
735 TimeUnit mNextRandomAccessPoint;
736 TimeIntervals mBuffered;
737 friend class DemuxerProxy;
739 ~Wrapper() {
740 RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);
741 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
742 "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
743 [trackDemuxer]() { trackDemuxer->BreakCycles(); }));
744 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
745 Unused << rv;
746 DecoderDoctorLogger::LogDestruction(
747 "MediaFormatReader::DemuxerProxy::Wrapper", this);
750 void UpdateRandomAccessPoint() {
751 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
752 if (!mTrackDemuxer) {
753 // Detached.
754 return;
756 MutexAutoLock lock(mMutex);
757 mNextRandomAccessPointResult =
758 mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);
761 void UpdateBuffered() {
762 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
763 if (!mTrackDemuxer) {
764 // Detached.
765 return;
767 MutexAutoLock lock(mMutex);
768 mBuffered = mTrackDemuxer->GetBuffered();
772 RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {
773 AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);
774 using InitPromise = MediaDataDemuxer::InitPromise;
776 RefPtr<Data> data = mData;
777 RefPtr<TaskQueue> taskQueue = mTaskQueue;
778 return InvokeAsync(mTaskQueue, __func__,
779 [data, taskQueue]() {
780 if (!data->mDemuxer) {
781 return InitPromise::CreateAndReject(
782 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
784 return data->mDemuxer->Init();
786 ->Then(
787 taskQueue, __func__,
788 [data, taskQueue]() {
789 AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);
790 if (!data->mDemuxer) { // Was shutdown.
791 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
792 __func__);
794 data->mNumAudioTrack =
795 data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
796 if (data->mNumAudioTrack) {
797 RefPtr<MediaTrackDemuxer> d =
798 data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
799 if (d) {
800 RefPtr<Wrapper> wrapper =
801 new DemuxerProxy::Wrapper(d, taskQueue);
802 wrapper->UpdateBuffered();
803 data->mAudioDemuxer = wrapper;
804 DecoderDoctorLogger::LinkParentAndChild(
805 data->mDemuxer.get(), "decoder factory wrapper",
806 "MediaFormatReader::DecoderFactory::Wrapper",
807 wrapper.get());
810 data->mNumVideoTrack =
811 data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
812 if (data->mNumVideoTrack) {
813 RefPtr<MediaTrackDemuxer> d =
814 data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
815 if (d) {
816 RefPtr<Wrapper> wrapper =
817 new DemuxerProxy::Wrapper(d, taskQueue);
818 wrapper->UpdateBuffered();
819 data->mVideoDemuxer = wrapper;
820 DecoderDoctorLogger::LinkParentAndChild(
821 data->mDemuxer.get(), "decoder factory wrapper",
822 "MediaFormatReader::DecoderFactory::Wrapper",
823 wrapper.get());
826 data->mCrypto = data->mDemuxer->GetCrypto();
827 data->mSeekable = data->mDemuxer->IsSeekable();
828 data->mSeekableOnlyInBufferedRange =
829 data->mDemuxer->IsSeekableOnlyInBufferedRanges();
830 data->mShouldComputeStartTime =
831 data->mDemuxer->ShouldComputeStartTime();
832 data->mInitDone = true;
833 return InitPromise::CreateAndResolve(NS_OK, __func__);
835 [](const MediaResult& aError) {
836 return InitPromise::CreateAndReject(aError, __func__);
840 RefPtr<MediaFormatReader::NotifyDataArrivedPromise>
841 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
842 RefPtr<Data> data = mData;
843 return InvokeAsync(mTaskQueue, __func__, [data]() {
844 if (!data->mDemuxer) {
845 // Was shutdown.
846 return NotifyDataArrivedPromise::CreateAndReject(
847 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
849 data->mDemuxer->NotifyDataArrived();
850 if (data->mAudioDemuxer) {
851 data->mAudioDemuxer->UpdateBuffered();
853 if (data->mVideoDemuxer) {
854 data->mVideoDemuxer->UpdateBuffered();
856 return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);
860 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
861 MediaDataDemuxer* aDemuxer)
862 : mTaskQueue(
863 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
864 "MediaFormatReader::mTaskQueue",
865 /* aSupportsTailDispatch = */ true)),
866 mAudio(this, MediaData::Type::AUDIO_DATA,
867 StaticPrefs::media_audio_max_decode_error()),
868 mVideo(this, MediaData::Type::VIDEO_DATA,
869 StaticPrefs::media_video_max_decode_error()),
870 mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),
871 mWatchManager(this, OwnerThread()),
872 mIsWatchingWorkingInfo(false),
873 mDemuxer(new DemuxerProxy(aDemuxer)),
874 mDemuxerInitDone(false),
875 mPendingNotifyDataArrived(false),
876 mLastReportedNumDecodedFrames(0),
877 mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),
878 mKnowsCompositor(aInit.mKnowsCompositor),
879 mInitDone(false),
880 mTrackDemuxersMayBlock(false),
881 mSeekScheduled(false),
882 mVideoFrameContainer(aInit.mVideoFrameContainer),
883 mCrashHelper(aInit.mCrashHelper),
884 mDecoderFactory(new DecoderFactory(this)),
885 mShutdownPromisePool(new ShutdownPromisePool()),
886 mBuffered(mTaskQueue, TimeIntervals(),
887 "MediaFormatReader::mBuffered (Canonical)"),
888 mFrameStats(aInit.mFrameStats),
889 mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID),
890 mTrackingId(std::move(aInit.mTrackingId)) {
891 MOZ_ASSERT(aDemuxer);
892 MOZ_COUNT_CTOR(MediaFormatReader);
893 DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
894 &mAudio);
895 DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
896 &mVideo);
897 DDLINKCHILD("demuxer", aDemuxer);
898 mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(
899 mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);
902 MediaFormatReader::~MediaFormatReader() {
903 MOZ_COUNT_DTOR(MediaFormatReader);
904 MOZ_ASSERT(mShutdown);
907 RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
908 MOZ_ASSERT(OnTaskQueue());
909 LOG("");
911 mDemuxerInitRequest.DisconnectIfExists();
912 mNotifyDataArrivedPromise.DisconnectIfExists();
913 mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
914 mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
915 mSkipRequest.DisconnectIfExists();
916 mSetCDMPromise.RejectIfExists(
917 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
918 "MediaFormatReader is shutting down"),
919 __func__);
921 if (mIsWatchingWorkingInfo) {
922 mWatchManager.Unwatch(mWorkingInfoChanged,
923 &MediaFormatReader::NotifyTrackInfoUpdated);
925 mWatchManager.Shutdown();
927 if (mAudio.HasPromise()) {
928 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
930 if (mVideo.HasPromise()) {
931 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
934 if (HasAudio()) {
935 mAudio.ResetDemuxer();
936 mAudio.mTrackDemuxer->BreakCycles();
938 MutexAutoLock lock(mAudio.mMutex);
939 mAudio.mTrackDemuxer = nullptr;
941 mAudio.ResetState();
942 ShutdownDecoder(TrackInfo::kAudioTrack);
945 if (HasVideo()) {
946 mVideo.ResetDemuxer();
947 mVideo.mTrackDemuxer->BreakCycles();
949 MutexAutoLock lock(mVideo.mMutex);
950 mVideo.mTrackDemuxer = nullptr;
952 mVideo.ResetState();
953 ShutdownDecoder(TrackInfo::kVideoTrack);
956 mShutdownPromisePool->Track(mDemuxer->Shutdown());
957 mDemuxer = nullptr;
959 mOnTrackWaitingForKeyListener.Disconnect();
961 mShutdown = true;
962 return mShutdownPromisePool->Shutdown()->Then(
963 OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,
964 &MediaFormatReader::TearDownDecoders);
967 void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {
968 LOGV("%s", TrackTypeToStr(aTrack));
970 // Shut down the pending decoder if any.
971 mDecoderFactory->ShutdownDecoder(aTrack);
973 auto& decoder = GetDecoderData(aTrack);
974 // Flush the decoder if necessary.
975 decoder.Flush();
977 // Shut down the decoder if any.
978 decoder.ShutdownDecoder();
981 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
982 MOZ_ASSERT(OnTaskQueue());
983 if (!StaticPrefs::media_mediacapabilities_from_database()) {
984 return;
986 auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);
987 if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {
988 VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());
989 info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));
990 mOnStoreDecoderBenchmark.Notify(std::move(info));
994 void MediaFormatReader::NotifyTrackInfoUpdated() {
995 MOZ_ASSERT(OnTaskQueue());
996 if (mWorkingInfoChanged) {
997 mWorkingInfoChanged = false;
999 VideoInfo videoInfo;
1000 AudioInfo audioInfo;
1002 MutexAutoLock lock(mVideo.mMutex);
1003 if (HasVideo()) {
1004 videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1008 MutexAutoLock lock(mAudio.mMutex);
1009 if (HasAudio()) {
1010 audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1014 mTrackInfoUpdatedEvent.Notify(videoInfo, audioInfo);
1018 RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
1019 if (mAudio.mTaskQueue) {
1020 mAudio.mTaskQueue->BeginShutdown();
1021 mAudio.mTaskQueue->AwaitShutdownAndIdle();
1022 mAudio.mTaskQueue = nullptr;
1024 if (mVideo.mTaskQueue) {
1025 mVideo.mTaskQueue->BeginShutdown();
1026 mVideo.mTaskQueue->AwaitShutdownAndIdle();
1027 mVideo.mTaskQueue = nullptr;
1030 mDecoderFactory = nullptr;
1031 mVideoFrameContainer = nullptr;
1033 ReleaseResources();
1034 mBuffered.DisconnectAll();
1035 return mTaskQueue->BeginShutdown();
1038 nsresult MediaFormatReader::Init() {
1039 MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1041 mAudio.mTaskQueue =
1042 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1043 "MFR::mAudio::mTaskQueue");
1045 mVideo.mTaskQueue =
1046 TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1047 "MFR::mVideo::mTaskQueue");
1049 return NS_OK;
1052 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {
1053 // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
1054 // and would create new one for specific track in the next Update.
1055 MOZ_ASSERT(OnTaskQueue());
1057 if (mSetCDMPromise.IsEmpty()) {
1058 return true;
1061 MOZ_ASSERT(mCDMProxy);
1062 if (mSetCDMForTracks.contains(aTrack)) {
1063 mSetCDMForTracks -= aTrack;
1066 if (mSetCDMForTracks.isEmpty()) {
1067 LOGV("%s : Done ", __func__);
1068 mSetCDMPromise.Resolve(/* aIgnored = */ true, __func__);
1069 if (HasAudio()) {
1070 ScheduleUpdate(TrackInfo::kAudioTrack);
1072 if (HasVideo()) {
1073 ScheduleUpdate(TrackInfo::kVideoTrack);
1075 return true;
1077 LOGV("%s : %s track is ready.", __func__, TrackTypeToStr(aTrack));
1078 return false;
1081 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack) {
1082 MOZ_ASSERT(OnTaskQueue());
1083 LOGV("%s : %s", __func__, TrackTypeToStr(aTrack));
1085 mSetCDMForTracks += aTrack;
1086 if (mCDMProxy) {
1087 // An old cdm proxy exists, so detaching old cdm proxy by shutting down
1088 // MediaDataDecoder.
1089 ShutdownDecoder(aTrack);
1091 ScheduleUpdate(aTrack);
1094 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack) {
1095 MOZ_ASSERT(OnTaskQueue());
1096 return GetDecoderData(aTrack).IsEncrypted() &&
1097 mSetCDMForTracks.contains(aTrack) && !mCDMProxy;
1100 RefPtr<SetCDMPromise> MediaFormatReader::SetCDMProxy(CDMProxy* aProxy) {
1101 MOZ_ASSERT(OnTaskQueue());
1102 LOGV("SetCDMProxy (%p)", aProxy);
1104 if (mShutdown) {
1105 return SetCDMPromise::CreateAndReject(
1106 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1107 "MediaFormatReader is shutting down"),
1108 __func__);
1111 mSetCDMPromise.RejectIfExists(
1112 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1113 "Another new CDM proxy is being set."),
1114 __func__);
1116 // Shutdown all decoders as switching CDM proxy indicates that it's
1117 // inappropriate for the existing decoders to continue decoding via the old
1118 // CDM proxy.
1119 if (HasAudio()) {
1120 PrepareToSetCDMForTrack(TrackInfo::kAudioTrack);
1122 if (HasVideo()) {
1123 PrepareToSetCDMForTrack(TrackInfo::kVideoTrack);
1126 mCDMProxy = aProxy;
1128 if (!mInitDone || mSetCDMForTracks.isEmpty() || !mCDMProxy) {
1129 // 1) MFR is not initialized yet or
1130 // 2) Demuxer is initialized without active audio and video or
1131 // 3) A null cdm proxy is set
1132 // the promise can be resolved directly.
1133 mSetCDMForTracks.clear();
1134 return SetCDMPromise::CreateAndResolve(/* aIgnored = */ true, __func__);
1137 RefPtr<SetCDMPromise> p = mSetCDMPromise.Ensure(__func__);
1138 return p;
1141 bool MediaFormatReader::IsWaitingOnCDMResource() {
1142 MOZ_ASSERT(OnTaskQueue());
1143 return IsEncrypted() && !mCDMProxy;
1146 RefPtr<MediaFormatReader::MetadataPromise>
1147 MediaFormatReader::AsyncReadMetadata() {
1148 AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK);
1149 MOZ_ASSERT(OnTaskQueue());
1151 MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
1153 if (mInitDone) {
1154 // We are returning from dormant.
1155 MetadataHolder metadata;
1156 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1157 return MetadataPromise::CreateAndResolve(std::move(metadata), __func__);
1160 RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
1162 mDemuxer->Init()
1163 ->Then(OwnerThread(), __func__, this,
1164 &MediaFormatReader::OnDemuxerInitDone,
1165 &MediaFormatReader::OnDemuxerInitFailed)
1166 ->Track(mDemuxerInitRequest);
1167 return p;
1170 void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
1171 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK);
1172 MOZ_ASSERT(OnTaskQueue());
1173 mDemuxerInitRequest.Complete();
1175 if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1176 mMetadataPromise.Reject(aResult, __func__);
1177 return;
1180 mDemuxerInitDone = true;
1182 UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
1184 RefPtr<PDMFactory> platform;
1185 if (!IsWaitingOnCDMResource()) {
1186 platform = new PDMFactory();
1189 // To decode, we need valid video and a place to put it.
1190 bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
1191 GetImageContainer();
1193 if (videoActive) {
1194 // We currently only handle the first video track.
1195 MutexAutoLock lock(mVideo.mMutex);
1196 mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1197 if (!mVideo.mTrackDemuxer) {
1198 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1199 return;
1202 UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
1203 videoActive = videoInfo && videoInfo->IsValid();
1204 if (videoActive) {
1205 if (platform && platform->SupportsMimeType(videoInfo->mMimeType) ==
1206 media::DecodeSupport::Unsupported) {
1207 // We have no decoder for this track. Error.
1208 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1209 return;
1211 mInfo.mVideo = *videoInfo->GetAsVideoInfo();
1212 mVideo.mWorkingInfo = MakeUnique<VideoInfo>(mInfo.mVideo);
1213 for (const MetadataTag& tag : videoInfo->mTags) {
1214 tags->InsertOrUpdate(tag.mKey, tag.mValue);
1216 mWorkingInfoChanged = true;
1217 mVideo.mOriginalInfo = std::move(videoInfo);
1218 mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
1219 } else {
1220 mVideo.mTrackDemuxer->BreakCycles();
1221 mVideo.mTrackDemuxer = nullptr;
1225 bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1226 if (audioActive) {
1227 MutexAutoLock lock(mAudio.mMutex);
1228 mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1229 if (!mAudio.mTrackDemuxer) {
1230 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1231 return;
1234 UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
1235 // We actively ignore audio tracks that we know we can't play.
1236 audioActive =
1237 audioInfo && audioInfo->IsValid() &&
1238 (!platform || platform->SupportsMimeType(audioInfo->mMimeType) !=
1239 media::DecodeSupport::Unsupported);
1241 if (audioActive) {
1242 mInfo.mAudio = *audioInfo->GetAsAudioInfo();
1243 mAudio.mWorkingInfo = MakeUnique<AudioInfo>(mInfo.mAudio);
1244 for (const MetadataTag& tag : audioInfo->mTags) {
1245 tags->InsertOrUpdate(tag.mKey, tag.mValue);
1247 mWorkingInfoChanged = true;
1248 mAudio.mOriginalInfo = std::move(audioInfo);
1249 mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
1250 } else {
1251 mAudio.mTrackDemuxer->BreakCycles();
1252 mAudio.mTrackDemuxer = nullptr;
1256 UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
1257 if (crypto && crypto->IsEncrypted()) {
1258 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1259 for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
1260 mOnEncrypted.Notify(crypto->mInitDatas[i].mInitData,
1261 crypto->mInitDatas[i].mType);
1263 mInfo.mCrypto = *crypto;
1266 auto videoDuration = HasVideo() ? mInfo.mVideo.mDuration : TimeUnit::Zero();
1267 auto audioDuration = HasAudio() ? mInfo.mAudio.mDuration : TimeUnit::Zero();
1269 auto duration = std::max(videoDuration, audioDuration);
1270 if (duration.IsPositive()) {
1271 mInfo.mMetadataDuration = Some(duration);
1274 mInfo.mMediaSeekable = mDemuxer->IsSeekable();
1275 mInfo.mMediaSeekableOnlyInBufferedRanges =
1276 mDemuxer->IsSeekableOnlyInBufferedRanges();
1278 if (!videoActive && !audioActive) {
1279 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1280 return;
1283 mTags = std::move(tags);
1284 mInitDone = true;
1286 // Try to get the start time.
1287 // For MSE case, the start time of each track is assumed to be 0.
1288 // For others, we must demux the first sample to know the start time for each
1289 // track.
1290 if (!mDemuxer->ShouldComputeStartTime()) {
1291 mAudio.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1292 mVideo.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1293 } else {
1294 if (HasAudio()) {
1295 RequestDemuxSamples(TrackInfo::kAudioTrack);
1298 if (HasVideo()) {
1299 RequestDemuxSamples(TrackInfo::kVideoTrack);
1303 if (aResult != NS_OK) {
1304 mOnDecodeWarning.Notify(aResult);
1307 MaybeResolveMetadataPromise();
1310 void MediaFormatReader::MaybeResolveMetadataPromise() {
1311 MOZ_ASSERT(OnTaskQueue());
1313 if ((HasAudio() && mAudio.mFirstDemuxedSampleTime.isNothing()) ||
1314 (HasVideo() && mVideo.mFirstDemuxedSampleTime.isNothing())) {
1315 return;
1318 TimeUnit startTime =
1319 std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
1320 mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
1322 if (!startTime.IsInfinite()) {
1323 mInfo.mStartTime = startTime; // mInfo.mStartTime is initialized to 0.
1326 MetadataHolder metadata;
1327 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1328 metadata.mTags = mTags->Count() ? std::move(mTags) : nullptr;
1330 // We now have all the informations required to calculate the initial buffered
1331 // range.
1332 mHasStartTime = true;
1333 UpdateBuffered();
1335 mWatchManager.Watch(mWorkingInfoChanged,
1336 &MediaFormatReader::NotifyTrackInfoUpdated);
1337 mIsWatchingWorkingInfo = true;
1339 mMetadataPromise.Resolve(std::move(metadata), __func__);
1342 bool MediaFormatReader::IsEncrypted() const {
1343 return (HasAudio() && mAudio.GetCurrentInfo()->mCrypto.IsEncrypted()) ||
1344 (HasVideo() && mVideo.GetCurrentInfo()->mCrypto.IsEncrypted());
1347 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError) {
1348 mDemuxerInitRequest.Complete();
1349 mMetadataPromise.Reject(aError, __func__);
1352 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) {
1353 // Called on the MDSM's TaskQueue.
1355 MutexAutoLock lock(mVideo.mMutex);
1356 if (HasVideo()) {
1357 aInfo->mVideo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1361 MutexAutoLock lock(mAudio.mMutex);
1362 if (HasAudio()) {
1363 aInfo->mAudio = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1364 Maybe<nsCString> audioProcessPerCodecName = GetAudioProcessPerCodec();
1365 if (audioProcessPerCodecName.isSome()) {
1366 Telemetry::ScalarAdd(
1367 Telemetry::ScalarID::MEDIA_AUDIO_PROCESS_PER_CODEC_NAME,
1368 NS_ConvertUTF8toUTF16(*audioProcessPerCodecName), 1);
1374 MediaFormatReader::DecoderData& MediaFormatReader::GetDecoderData(
1375 TrackType aTrack) {
1376 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
1377 aTrack == TrackInfo::kVideoTrack);
1378 if (aTrack == TrackInfo::kAudioTrack) {
1379 return mAudio;
1381 return mVideo;
1384 Maybe<TimeUnit> MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold,
1385 bool aRequestNextVideoKeyFrame) {
1386 MOZ_ASSERT(OnTaskQueue());
1387 MOZ_ASSERT(HasVideo());
1389 if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1390 return Nothing();
1393 // Ensure we have no pending seek going as skip-to-keyframe could return out
1394 // of date information.
1395 if (mVideo.HasInternalSeekPending()) {
1396 return Nothing();
1399 TimeUnit nextKeyframe;
1400 nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
1401 if (NS_FAILED(rv)) {
1402 // Only OggTrackDemuxer with video type gets into here.
1403 // We don't support skip-to-next-frame for this case.
1404 return Nothing();
1407 const bool isNextKeyframeValid =
1408 nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
1409 // If we request the next keyframe, only return times greater than
1410 // aTimeThreshold. Otherwise, data will be already behind the threshold and
1411 // will be eventually discarded somewhere in the media pipeline.
1412 if (aRequestNextVideoKeyFrame && isNextKeyframeValid &&
1413 nextKeyframe > aTimeThreshold) {
1414 return Some(nextKeyframe);
1417 const bool isNextVideoBehindTheThreshold =
1418 (isNextKeyframeValid && nextKeyframe <= aTimeThreshold) ||
1419 GetInternalSeekTargetEndTime() < aTimeThreshold;
1420 return isNextVideoBehindTheThreshold ? Some(aTimeThreshold) : Nothing();
1423 RefPtr<MediaFormatReader::VideoDataPromise> MediaFormatReader::RequestVideoData(
1424 const TimeUnit& aTimeThreshold, bool aRequestNextVideoKeyFrame) {
1425 MOZ_ASSERT(OnTaskQueue());
1426 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
1427 // Requesting video can be done independently from audio, even during audio
1428 // seeking. But it shouldn't happen if we're doing video seek.
1429 if (!IsAudioOnlySeeking()) {
1430 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1431 "No sample requests allowed while seeking");
1432 MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
1433 mVideo.mTimeThreshold.isSome());
1434 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1436 LOGV("RequestVideoData(%" PRId64 "), requestNextKeyFrame=%d",
1437 aTimeThreshold.ToMicroseconds(), aRequestNextVideoKeyFrame);
1439 if (!HasVideo()) {
1440 LOG("called with no video track");
1441 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1442 __func__);
1445 if (IsSeeking()) {
1446 LOG("called mid-seek. Rejecting.");
1447 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1448 __func__);
1451 if (mShutdown) {
1452 NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1453 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1454 __func__);
1457 if (Maybe<TimeUnit> target =
1458 ShouldSkip(aTimeThreshold, aRequestNextVideoKeyFrame)) {
1459 PROFILER_MARKER_UNTYPED("RequestVideoData SkipVideoDemuxToNextKeyFrame",
1460 MEDIA_PLAYBACK);
1461 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1462 SkipVideoDemuxToNextKeyFrame(*target);
1463 return p;
1466 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1467 ScheduleUpdate(TrackInfo::kVideoTrack);
1469 return p;
1472 void MediaFormatReader::OnDemuxFailed(TrackType aTrack,
1473 const MediaResult& aError) {
1474 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK);
1475 MOZ_ASSERT(OnTaskQueue());
1476 LOG("Failed to demux %s, failure:%s",
1477 aTrack == TrackType::kVideoTrack ? "video" : "audio",
1478 aError.ErrorName().get());
1479 auto& decoder = GetDecoderData(aTrack);
1480 decoder.mDemuxRequest.Complete();
1481 switch (aError.Code()) {
1482 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1483 DDLOG(DDLogCategory::Log,
1484 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1485 : "audio_demux_interruption",
1486 aError);
1487 if (!decoder.mWaitingForData) {
1488 decoder.RequestDrain();
1490 NotifyEndOfStream(aTrack);
1491 break;
1492 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1493 DDLOG(DDLogCategory::Log,
1494 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1495 : "audio_demux_interruption",
1496 aError);
1497 if (!decoder.mWaitingForData) {
1498 decoder.RequestDrain();
1500 NotifyWaitingForData(aTrack);
1501 break;
1502 case NS_ERROR_DOM_MEDIA_CANCELED:
1503 DDLOG(DDLogCategory::Log,
1504 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1505 : "audio_demux_interruption",
1506 aError);
1507 if (decoder.HasPromise()) {
1508 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
1510 break;
1511 default:
1512 DDLOG(DDLogCategory::Log,
1513 aTrack == TrackType::kVideoTrack ? "video_demux_error"
1514 : "audio_demux_error",
1515 aError);
1516 NotifyError(aTrack, aError);
1517 break;
1521 void MediaFormatReader::DoDemuxVideo() {
1522 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK);
1523 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1525 DDLOG(DDLogCategory::Log, "video_demuxing", DDNoValue{});
1526 PerformanceRecorder<PlaybackStage> perfRecorder(
1527 MediaStage::RequestDemux,
1528 mVideo.GetCurrentInfo()->GetAsVideoInfo()->mImage.height);
1529 auto p = mVideo.mTrackDemuxer->GetSamples(1);
1531 RefPtr<MediaFormatReader> self = this;
1532 if (mVideo.mFirstDemuxedSampleTime.isNothing()) {
1533 p = p->Then(
1534 OwnerThread(), __func__,
1535 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1536 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1537 MEDIA_PLAYBACK);
1538 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxed",
1539 DDNoValue{});
1540 self->OnFirstDemuxCompleted(TrackInfo::kVideoTrack, aSamples);
1541 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1543 [self](const MediaResult& aError) {
1544 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1545 MEDIA_PLAYBACK);
1546 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxing_error",
1547 aError);
1548 self->OnFirstDemuxFailed(TrackInfo::kVideoTrack, aError);
1549 return SamplesPromise::CreateAndReject(aError, __func__);
1553 p->Then(
1554 OwnerThread(), __func__,
1555 [self, perfRecorder(std::move(perfRecorder))](
1556 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) mutable {
1557 perfRecorder.Record();
1558 self->OnVideoDemuxCompleted(std::move(aSamples));
1560 [self](const MediaResult& aError) { self->OnVideoDemuxFailed(aError); })
1561 ->Track(mVideo.mDemuxRequest);
1564 void MediaFormatReader::OnVideoDemuxCompleted(
1565 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1566 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1567 MEDIA_PLAYBACK);
1568 LOGV("%zu video samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1569 aSamples->GetSamples()[0]->mTrackInfo
1570 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1571 : 0);
1572 DDLOG(DDLogCategory::Log, "video_demuxed_samples",
1573 uint64_t(aSamples->GetSamples().Length()));
1574 mVideo.mDemuxRequest.Complete();
1575 mVideo.mQueuedSamples.AppendElements(aSamples->GetSamples());
1576 ScheduleUpdate(TrackInfo::kVideoTrack);
1579 RefPtr<MediaFormatReader::AudioDataPromise>
1580 MediaFormatReader::RequestAudioData() {
1581 MOZ_ASSERT(OnTaskQueue());
1582 MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
1583 // Requesting audio can be done independently from video, even during video
1584 // seeking. But it shouldn't happen if we're doing audio seek.
1585 if (!IsVideoOnlySeeking()) {
1586 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1587 "No sample requests allowed while seeking");
1588 MOZ_DIAGNOSTIC_ASSERT(!mAudio.mSeekRequest.Exists() ||
1589 mAudio.mTimeThreshold.isSome());
1590 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1592 LOGV("");
1594 if (!HasAudio()) {
1595 LOG("called with no audio track");
1596 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1597 __func__);
1600 if (IsSeeking()) {
1601 LOG("called mid-seek. Rejecting.");
1602 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1603 __func__);
1606 if (mShutdown) {
1607 NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1608 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1609 __func__);
1612 RefPtr<AudioDataPromise> p = mAudio.EnsurePromise(__func__);
1613 ScheduleUpdate(TrackInfo::kAudioTrack);
1615 return p;
1618 void MediaFormatReader::DoDemuxAudio() {
1619 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK);
1620 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1622 DDLOG(DDLogCategory::Log, "audio_demuxing", DDNoValue{});
1623 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestDemux);
1624 auto p = mAudio.mTrackDemuxer->GetSamples(1);
1626 RefPtr<MediaFormatReader> self = this;
1627 if (mAudio.mFirstDemuxedSampleTime.isNothing()) {
1628 p = p->Then(
1629 OwnerThread(), __func__,
1630 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1631 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1632 MEDIA_PLAYBACK);
1633 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxed",
1634 DDNoValue{});
1635 self->OnFirstDemuxCompleted(TrackInfo::kAudioTrack, aSamples);
1636 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1638 [self](const MediaResult& aError) {
1639 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1640 MEDIA_PLAYBACK);
1641 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxing_error",
1642 aError);
1643 self->OnFirstDemuxFailed(TrackInfo::kAudioTrack, aError);
1644 return SamplesPromise::CreateAndReject(aError, __func__);
1648 p->Then(
1649 OwnerThread(), __func__,
1650 [self, perfRecorder(std::move(perfRecorder))](
1651 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) mutable {
1652 perfRecorder.Record();
1653 self->OnAudioDemuxCompleted(std::move(aSamples));
1655 [self](const MediaResult& aError) { self->OnAudioDemuxFailed(aError); })
1656 ->Track(mAudio.mDemuxRequest);
1659 void MediaFormatReader::OnAudioDemuxCompleted(
1660 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1661 LOGV("%zu audio samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1662 aSamples->GetSamples()[0]->mTrackInfo
1663 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1664 : 0);
1665 DDLOG(DDLogCategory::Log, "audio_demuxed_samples",
1666 uint64_t(aSamples->GetSamples().Length()));
1667 mAudio.mDemuxRequest.Complete();
1668 mAudio.mQueuedSamples.AppendElements(aSamples->GetSamples());
1669 ScheduleUpdate(TrackInfo::kAudioTrack);
1672 void MediaFormatReader::NotifyNewOutput(
1673 TrackType aTrack, MediaDataDecoder::DecodedData&& aResults) {
1674 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK);
1675 MOZ_ASSERT(OnTaskQueue());
1676 auto& decoder = GetDecoderData(aTrack);
1677 if (aResults.IsEmpty()) {
1678 DDLOG(DDLogCategory::Log,
1679 aTrack == TrackInfo::kAudioTrack ? "decoded_audio" : "decoded_video",
1680 "no output samples");
1681 } else
1682 for (auto&& sample : aResults) {
1683 if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1684 switch (sample->mType) {
1685 case MediaData::Type::AUDIO_DATA:
1686 DDLOGPR(DDLogCategory::Log,
1687 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1688 : "decoded_got_audio!?",
1689 "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1690 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1691 ", \"duration_us\":%" PRIi64 ", \"frames\":%" PRIu32
1692 ", \"channels\":%" PRIu32 ", \"rate\":%" PRIu32
1693 ", \"bytes\":%zu}",
1694 sample->mOffset, sample->mTime.ToMicroseconds(),
1695 sample->mTimecode.ToMicroseconds(),
1696 sample->mDuration.ToMicroseconds(),
1697 sample->As<AudioData>()->Frames(),
1698 sample->As<AudioData>()->mChannels,
1699 sample->As<AudioData>()->mRate,
1700 sample->As<AudioData>()->Data().Length());
1701 break;
1702 case MediaData::Type::VIDEO_DATA:
1703 DDLOGPR(DDLogCategory::Log,
1704 aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1705 : "decoded_got_video!?",
1706 "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1707 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1708 ", \"duration_us\":%" PRIi64
1709 ", \"kf\":%s, \"size\":[%" PRIi32 ",%" PRIi32 "]}",
1710 sample->mOffset, sample->mTime.ToMicroseconds(),
1711 sample->mTimecode.ToMicroseconds(),
1712 sample->mDuration.ToMicroseconds(),
1713 sample->mKeyframe ? "true" : "false",
1714 sample->As<VideoData>()->mDisplay.width,
1715 sample->As<VideoData>()->mDisplay.height);
1716 break;
1717 case MediaData::Type::RAW_DATA:
1718 DDLOGPR(DDLogCategory::Log,
1719 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1720 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1721 : "decoded_?",
1722 "{\"type\":\"RawData\", \"offset\":%" PRIi64
1723 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1724 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1725 sample->mOffset, sample->mTime.ToMicroseconds(),
1726 sample->mTimecode.ToMicroseconds(),
1727 sample->mDuration.ToMicroseconds(),
1728 sample->mKeyframe ? "true" : "false");
1729 break;
1730 case MediaData::Type::NULL_DATA:
1731 DDLOGPR(DDLogCategory::Log,
1732 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1733 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1734 : "decoded_?",
1735 "{\"type\":\"NullData\", \"offset\":%" PRIi64
1736 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1737 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1738 sample->mOffset, sample->mTime.ToMicroseconds(),
1739 sample->mTimecode.ToMicroseconds(),
1740 sample->mDuration.ToMicroseconds(),
1741 sample->mKeyframe ? "true" : "false");
1742 break;
1745 LOGV("Received new %s sample time:%" PRId64 " duration:%" PRId64,
1746 TrackTypeToStr(aTrack), sample->mTime.ToMicroseconds(),
1747 sample->mDuration.ToMicroseconds());
1748 decoder.mOutput.AppendElement(sample);
1749 decoder.mNumSamplesOutput++;
1750 decoder.mNumOfConsecutiveDecodingError = 0;
1751 decoder.mNumOfConsecutiveRDDOrGPUCrashes = 0;
1752 if (aTrack == TrackInfo::kAudioTrack) {
1753 decoder.mNumOfConsecutiveUtilityCrashes = 0;
1756 LOG("Done processing new %s samples", TrackTypeToStr(aTrack));
1758 if (!aResults.IsEmpty()) {
1759 // We have decoded our first frame, we can now starts to skip future errors.
1760 decoder.mFirstFrameTime.reset();
1762 ScheduleUpdate(aTrack);
1765 void MediaFormatReader::NotifyError(TrackType aTrack,
1766 const MediaResult& aError) {
1767 MOZ_ASSERT(OnTaskQueue());
1768 NS_WARNING(aError.Description().get());
1769 LOGV("%s Decoding error", TrackTypeToStr(aTrack));
1770 auto& decoder = GetDecoderData(aTrack);
1771 decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
1773 ScheduleUpdate(aTrack);
1776 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack) {
1777 MOZ_ASSERT(OnTaskQueue());
1778 auto& decoder = GetDecoderData(aTrack);
1779 decoder.mWaitingForData = true;
1780 if (decoder.mTimeThreshold) {
1781 decoder.mTimeThreshold.ref().mWaiting = true;
1783 ScheduleUpdate(aTrack);
1786 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack) {
1787 MOZ_ASSERT(OnTaskQueue());
1788 auto& decoder = GetDecoderData(aTrack);
1789 mOnWaitingForKey.Notify();
1790 if (!decoder.mDecodeRequest.Exists()) {
1791 LOGV("WaitingForKey received while no pending decode. Ignoring");
1792 return;
1794 decoder.mWaitingForKey = true;
1795 ScheduleUpdate(aTrack);
1798 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack) {
1799 MOZ_ASSERT(OnTaskQueue());
1800 auto& decoder = GetDecoderData(aTrack);
1801 decoder.mDemuxEOS = true;
1802 ScheduleUpdate(aTrack);
1805 bool MediaFormatReader::NeedInput(DecoderData& aDecoder) {
1806 // The decoder will not be fed a new raw sample until the current decoding
1807 // requests has completed.
1808 return (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
1809 !aDecoder.HasPendingDrain() && !aDecoder.HasFatalError() &&
1810 !aDecoder.mDemuxRequest.Exists() && !aDecoder.mOutput.Length() &&
1811 !aDecoder.HasInternalSeekPending() &&
1812 !aDecoder.mDecodeRequest.Exists();
1815 void MediaFormatReader::ScheduleUpdate(TrackType aTrack) {
1816 MOZ_ASSERT(OnTaskQueue());
1817 if (mShutdown) {
1818 return;
1820 auto& decoder = GetDecoderData(aTrack);
1821 MOZ_RELEASE_ASSERT(decoder.GetCurrentInfo(),
1822 "Can only schedule update when track exists");
1824 if (decoder.mUpdateScheduled) {
1825 return;
1827 LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
1828 decoder.mUpdateScheduled = true;
1829 RefPtr<nsIRunnable> task(NewRunnableMethod<TrackType>(
1830 "MediaFormatReader::Update", this, &MediaFormatReader::Update, aTrack));
1831 nsresult rv = OwnerThread()->Dispatch(task.forget());
1832 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1833 Unused << rv;
1836 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) {
1837 MOZ_ASSERT(OnTaskQueue());
1838 auto& decoder = GetDecoderData(aTrack);
1840 if (!decoder.mReceivedNewData) {
1841 return false;
1844 // We do not want to clear mWaitingForData while there are pending
1845 // demuxing or seeking operations that could affect the value of this flag.
1846 // This is in order to ensure that we will retry once they complete as we may
1847 // now have new data that could potentially allow those operations to
1848 // successfully complete if tried again.
1849 if (decoder.mSeekRequest.Exists()) {
1850 // Nothing more to do until this operation complete.
1851 return true;
1854 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
1855 LOGV("Skipping in progress, nothing more to do");
1856 return true;
1859 if (decoder.mDemuxRequest.Exists()) {
1860 // We may have pending operations to process, so we want to continue
1861 // after UpdateReceivedNewData returns.
1862 return false;
1865 if (decoder.HasPendingDrain()) {
1866 // We do not want to clear mWaitingForData or mDemuxEOS while
1867 // a drain is in progress in order to properly complete the operation.
1868 return false;
1871 decoder.mReceivedNewData = false;
1872 if (decoder.mTimeThreshold) {
1873 decoder.mTimeThreshold.ref().mWaiting = false;
1875 decoder.mWaitingForData = false;
1877 if (decoder.HasFatalError()) {
1878 return false;
1881 if (!mSeekPromise.IsEmpty() &&
1882 (!IsVideoOnlySeeking() || aTrack == TrackInfo::kVideoTrack)) {
1883 MOZ_ASSERT(!decoder.HasPromise());
1884 MOZ_DIAGNOSTIC_ASSERT(
1885 (IsVideoOnlySeeking() || !mAudio.mTimeThreshold) &&
1886 !mVideo.mTimeThreshold,
1887 "InternalSeek must have been aborted when Seek was first called");
1888 MOZ_DIAGNOSTIC_ASSERT(
1889 (IsVideoOnlySeeking() || !mAudio.HasWaitingPromise()) &&
1890 !mVideo.HasWaitingPromise(),
1891 "Waiting promises must have been rejected when Seek was first called");
1892 if (mVideo.mSeekRequest.Exists() ||
1893 (!IsVideoOnlySeeking() && mAudio.mSeekRequest.Exists())) {
1894 // Already waiting for a seek to complete. Nothing more to do.
1895 return true;
1897 LOG("Attempting Seek");
1898 ScheduleSeek();
1899 return true;
1901 if (decoder.HasInternalSeekPending() || decoder.HasWaitingPromise()) {
1902 if (decoder.HasInternalSeekPending()) {
1903 LOG("Attempting Internal Seek");
1904 InternalSeek(aTrack, decoder.mTimeThreshold.ref());
1906 if (decoder.HasWaitingPromise() && !decoder.IsWaitingForKey() &&
1907 !decoder.IsWaitingForData()) {
1908 MOZ_ASSERT(!decoder.HasPromise());
1909 LOG("We have new data. Resolving WaitingPromise");
1910 decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
1912 return true;
1914 return false;
1917 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack) {
1918 MOZ_ASSERT(OnTaskQueue());
1919 auto& decoder = GetDecoderData(aTrack);
1920 MOZ_ASSERT(!decoder.mDemuxRequest.Exists());
1922 if (!decoder.mQueuedSamples.IsEmpty()) {
1923 // No need to demux new samples.
1924 return;
1927 if (decoder.mDemuxEOS) {
1928 // Nothing left to demux.
1929 // We do not want to attempt to demux while in waiting for data mode
1930 // as it would retrigger an unnecessary drain.
1931 return;
1934 LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
1935 if (aTrack == TrackInfo::kVideoTrack) {
1936 DoDemuxVideo();
1937 } else {
1938 DoDemuxAudio();
1942 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
1943 MediaRawData* aSample) {
1944 MOZ_ASSERT(OnTaskQueue());
1945 auto& decoder = GetDecoderData(aTrack);
1946 RefPtr<MediaFormatReader> self = this;
1947 decoder.mFlushed = false;
1948 DDLOGPR(DDLogCategory::Log,
1949 aTrack == TrackInfo::kAudioTrack ? "decode_audio"
1950 : aTrack == TrackInfo::kVideoTrack ? "decode_video"
1951 : "decode_?",
1952 "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1953 ", \"bytes\":%zu, \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1954 ", \"duration_us\":%" PRIi64 ",%s%s}",
1955 aSample->mOffset, aSample->Size(), aSample->mTime.ToMicroseconds(),
1956 aSample->mTimecode.ToMicroseconds(),
1957 aSample->mDuration.ToMicroseconds(), aSample->mKeyframe ? " kf" : "",
1958 aSample->mEOS ? " eos" : "");
1960 const int32_t height =
1961 aTrack == TrackInfo::kVideoTrack
1962 ? decoder.GetCurrentInfo()->GetAsVideoInfo()->mImage.height
1963 : 0;
1964 MediaInfoFlag flag = MediaInfoFlag::None;
1965 flag |=
1966 aSample->mKeyframe ? MediaInfoFlag::KeyFrame : MediaInfoFlag::NonKeyFrame;
1967 if (aTrack == TrackInfo::kVideoTrack) {
1968 flag |= VideoIsHardwareAccelerated() ? MediaInfoFlag::HardwareDecoding
1969 : MediaInfoFlag::SoftwareDecoding;
1970 const nsCString& mimeType = decoder.GetCurrentInfo()->mMimeType;
1971 if (MP4Decoder::IsH264(mimeType)) {
1972 flag |= MediaInfoFlag::VIDEO_H264;
1973 } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP8)) {
1974 flag |= MediaInfoFlag::VIDEO_VP8;
1975 } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP9)) {
1976 flag |= MediaInfoFlag::VIDEO_VP9;
1978 #ifdef MOZ_AV1
1979 else if (AOMDecoder::IsAV1(mimeType)) {
1980 flag |= MediaInfoFlag::VIDEO_AV1;
1982 #endif
1984 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestDecode,
1985 height, flag);
1986 if (mMediaEngineId && aSample->mCrypto.IsEncrypted()) {
1987 aSample->mShouldCopyCryptoToRemoteRawData = true;
1989 decoder.mDecoder->Decode(aSample)
1990 ->Then(
1991 mTaskQueue, __func__,
1992 [self, aTrack, &decoder, perfRecorder(std::move(perfRecorder))](
1993 MediaDataDecoder::DecodedData&& aResults) mutable {
1994 perfRecorder.Record();
1995 decoder.mDecodeRequest.Complete();
1996 self->NotifyNewOutput(aTrack, std::move(aResults));
1998 [self, aTrack, &decoder](const MediaResult& aError) {
1999 decoder.mDecodeRequest.Complete();
2000 self->NotifyError(aTrack, aError);
2002 ->Track(decoder.mDecodeRequest);
2005 void MediaFormatReader::HandleDemuxedSamples(
2006 TrackType aTrack, FrameStatistics::AutoNotifyDecoded& aA) {
2007 MOZ_ASSERT(OnTaskQueue());
2009 auto& decoder = GetDecoderData(aTrack);
2011 if (decoder.mFlushing) {
2012 LOGV("Decoder operation in progress, let it complete.");
2013 return;
2016 if (decoder.mQueuedSamples.IsEmpty()) {
2017 return;
2020 RefPtr<MediaRawData> sample = decoder.mQueuedSamples[0];
2021 const RefPtr<TrackInfoSharedPtr> info = sample->mTrackInfo;
2023 if (info && decoder.mLastStreamSourceID != info->GetID()) {
2024 nsTArray<RefPtr<MediaRawData>> samples;
2025 if (decoder.mDecoder) {
2026 bool recyclable =
2027 StaticPrefs::media_decoder_recycle_enabled() &&
2028 decoder.mDecoder->SupportDecoderRecycling() &&
2029 (*info)->mCrypto.mCryptoScheme ==
2030 decoder.GetCurrentInfo()->mCrypto.mCryptoScheme &&
2031 (*info)->mMimeType == decoder.GetCurrentInfo()->mMimeType;
2032 if (!recyclable && decoder.mTimeThreshold.isNothing() &&
2033 (decoder.mNextStreamSourceID.isNothing() ||
2034 decoder.mNextStreamSourceID.ref() != info->GetID())) {
2035 LOG("%s stream id has changed from:%d to:%d, draining decoder.",
2036 TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
2037 decoder.RequestDrain();
2038 decoder.mNextStreamSourceID = Some(info->GetID());
2039 ScheduleUpdate(aTrack);
2040 return;
2043 // If flushing is required, it will clear our array of queued samples.
2044 // So we may need to make a copy.
2045 samples = decoder.mQueuedSamples.Clone();
2046 if (!recyclable) {
2047 LOG("Decoder does not support recycling, recreate decoder.");
2048 ShutdownDecoder(aTrack);
2049 // We're going to be using a new decoder following the change of content
2050 // We can attempt to use hardware decoding again.
2051 decoder.mHardwareDecodingDisabled = false;
2052 } else if (decoder.HasWaitingPromise()) {
2053 decoder.Flush();
2057 nsPrintfCString markerString(
2058 "%s stream id changed from:%" PRIu32 " to:%" PRIu32,
2059 TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
2060 PROFILER_MARKER_TEXT("StreamID Change", MEDIA_PLAYBACK, {}, markerString);
2061 LOG("%s", markerString.get());
2063 if (aTrack == TrackInfo::kVideoTrack) {
2064 // We are about to create a new decoder thus the benchmark,
2065 // up to this point, is stored.
2066 NotifyDecoderBenchmarkStore();
2068 decoder.mNextStreamSourceID.reset();
2069 decoder.mLastStreamSourceID = info->GetID();
2070 decoder.mInfo = info;
2072 MutexAutoLock lock(decoder.mMutex);
2073 if (aTrack == TrackInfo::kAudioTrack) {
2074 decoder.mWorkingInfo = MakeUnique<AudioInfo>(*info->GetAsAudioInfo());
2075 } else if (aTrack == TrackInfo::kVideoTrack) {
2076 decoder.mWorkingInfo = MakeUnique<VideoInfo>(*info->GetAsVideoInfo());
2078 mWorkingInfoChanged = true;
2081 decoder.mMeanRate.Reset();
2083 if (sample->mKeyframe) {
2084 if (samples.Length()) {
2085 decoder.mQueuedSamples = std::move(samples);
2087 } else {
2088 auto time = TimeInterval(sample->mTime, sample->GetEndTime());
2089 InternalSeekTarget seekTarget =
2090 decoder.mTimeThreshold.refOr(InternalSeekTarget(time, false));
2091 LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64,
2092 sample->mTime.ToMicroseconds());
2093 InternalSeek(aTrack, seekTarget);
2094 return;
2098 // Calculate the average frame rate. The first frame will be accounted
2099 // for twice.
2100 decoder.mMeanRate.Update(sample->mDuration);
2102 if (!decoder.mDecoder) {
2103 mDecoderFactory->CreateDecoder(aTrack);
2104 return;
2107 LOGV("Input:%" PRId64 " (dts:%" PRId64 " kf:%d)",
2108 sample->mTime.ToMicroseconds(), sample->mTimecode.ToMicroseconds(),
2109 sample->mKeyframe);
2110 decoder.mNumSamplesInput++;
2111 decoder.mSizeOfQueue++;
2112 if (aTrack == TrackInfo::kVideoTrack) {
2113 aA.mStats.mParsedFrames++;
2116 DecodeDemuxedSamples(aTrack, sample);
2118 decoder.mQueuedSamples.RemoveElementAt(0);
2121 media::TimeUnit MediaFormatReader::GetInternalSeekTargetEndTime() const {
2122 MOZ_ASSERT(OnTaskQueue());
2123 return mVideo.mTimeThreshold ? mVideo.mTimeThreshold->EndTime()
2124 : TimeUnit::FromInfinity();
2127 void MediaFormatReader::InternalSeek(TrackType aTrack,
2128 const InternalSeekTarget& aTarget) {
2129 MOZ_ASSERT(OnTaskQueue());
2130 LOG("%s internal seek to %f", TrackTypeToStr(aTrack),
2131 aTarget.Time().ToSeconds());
2133 auto& decoder = GetDecoderData(aTrack);
2134 decoder.Flush();
2135 decoder.ResetDemuxer();
2136 decoder.mTimeThreshold = Some(aTarget);
2137 DDLOG(DDLogCategory::Log, "seeking", DDNoValue{});
2138 RefPtr<MediaFormatReader> self = this;
2139 decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().Time())
2140 ->Then(
2141 OwnerThread(), __func__,
2142 [self, aTrack](TimeUnit aTime) {
2143 DDLOGEX(self.get(), DDLogCategory::Log, "seeked", DDNoValue{});
2144 auto& decoder = self->GetDecoderData(aTrack);
2145 decoder.mSeekRequest.Complete();
2146 MOZ_ASSERT(decoder.mTimeThreshold,
2147 "Seek promise must be disconnected when "
2148 "timethreshold is reset");
2149 decoder.mTimeThreshold.ref().mHasSeeked = true;
2150 self->SetVideoDecodeThreshold();
2151 self->ScheduleUpdate(aTrack);
2153 [self, aTrack](const MediaResult& aError) {
2154 auto& decoder = self->GetDecoderData(aTrack);
2155 decoder.mSeekRequest.Complete();
2156 switch (aError.Code()) {
2157 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2158 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2159 aError);
2160 self->NotifyWaitingForData(aTrack);
2161 break;
2162 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2163 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2164 aError);
2165 decoder.mTimeThreshold.reset();
2166 self->NotifyEndOfStream(aTrack);
2167 break;
2168 case NS_ERROR_DOM_MEDIA_CANCELED:
2169 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2170 aError);
2171 decoder.mTimeThreshold.reset();
2172 break;
2173 default:
2174 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_error",
2175 aError);
2176 decoder.mTimeThreshold.reset();
2177 self->NotifyError(aTrack, aError);
2178 break;
2181 ->Track(decoder.mSeekRequest);
2184 void MediaFormatReader::DrainDecoder(TrackType aTrack) {
2185 AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK);
2186 MOZ_ASSERT(OnTaskQueue());
2188 auto& decoder = GetDecoderData(aTrack);
2189 if (decoder.mDrainState == DrainState::Draining) {
2190 return;
2192 if (!decoder.mDecoder ||
2193 (decoder.mDrainState != DrainState::PartialDrainPending &&
2194 decoder.mNumSamplesInput == decoder.mNumSamplesOutput)) {
2195 // No frames to drain.
2196 LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack));
2197 decoder.mDrainState = DrainState::DrainAborted;
2198 ScheduleUpdate(aTrack);
2199 return;
2202 decoder.mDrainState = DrainState::Draining;
2204 DDLOG(DDLogCategory::Log, "draining", DDNoValue{});
2205 RefPtr<MediaFormatReader> self = this;
2206 decoder.mDecoder->Drain()
2207 ->Then(
2208 mTaskQueue, __func__,
2209 [self, aTrack, &decoder](MediaDataDecoder::DecodedData&& aResults) {
2210 decoder.mDrainRequest.Complete();
2211 DDLOGEX(self.get(), DDLogCategory::Log, "drained", DDNoValue{});
2212 if (aResults.IsEmpty()) {
2213 decoder.mDrainState = DrainState::DrainCompleted;
2214 } else {
2215 self->NotifyNewOutput(aTrack, std::move(aResults));
2216 // Let's see if we have any more data available to drain.
2217 decoder.mDrainState = DrainState::PartialDrainPending;
2219 self->ScheduleUpdate(aTrack);
2221 [self, aTrack, &decoder](const MediaResult& aError) {
2222 decoder.mDrainRequest.Complete();
2223 DDLOGEX(self.get(), DDLogCategory::Log, "draining_error", aError);
2224 self->NotifyError(aTrack, aError);
2226 ->Track(decoder.mDrainRequest);
2227 LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack));
2230 void MediaFormatReader::Update(TrackType aTrack) {
2231 AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK);
2232 MOZ_ASSERT(OnTaskQueue());
2234 if (mShutdown) {
2235 return;
2238 LOGV("Processing update for %s", TrackTypeToStr(aTrack));
2240 bool needOutput = false;
2241 auto& decoder = GetDecoderData(aTrack);
2242 decoder.mUpdateScheduled = false;
2244 if (!mInitDone) {
2245 return;
2248 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
2249 LOGV("Skipping in progress, nothing more to do");
2250 return;
2253 if (UpdateReceivedNewData(aTrack)) {
2254 LOGV("Nothing more to do");
2255 return;
2258 if (decoder.mSeekRequest.Exists()) {
2259 LOGV("Seeking hasn't completed, nothing more to do");
2260 return;
2263 MOZ_DIAGNOSTIC_ASSERT(
2264 !decoder.HasInternalSeekPending() ||
2265 (!decoder.mOutput.Length() && !decoder.mQueuedSamples.Length()),
2266 "No frames can be demuxed or decoded while an internal seek is pending");
2268 // Record number of frames decoded and parsed. Automatically update the
2269 // stats counters using the AutoNotifyDecoded stack-based class.
2270 FrameStatistics::AutoNotifyDecoded a(mFrameStats);
2272 // Drop any frames found prior our internal seek target.
2273 while (decoder.mTimeThreshold && decoder.mOutput.Length()) {
2274 RefPtr<MediaData>& output = decoder.mOutput[0];
2275 InternalSeekTarget target = decoder.mTimeThreshold.ref();
2276 auto time = output->mTime;
2277 if (time >= target.Time()) {
2278 // We have reached our internal seek target.
2279 decoder.mTimeThreshold.reset();
2280 // We might have dropped some keyframes.
2281 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2283 if (time < target.Time() || (target.mDropTarget && target.Contains(time))) {
2284 LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2285 TrackTypeToStr(aTrack), output->mTime.ToSeconds(),
2286 target.Time().ToSeconds(), output->mKeyframe);
2287 decoder.mOutput.RemoveElementAt(0);
2288 decoder.mSizeOfQueue -= 1;
2292 while (decoder.mOutput.Length() &&
2293 decoder.mOutput[0]->mType == MediaData::Type::NULL_DATA) {
2294 LOGV("Dropping null data. Time: %" PRId64,
2295 decoder.mOutput[0]->mTime.ToMicroseconds());
2296 decoder.mOutput.RemoveElementAt(0);
2297 decoder.mSizeOfQueue -= 1;
2300 if (decoder.HasPromise()) {
2301 needOutput = true;
2302 if (decoder.mOutput.Length()) {
2303 RefPtr<MediaData> output = decoder.mOutput[0];
2304 decoder.mOutput.RemoveElementAt(0);
2305 decoder.mSizeOfQueue -= 1;
2306 decoder.mLastDecodedSampleTime =
2307 Some(TimeInterval(output->mTime, output->GetEndTime()));
2308 decoder.mNumSamplesOutputTotal++;
2309 ReturnOutput(output, aTrack);
2310 // We have a decoded sample ready to be returned.
2311 if (aTrack == TrackType::kVideoTrack) {
2312 uint64_t delta =
2313 decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
2314 a.mStats.mDecodedFrames = static_cast<uint32_t>(delta);
2315 mLastReportedNumDecodedFrames = decoder.mNumSamplesOutputTotal;
2316 if (output->mKeyframe) {
2317 if (mPreviousDecodedKeyframeTime_us <
2318 output->mTime.ToMicroseconds()) {
2319 // There is a previous keyframe -> Record inter-keyframe stats.
2320 uint64_t segment_us = output->mTime.ToMicroseconds() -
2321 mPreviousDecodedKeyframeTime_us;
2322 a.mStats.mInterKeyframeSum_us += segment_us;
2323 a.mStats.mInterKeyframeCount += 1;
2324 if (a.mStats.mInterKeyFrameMax_us < segment_us) {
2325 a.mStats.mInterKeyFrameMax_us = segment_us;
2328 mPreviousDecodedKeyframeTime_us = output->mTime.ToMicroseconds();
2330 bool wasHardwareAccelerated = mVideo.mIsHardwareAccelerated;
2331 nsCString error;
2332 mVideo.mIsHardwareAccelerated =
2333 mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
2334 VideoData* videoData = output->As<VideoData>();
2335 if (!mVideo.mHasReportedVideoHardwareSupportTelemtry ||
2336 wasHardwareAccelerated != mVideo.mIsHardwareAccelerated) {
2337 mVideo.mHasReportedVideoHardwareSupportTelemtry = true;
2338 Telemetry::ScalarSet(
2339 Telemetry::ScalarID::MEDIA_VIDEO_HARDWARE_DECODING_SUPPORT,
2340 NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2341 !!mVideo.mIsHardwareAccelerated);
2342 static constexpr gfx::IntSize HD_VIDEO_SIZE{1280, 720};
2343 if (videoData->mDisplay.width >= HD_VIDEO_SIZE.Width() &&
2344 videoData->mDisplay.height >= HD_VIDEO_SIZE.Height()) {
2345 Telemetry::ScalarSet(
2346 Telemetry::ScalarID::MEDIA_VIDEO_HD_HARDWARE_DECODING_SUPPORT,
2347 NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2348 !!mVideo.mIsHardwareAccelerated);
2351 #ifdef XP_WIN
2352 // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2353 // of GPU RAM used.
2354 mVideo.mIsHardwareAccelerated =
2355 mVideo.mIsHardwareAccelerated ||
2356 (videoData->mImage &&
2357 videoData->mImage->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE);
2358 #endif
2360 } else if (decoder.HasFatalError()) {
2361 nsCString mimeType = decoder.GetCurrentInfo()->mMimeType;
2362 if (!mimeType.IsEmpty()) {
2363 Telemetry::ScalarAdd(
2364 Telemetry::ScalarID::MEDIA_DECODE_ERROR_PER_MIME_TYPE,
2365 NS_ConvertUTF8toUTF16(mimeType), 1 /* error count */);
2367 LOG("Rejecting %s promise for %s : DECODE_ERROR", TrackTypeToStr(aTrack),
2368 mimeType.get());
2369 decoder.RejectPromise(decoder.mError.ref(), __func__);
2370 return;
2371 } else if (decoder.HasCompletedDrain()) {
2372 if (decoder.mDemuxEOS) {
2373 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2374 if (aTrack == TrackInfo::kVideoTrack) {
2375 // End of video, store the benchmark of the decoder.
2376 NotifyDecoderBenchmarkStore();
2378 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2379 } else if (decoder.mWaitingForData) {
2380 if (decoder.mDrainState == DrainState::DrainCompleted &&
2381 decoder.mLastDecodedSampleTime && !decoder.mNextStreamSourceID) {
2382 // We have completed draining the decoder following WaitingForData.
2383 // Set up the internal seek machinery to be able to resume from the
2384 // last sample decoded.
2385 LOG("Seeking to last sample time: %" PRId64,
2386 decoder.mLastDecodedSampleTime.ref().mStart.ToMicroseconds());
2387 InternalSeek(aTrack, InternalSeekTarget(
2388 decoder.mLastDecodedSampleTime.ref(), true));
2390 if (!decoder.mReceivedNewData) {
2391 LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack));
2392 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2396 decoder.mDrainState = DrainState::None;
2398 // Now that draining has completed, we check if we have received
2399 // new data again as the result may now be different from the earlier
2400 // run.
2401 if (UpdateReceivedNewData(aTrack) || decoder.mSeekRequest.Exists()) {
2402 LOGV("Nothing more to do");
2403 return;
2405 } else if (decoder.mDemuxEOS && !decoder.HasPendingDrain() &&
2406 decoder.mQueuedSamples.IsEmpty()) {
2407 // It is possible to transition from WAITING_FOR_DATA directly to EOS
2408 // state during the internal seek; in which case no draining would occur.
2409 // There is no more samples left to be decoded and we are already in
2410 // EOS state. We can immediately reject the data promise.
2411 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2412 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2413 } else if (decoder.mWaitingForKey) {
2414 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2415 TrackTypeToStr(aTrack));
2416 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2417 } else if (IsDecoderWaitingForCDM(aTrack)) {
2418 // Rejecting the promise could lead to entering buffering state for MDSM,
2419 // once a qualified(with the same key system and sessions created by the
2420 // same InitData) new cdm proxy is set, decoding can be resumed.
2421 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2422 TrackTypeToStr(aTrack));
2423 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2427 if (decoder.mDrainState == DrainState::DrainRequested ||
2428 decoder.mDrainState == DrainState::PartialDrainPending) {
2429 if (decoder.mOutput.IsEmpty()) {
2430 DrainDecoder(aTrack);
2432 return;
2435 if (decoder.mError && !decoder.HasFatalError()) {
2436 MOZ_RELEASE_ASSERT(!decoder.HasInternalSeekPending(),
2437 "No error can occur while an internal seek is pending");
2439 nsCString error;
2440 bool firstFrameDecodingFailedWithHardware =
2441 decoder.mFirstFrameTime &&
2442 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2443 decoder.mDecoder && decoder.mDecoder->IsHardwareAccelerated(error) &&
2444 !decoder.mHardwareDecodingDisabled;
2445 bool needsNewDecoder =
2446 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER ||
2447 firstFrameDecodingFailedWithHardware;
2448 // Limit number of process restarts after crash
2449 if ((decoder.mError.ref() ==
2450 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR &&
2451 decoder.mNumOfConsecutiveRDDOrGPUCrashes++ <
2452 decoder.mMaxConsecutiveRDDOrGPUCrashes) ||
2453 (decoder.mError.ref() ==
2454 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR &&
2455 decoder.mNumOfConsecutiveUtilityCrashes++ <
2456 decoder.mMaxConsecutiveUtilityCrashes)) {
2457 needsNewDecoder = true;
2459 // For MF CDM crash, it needs to be handled differently. We need to shutdown
2460 // current decoder and report that error to the state machine in order to
2461 // let it to determine if playback can keep going or not.
2462 if (decoder.mError.ref() ==
2463 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR) {
2464 LOG("Error: notify MF CDM crash and shutdown %s decoder",
2465 TrackTypeToStr(aTrack));
2466 ShutdownDecoder(aTrack);
2467 decoder.RejectPromise(decoder.mError.ref(), __func__);
2468 decoder.mError.reset();
2469 return;
2471 #ifdef XP_LINUX
2472 // We failed to decode on Linux with HW decoder,
2473 // give it another try without HW decoder.
2474 if (decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2475 decoder.mDecoder->IsHardwareAccelerated(error)) {
2476 LOG("Error: %s decode error, disable HW acceleration",
2477 TrackTypeToStr(aTrack));
2478 needsNewDecoder = true;
2479 decoder.mHardwareDecodingDisabled = true;
2481 // RDD process crashed on Linux, give it another try without HW decoder.
2482 if (decoder.mError.ref() ==
2483 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR) {
2484 LOG("Error: %s remote decoder crashed, disable HW acceleration",
2485 TrackTypeToStr(aTrack));
2486 decoder.mHardwareDecodingDisabled = true;
2488 #endif
2489 // We don't want to expose crash error so switch to
2490 // NS_ERROR_DOM_MEDIA_DECODE_ERR.
2491 if (decoder.mError.ref() ==
2492 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR ||
2493 decoder.mError.ref() ==
2494 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR) {
2495 decoder.mError = Some(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
2496 RESULT_DETAIL("Unable to decode")));
2498 if (!needsNewDecoder && ++decoder.mNumOfConsecutiveDecodingError >
2499 decoder.mMaxConsecutiveDecodingError) {
2500 DDLOG(DDLogCategory::Log, "too_many_decode_errors", decoder.mError.ref());
2501 NotifyError(aTrack, decoder.mError.ref());
2502 return;
2505 if (firstFrameDecodingFailedWithHardware) {
2506 decoder.mHardwareDecodingDisabled = true;
2508 decoder.mError.reset();
2510 LOG("%s decoded error count %d RDD crashes count %d",
2511 TrackTypeToStr(aTrack), decoder.mNumOfConsecutiveDecodingError,
2512 decoder.mNumOfConsecutiveRDDOrGPUCrashes);
2514 if (needsNewDecoder) {
2515 LOG("Error: %s needs a new decoder", TrackTypeToStr(aTrack));
2516 ShutdownDecoder(aTrack);
2518 if (decoder.mFirstFrameTime) {
2519 TimeInterval seekInterval = TimeInterval(decoder.mFirstFrameTime.ref(),
2520 decoder.mFirstFrameTime.ref());
2521 InternalSeek(aTrack, InternalSeekTarget(seekInterval, false));
2522 return;
2525 TimeUnit nextKeyframe;
2526 if (aTrack == TrackType::kVideoTrack &&
2527 NS_SUCCEEDED(
2528 decoder.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe)) &&
2529 !nextKeyframe.IsInfinite()) {
2530 SkipVideoDemuxToNextKeyFrame(
2531 decoder.mLastDecodedSampleTime.refOr(TimeInterval()).Length());
2532 } else if (aTrack == TrackType::kAudioTrack) {
2533 decoder.Flush();
2534 } else {
2535 DDLOG(DDLogCategory::Log, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR);
2536 // We can't recover from this error.
2537 NotifyError(aTrack, NS_ERROR_DOM_MEDIA_FATAL_ERR);
2539 return;
2542 bool needInput = NeedInput(decoder);
2544 LOGV("Update(%s) ni=%d no=%d in:%" PRIu64 " out:%" PRIu64
2545 " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2546 "ds:%d sid:%u waitcdm:%d",
2547 TrackTypeToStr(aTrack), needInput, needOutput, decoder.mNumSamplesInput,
2548 decoder.mNumSamplesOutput, uint32_t(size_t(decoder.mSizeOfQueue)),
2549 decoder.mDecodeRequest.Exists(), decoder.mFlushing,
2550 decoder.mDescription.get(), uint32_t(decoder.mOutput.Length()),
2551 decoder.mWaitingForData, decoder.mDemuxEOS, int32_t(decoder.mDrainState),
2552 decoder.mLastStreamSourceID, IsDecoderWaitingForCDM(aTrack));
2554 if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack)) {
2555 // If the content is encrypted, MFR won't start to create decoder until
2556 // CDMProxy is set.
2557 return;
2560 if ((decoder.IsWaitingForData() &&
2561 (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) ||
2562 (decoder.IsWaitingForKey())) {
2563 // Nothing more we can do at present.
2564 LOGV("Still waiting for data or key. data(%d)/key(%d)",
2565 decoder.mWaitingForData, decoder.mWaitingForKey);
2566 return;
2569 if (decoder.CancelWaitingForKey()) {
2570 LOGV("No longer waiting for key. Resolving waiting promise");
2571 return;
2574 if (!needInput) {
2575 LOGV("No need for additional input (pending:%u)",
2576 uint32_t(decoder.mOutput.Length()));
2577 return;
2580 // Demux samples if we don't have some.
2581 RequestDemuxSamples(aTrack);
2583 HandleDemuxedSamples(aTrack, a);
2586 void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
2587 AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK);
2588 MOZ_ASSERT(GetDecoderData(aTrack).HasPromise());
2589 MOZ_DIAGNOSTIC_ASSERT(aData->mType != MediaData::Type::NULL_DATA);
2590 LOG("Resolved data promise for %s [%" PRId64 ", %" PRId64 "]",
2591 TrackTypeToStr(aTrack), aData->mTime.ToMicroseconds(),
2592 aData->GetEndTime().ToMicroseconds());
2594 if (aTrack == TrackInfo::kAudioTrack) {
2595 AudioData* audioData = aData->As<AudioData>();
2597 if (audioData->mChannels != mInfo.mAudio.mChannels ||
2598 audioData->mRate != mInfo.mAudio.mRate) {
2599 LOG("change of audio format (rate:%d->%d). "
2600 "This is an unsupported configuration",
2601 mInfo.mAudio.mRate, audioData->mRate);
2602 mInfo.mAudio.mRate = audioData->mRate;
2603 mInfo.mAudio.mChannels = audioData->mChannels;
2604 MutexAutoLock lock(mAudio.mMutex);
2605 mAudio.mWorkingInfo->GetAsAudioInfo()->mRate = audioData->mRate;
2606 mAudio.mWorkingInfo->GetAsAudioInfo()->mChannels = audioData->mChannels;
2607 mWorkingInfoChanged = true;
2609 mAudio.ResolvePromise(audioData, __func__);
2610 } else if (aTrack == TrackInfo::kVideoTrack) {
2611 VideoData* videoData = aData->As<VideoData>();
2613 if (videoData->mDisplay != mInfo.mVideo.mDisplay) {
2614 LOG("change of video display size (%dx%d->%dx%d)",
2615 mInfo.mVideo.mDisplay.width, mInfo.mVideo.mDisplay.height,
2616 videoData->mDisplay.width, videoData->mDisplay.height);
2617 mInfo.mVideo.mDisplay = videoData->mDisplay;
2618 MutexAutoLock lock(mVideo.mMutex);
2619 mVideo.mWorkingInfo->GetAsVideoInfo()->mDisplay = videoData->mDisplay;
2620 mWorkingInfoChanged = true;
2623 mozilla::gfx::ColorDepth colorDepth = videoData->GetColorDepth();
2624 if (colorDepth != mInfo.mVideo.mColorDepth) {
2625 LOG("change of video color depth (enum %u -> enum %u)",
2626 (unsigned)mInfo.mVideo.mColorDepth, (unsigned)colorDepth);
2627 mInfo.mVideo.mColorDepth = colorDepth;
2628 MutexAutoLock lock(mVideo.mMutex);
2629 mVideo.mWorkingInfo->GetAsVideoInfo()->mColorDepth = colorDepth;
2630 mWorkingInfoChanged = true;
2633 TimeUnit nextKeyframe;
2634 if (!mVideo.HasInternalSeekPending() &&
2635 NS_SUCCEEDED(
2636 mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe))) {
2637 videoData->SetNextKeyFrameTime(nextKeyframe);
2640 mVideo.ResolvePromise(videoData, __func__);
2644 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2645 return SizeOfQueue(TrackInfo::kVideoTrack);
2648 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2649 return SizeOfQueue(TrackInfo::kAudioTrack);
2652 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack) {
2653 auto& decoder = GetDecoderData(aTrack);
2654 return decoder.mSizeOfQueue;
2657 RefPtr<MediaFormatReader::WaitForDataPromise> MediaFormatReader::WaitForData(
2658 MediaData::Type aType) {
2659 MOZ_ASSERT(OnTaskQueue());
2660 TrackType trackType = aType == MediaData::Type::VIDEO_DATA
2661 ? TrackType::kVideoTrack
2662 : TrackType::kAudioTrack;
2663 auto& decoder = GetDecoderData(trackType);
2664 if (!decoder.IsWaitingForData() && !decoder.IsWaitingForKey()) {
2665 // We aren't waiting for anything.
2666 return WaitForDataPromise::CreateAndResolve(decoder.mType, __func__);
2668 RefPtr<WaitForDataPromise> p = decoder.mWaitingPromise.Ensure(__func__);
2669 ScheduleUpdate(trackType);
2670 return p;
2673 nsresult MediaFormatReader::ResetDecode(TrackSet aTracks) {
2674 AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK);
2675 MOZ_ASSERT(OnTaskQueue());
2676 LOGV("");
2678 mSeekPromise.RejectIfExists(NS_OK, __func__);
2679 mSkipRequest.DisconnectIfExists();
2681 // Do the same for any data wait promises.
2682 if (aTracks.contains(TrackInfo::kAudioTrack)) {
2683 mAudio.mWaitingPromise.RejectIfExists(
2684 WaitForDataRejectValue(MediaData::Type::AUDIO_DATA,
2685 WaitForDataRejectValue::CANCELED),
2686 __func__);
2689 if (aTracks.contains(TrackInfo::kVideoTrack)) {
2690 mVideo.mWaitingPromise.RejectIfExists(
2691 WaitForDataRejectValue(MediaData::Type::VIDEO_DATA,
2692 WaitForDataRejectValue::CANCELED),
2693 __func__);
2696 // Reset miscellaneous seeking state.
2697 mPendingSeekTime.reset();
2699 if (HasVideo() && aTracks.contains(TrackInfo::kVideoTrack)) {
2700 mVideo.ResetDemuxer();
2701 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2702 Reset(TrackInfo::kVideoTrack);
2703 if (mVideo.HasPromise()) {
2704 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2708 if (HasAudio() && aTracks.contains(TrackInfo::kAudioTrack)) {
2709 mAudio.ResetDemuxer();
2710 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2711 Reset(TrackInfo::kAudioTrack);
2712 if (mAudio.HasPromise()) {
2713 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2717 return NS_OK;
2720 void MediaFormatReader::Reset(TrackType aTrack) {
2721 MOZ_ASSERT(OnTaskQueue());
2722 LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack));
2724 auto& decoder = GetDecoderData(aTrack);
2726 decoder.ResetState();
2727 decoder.Flush();
2729 LOG("Reset(%s) END", TrackTypeToStr(aTrack));
2732 void MediaFormatReader::DropDecodedSamples(TrackType aTrack) {
2733 MOZ_ASSERT(OnTaskQueue());
2734 auto& decoder = GetDecoderData(aTrack);
2735 size_t lengthDecodedQueue = decoder.mOutput.Length();
2736 if (lengthDecodedQueue && decoder.mTimeThreshold.isSome()) {
2737 auto time = decoder.mOutput.LastElement()->mTime;
2738 if (time >= decoder.mTimeThreshold.ref().Time()) {
2739 // We would have reached our internal seek target.
2740 decoder.mTimeThreshold.reset();
2743 decoder.mOutput.Clear();
2744 decoder.mSizeOfQueue -= lengthDecodedQueue;
2745 if (aTrack == TrackInfo::kVideoTrack && mFrameStats) {
2746 mFrameStats->Accumulate({0, 0, 0, lengthDecodedQueue, 0, 0});
2750 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold) {
2751 AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2752 MEDIA_PLAYBACK);
2753 MOZ_ASSERT(OnTaskQueue());
2754 LOG("Skipping up to %" PRId64, aTimeThreshold.ToMicroseconds());
2756 // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2757 // As such we can drop all already decoded samples and discard all pending
2758 // samples.
2759 DropDecodedSamples(TrackInfo::kVideoTrack);
2761 mVideo.mTrackDemuxer->SkipToNextRandomAccessPoint(aTimeThreshold)
2762 ->Then(OwnerThread(), __func__, this,
2763 &MediaFormatReader::OnVideoSkipCompleted,
2764 &MediaFormatReader::OnVideoSkipFailed)
2765 ->Track(mSkipRequest);
2768 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped) {
2769 PROFILER_MARKER_UNTYPED("SkippedVideoDecode", MEDIA_PLAYBACK);
2770 MOZ_ASSERT(OnTaskQueue());
2772 // Some frames may have been output by the decoder since we initiated the
2773 // videoskip process and we know they would be late.
2774 DropDecodedSamples(TrackInfo::kVideoTrack);
2775 // Report the pending frames as dropped.
2776 if (mFrameStats) {
2777 uint32_t droppedDecoderCount = SizeOfVideoQueueInFrames();
2778 mFrameStats->Accumulate({0, 0, 0, droppedDecoderCount, 0, 0});
2781 // Cancel any pending demux request and pending demuxed samples.
2782 mVideo.mDemuxRequest.DisconnectIfExists();
2783 Reset(TrackType::kVideoTrack);
2785 if (mFrameStats) {
2786 mFrameStats->Accumulate({aSkipped, 0, 0, aSkipped, 0, 0});
2789 mVideo.mNumSamplesSkippedTotal += aSkipped;
2792 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped) {
2793 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2794 MEDIA_PLAYBACK);
2795 MOZ_ASSERT(OnTaskQueue());
2796 LOG("Skipping succeeded, skipped %u frames", aSkipped);
2797 mSkipRequest.Complete();
2799 DDLOG(DDLogCategory::Log, "video_skipped", DDNoValue());
2801 VideoSkipReset(aSkipped);
2803 ScheduleUpdate(TrackInfo::kVideoTrack);
2806 void MediaFormatReader::OnVideoSkipFailed(
2807 MediaTrackDemuxer::SkipFailureHolder aFailure) {
2808 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK);
2809 MOZ_ASSERT(OnTaskQueue());
2810 LOG("Skipping failed, skipped %u frames", aFailure.mSkipped);
2811 mSkipRequest.Complete();
2813 switch (aFailure.mFailure.Code()) {
2814 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2815 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2816 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2817 aFailure.mFailure);
2818 // Some frames may have been output by the decoder since we initiated the
2819 // videoskip process and we know they would be late.
2820 DropDecodedSamples(TrackInfo::kVideoTrack);
2821 // We can't complete the skip operation, will just service a video frame
2822 // normally.
2823 ScheduleUpdate(TrackInfo::kVideoTrack);
2824 break;
2825 case NS_ERROR_DOM_MEDIA_CANCELED:
2826 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2827 aFailure.mFailure);
2828 if (mVideo.HasPromise()) {
2829 mVideo.RejectPromise(aFailure.mFailure, __func__);
2831 break;
2832 default:
2833 DDLOG(DDLogCategory::Log, "video_skipping_error", aFailure.mFailure);
2834 NotifyError(TrackType::kVideoTrack, aFailure.mFailure);
2835 break;
2839 RefPtr<MediaFormatReader::SeekPromise> MediaFormatReader::Seek(
2840 const SeekTarget& aTarget) {
2841 AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK);
2842 MOZ_ASSERT(OnTaskQueue());
2844 LOG("aTarget=(%" PRId64 "), track=%s", aTarget.GetTime().ToMicroseconds(),
2845 SeekTarget::TrackToStr(aTarget.GetTrack()));
2847 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty());
2848 MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime.isNothing());
2849 // Should reset data request, and no pending internal seek.
2850 if (aTarget.IsAllTracks()) {
2851 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2852 MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise());
2853 MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2854 MOZ_DIAGNOSTIC_ASSERT(mAudio.mTimeThreshold.isNothing());
2855 } else if (aTarget.IsVideoOnly()) {
2856 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2857 MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2858 } else if (aTarget.IsAudioOnly()) {
2859 MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise());
2860 MOZ_DIAGNOSTIC_ASSERT(mAudio.mTimeThreshold.isNothing());
2863 if (!mInfo.mMediaSeekable && !mInfo.mMediaSeekableOnlyInBufferedRanges) {
2864 LOG("Seek() END (Unseekable)");
2865 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2868 if (mShutdown) {
2869 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2872 SetSeekTarget(aTarget);
2874 RefPtr<SeekPromise> p = mSeekPromise.Ensure(__func__);
2876 ScheduleSeek();
2878 return p;
2881 void MediaFormatReader::SetSeekTarget(const SeekTarget& aTarget) {
2882 MOZ_ASSERT(OnTaskQueue());
2884 mOriginalSeekTarget = aTarget;
2885 mFallbackSeekTime = mPendingSeekTime = Some(aTarget.GetTime());
2888 void MediaFormatReader::ScheduleSeek() {
2889 if (mSeekScheduled) {
2890 return;
2892 mSeekScheduled = true;
2893 nsresult rv = OwnerThread()->Dispatch(NewRunnableMethod(
2894 "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek));
2895 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
2896 Unused << rv;
2899 void MediaFormatReader::AttemptSeek() {
2900 AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK);
2901 MOZ_ASSERT(OnTaskQueue());
2903 mSeekScheduled = false;
2905 if (mPendingSeekTime.isNothing()) {
2906 LOGV("AttemptSeek, no pending seek time?");
2907 return;
2910 // Only reset the demuxers targeted by this SeekTarget, to avoid A/V sync
2911 // issues.
2912 const bool isSeekingAudio = HasAudio() && !mOriginalSeekTarget.IsVideoOnly();
2913 const bool isSeekingVideo = HasVideo() && !mOriginalSeekTarget.IsAudioOnly();
2914 LOG("AttemptSeek, seekingAudio=%d, seekingVideo=%d", isSeekingAudio,
2915 isSeekingVideo);
2916 if (isSeekingVideo) {
2917 mVideo.ResetDemuxer();
2918 mVideo.ResetState();
2920 if (isSeekingAudio) {
2921 mAudio.ResetDemuxer();
2922 mAudio.ResetState();
2925 // If seeking both tracks, seek the video track, and then the audio track when
2926 // the video track seek has completed. Otherwise, only seek a specific track.
2927 if (isSeekingVideo) {
2928 DoVideoSeek();
2929 } else if (isSeekingAudio) {
2930 DoAudioSeek();
2931 } else {
2932 MOZ_CRASH();
2936 void MediaFormatReader::OnSeekFailed(TrackType aTrack,
2937 const MediaResult& aError) {
2938 AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK);
2939 MOZ_ASSERT(OnTaskQueue());
2940 LOGV("%s failure:%s", TrackTypeToStr(aTrack), aError.ErrorName().get());
2941 if (aTrack == TrackType::kVideoTrack) {
2942 mVideo.mSeekRequest.Complete();
2943 } else {
2944 mAudio.mSeekRequest.Complete();
2947 if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
2948 if (HasVideo() && aTrack == TrackType::kAudioTrack &&
2949 mFallbackSeekTime.isSome() &&
2950 mPendingSeekTime.ref() != mFallbackSeekTime.ref()) {
2951 // We have failed to seek audio where video seeked to earlier.
2952 // Attempt to seek instead to the closest point that we know we have in
2953 // order to limit A/V sync discrepency.
2955 // Ensure we have the most up to date buffered ranges.
2956 UpdateReceivedNewData(TrackType::kAudioTrack);
2957 Maybe<TimeUnit> nextSeekTime;
2958 // Find closest buffered time found after video seeked time.
2959 for (const auto& timeRange : mAudio.mTimeRanges) {
2960 if (timeRange.mStart >= mPendingSeekTime.ref()) {
2961 nextSeekTime.emplace(timeRange.mStart);
2962 break;
2965 if (nextSeekTime.isNothing() ||
2966 nextSeekTime.ref() > mFallbackSeekTime.ref()) {
2967 nextSeekTime = Some(mFallbackSeekTime.ref());
2968 LOG("Unable to seek audio to video seek time. A/V sync may be broken");
2969 } else {
2970 mFallbackSeekTime.reset();
2972 mPendingSeekTime = nextSeekTime;
2973 DoAudioSeek();
2974 return;
2976 NotifyWaitingForData(aTrack);
2978 MOZ_ASSERT(!mVideo.mSeekRequest.Exists() && !mAudio.mSeekRequest.Exists());
2979 mPendingSeekTime.reset();
2981 auto type = aTrack == TrackType::kAudioTrack ? MediaData::Type::AUDIO_DATA
2982 : MediaData::Type::VIDEO_DATA;
2983 mSeekPromise.RejectIfExists(SeekRejectValue(type, aError), __func__);
2986 void MediaFormatReader::DoVideoSeek() {
2987 AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK);
2988 MOZ_ASSERT(mPendingSeekTime.isSome());
2989 LOGV("Seeking video to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
2990 MOZ_DIAGNOSTIC_ASSERT(!IsAudioOnlySeeking());
2991 MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists());
2992 auto seekTime = mPendingSeekTime.ref();
2993 mVideo.mTrackDemuxer->Seek(seekTime)
2994 ->Then(OwnerThread(), __func__, this,
2995 &MediaFormatReader::OnVideoSeekCompleted,
2996 &MediaFormatReader::OnVideoSeekFailed)
2997 ->Track(mVideo.mSeekRequest);
3000 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime) {
3001 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
3002 MEDIA_PLAYBACK);
3003 MOZ_ASSERT(OnTaskQueue());
3004 LOGV("Video seeked to %" PRId64, aTime.ToMicroseconds());
3005 mVideo.mSeekRequest.Complete();
3007 mVideo.mFirstFrameTime = Some(aTime);
3008 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
3010 SetVideoDecodeThreshold();
3012 if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
3013 MOZ_ASSERT(mPendingSeekTime.isSome());
3014 if (mOriginalSeekTarget.IsFast()) {
3015 // We are performing a fast seek. We need to seek audio to where the
3016 // video seeked to, to ensure proper A/V sync once playback resume.
3017 mPendingSeekTime = Some(aTime);
3019 DoAudioSeek();
3020 } else {
3021 mPendingSeekTime.reset();
3022 mSeekPromise.ResolveIfExists(aTime, __func__);
3026 void MediaFormatReader::OnVideoSeekFailed(const MediaResult& aError) {
3027 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK);
3028 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
3029 OnSeekFailed(TrackType::kVideoTrack, aError);
3032 void MediaFormatReader::SetVideoDecodeThreshold() {
3033 MOZ_ASSERT(OnTaskQueue());
3035 if (!HasVideo() || !mVideo.mDecoder) {
3036 return;
3039 if (!mVideo.mTimeThreshold && !IsSeeking()) {
3040 return;
3043 TimeUnit threshold;
3044 if (mVideo.mTimeThreshold) {
3045 // For internalSeek.
3046 threshold = mVideo.mTimeThreshold.ref().Time();
3047 } else if (IsSeeking()) {
3048 // If IsSeeking() is true, then video seek must have completed already.
3049 TimeUnit keyframe;
3050 if (NS_FAILED(mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&keyframe))) {
3051 return;
3054 // If the key frame is invalid/infinite, it means the target position is
3055 // closing to end of stream. We don't want to skip any frame at this point.
3056 threshold = keyframe.IsValid() && !keyframe.IsInfinite()
3057 ? mOriginalSeekTarget.GetTime()
3058 : TimeUnit::Invalid();
3059 } else {
3060 return;
3063 if (threshold.IsValid()) {
3064 LOG("Set seek threshold to %" PRId64, threshold.ToMicroseconds());
3065 } else {
3066 LOG("Resetting seek threshold");
3068 mVideo.mDecoder->SetSeekThreshold(threshold);
3071 void MediaFormatReader::DoAudioSeek() {
3072 AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK);
3073 MOZ_ASSERT(mPendingSeekTime.isSome());
3074 LOGV("Seeking audio to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
3075 MOZ_DIAGNOSTIC_ASSERT(!IsVideoOnlySeeking());
3076 MOZ_DIAGNOSTIC_ASSERT(!mAudio.mSeekRequest.Exists());
3077 auto seekTime = mPendingSeekTime.ref();
3078 mAudio.mTrackDemuxer->Seek(seekTime)
3079 ->Then(OwnerThread(), __func__, this,
3080 &MediaFormatReader::OnAudioSeekCompleted,
3081 &MediaFormatReader::OnAudioSeekFailed)
3082 ->Track(mAudio.mSeekRequest);
3085 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime) {
3086 MOZ_ASSERT(OnTaskQueue());
3087 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
3088 MEDIA_PLAYBACK);
3089 LOGV("Audio seeked to %" PRId64, aTime.ToMicroseconds());
3090 mAudio.mSeekRequest.Complete();
3091 mAudio.mFirstFrameTime = Some(aTime);
3092 mPendingSeekTime.reset();
3093 mSeekPromise.ResolveIfExists(aTime, __func__);
3096 void MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError) {
3097 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK);
3098 OnSeekFailed(TrackType::kAudioTrack, aError);
3101 void MediaFormatReader::ReleaseResources() {
3102 LOGV("");
3103 if (mShutdown) {
3104 return;
3106 ShutdownDecoder(TrackInfo::kAudioTrack);
3107 ShutdownDecoder(TrackInfo::kVideoTrack);
3110 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
3111 return mVideo.mIsHardwareAccelerated;
3114 void MediaFormatReader::NotifyTrackDemuxers() {
3115 MOZ_ASSERT(OnTaskQueue());
3117 LOGV("");
3119 if (!mInitDone) {
3120 return;
3123 if (HasVideo()) {
3124 mVideo.mReceivedNewData = true;
3125 ScheduleUpdate(TrackType::kVideoTrack);
3127 if (HasAudio()) {
3128 mAudio.mReceivedNewData = true;
3129 ScheduleUpdate(TrackType::kAudioTrack);
3133 void MediaFormatReader::NotifyDataArrived() {
3134 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK);
3135 MOZ_ASSERT(OnTaskQueue());
3137 if (mShutdown || !mDemuxer || !mDemuxerInitDone) {
3138 return;
3141 if (mNotifyDataArrivedPromise.Exists()) {
3142 // Already one in progress. Set the dirty flag so we can process it later.
3143 mPendingNotifyDataArrived = true;
3144 return;
3147 RefPtr<MediaFormatReader> self = this;
3148 mDemuxer->NotifyDataArrived()
3149 ->Then(
3150 OwnerThread(), __func__,
3151 [self]() {
3152 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
3153 MEDIA_PLAYBACK);
3154 self->mNotifyDataArrivedPromise.Complete();
3155 self->UpdateBuffered();
3156 self->NotifyTrackDemuxers();
3157 if (self->mPendingNotifyDataArrived) {
3158 self->mPendingNotifyDataArrived = false;
3159 self->NotifyDataArrived();
3162 [self]() { self->mNotifyDataArrivedPromise.Complete(); })
3163 ->Track(mNotifyDataArrivedPromise);
3166 void MediaFormatReader::UpdateMediaEngineId(uint64_t aMediaEngineId) {
3167 LOG("Update external media engine Id %" PRIu64, aMediaEngineId);
3168 mMediaEngineId = Some(aMediaEngineId);
3171 void MediaFormatReader::UpdateBuffered() {
3172 AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK);
3173 MOZ_ASSERT(OnTaskQueue());
3175 if (mShutdown) {
3176 return;
3179 if (!mInitDone || !mHasStartTime) {
3180 mBuffered = TimeIntervals();
3181 return;
3184 if (HasVideo()) {
3185 mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered();
3186 bool hasLastEnd;
3187 auto lastEnd = mVideo.mTimeRanges.GetEnd(&hasLastEnd);
3188 if (hasLastEnd) {
3189 if (mVideo.mLastTimeRangesEnd &&
3190 mVideo.mLastTimeRangesEnd.ref() < lastEnd) {
3191 // New data was added after our previous end, we can clear the EOS flag.
3192 mVideo.mDemuxEOS = false;
3193 ScheduleUpdate(TrackInfo::kVideoTrack);
3195 mVideo.mLastTimeRangesEnd = Some(lastEnd);
3198 if (HasAudio()) {
3199 mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered();
3200 bool hasLastEnd;
3201 auto lastEnd = mAudio.mTimeRanges.GetEnd(&hasLastEnd);
3202 if (hasLastEnd) {
3203 if (mAudio.mLastTimeRangesEnd &&
3204 mAudio.mLastTimeRangesEnd.ref() < lastEnd) {
3205 // New data was added after our previous end, we can clear the EOS flag.
3206 mAudio.mDemuxEOS = false;
3207 ScheduleUpdate(TrackInfo::kAudioTrack);
3209 mAudio.mLastTimeRangesEnd = Some(lastEnd);
3213 media::TimeIntervals intervals;
3214 if (HasAudio() && HasVideo()) {
3215 intervals = media::Intersection(mVideo.mTimeRanges, mAudio.mTimeRanges);
3216 } else if (HasAudio()) {
3217 intervals = mAudio.mTimeRanges;
3218 } else if (HasVideo()) {
3219 intervals = mVideo.mTimeRanges;
3222 if (intervals.IsEmpty() || intervals.GetStart() == TimeUnit::Zero()) {
3223 // IntervalSet already starts at 0 or is empty, nothing to shift.
3224 mBuffered = intervals;
3225 } else {
3226 mBuffered = intervals.Shift(TimeUnit::Zero() - mInfo.mStartTime);
3230 layers::ImageContainer* MediaFormatReader::GetImageContainer() {
3231 return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
3232 : nullptr;
3235 RefPtr<GenericPromise> MediaFormatReader::RequestDebugInfo(
3236 dom::MediaFormatReaderDebugInfo& aInfo) {
3237 if (!OnTaskQueue()) {
3238 // Run the request on the task queue if it's not already.
3239 return InvokeAsync(mTaskQueue, __func__,
3240 [this, self = RefPtr{this}, &aInfo] {
3241 return RequestDebugInfo(aInfo);
3244 GetDebugInfo(aInfo);
3245 return GenericPromise::CreateAndResolve(true, __func__);
3248 Maybe<nsCString> MediaFormatReader::GetAudioProcessPerCodec() {
3249 if (mAudio.mDescription == "uninitialized"_ns) {
3250 return Nothing();
3253 MOZ_ASSERT(mAudio.mProcessName.Length() > 0,
3254 "Should have had a process name");
3255 MOZ_ASSERT(mAudio.mCodecName.Length() > 0, "Should have had a codec name");
3257 nsCString processName = mAudio.mProcessName;
3258 nsCString audioProcessPerCodecName(processName + ","_ns + mAudio.mCodecName);
3259 if (processName != "utility"_ns) {
3260 if (!StaticPrefs::media_rdd_process_enabled()) {
3261 audioProcessPerCodecName += ",rdd-disabled"_ns;
3263 if (!StaticPrefs::media_utility_process_enabled()) {
3264 audioProcessPerCodecName += ",utility-disabled"_ns;
3267 return Some(audioProcessPerCodecName);
3270 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo) {
3271 MOZ_ASSERT(OnTaskQueue(),
3272 "Don't call this off the task queue, it's going to touch a lot of "
3273 "data members");
3274 nsCString result;
3275 nsAutoCString audioDecoderName("unavailable");
3276 nsAutoCString videoDecoderName = audioDecoderName;
3277 nsAutoCString audioType("none");
3278 nsAutoCString videoType("none");
3280 AudioInfo audioInfo;
3281 if (HasAudio()) {
3282 audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
3283 audioDecoderName = mAudio.mDecoder ? mAudio.mDecoder->GetDescriptionName()
3284 : mAudio.mDescription;
3285 audioType = audioInfo.mMimeType;
3286 aInfo.mAudioState.mNeedInput = NeedInput(mAudio);
3287 aInfo.mAudioState.mHasPromise = mAudio.HasPromise();
3288 aInfo.mAudioState.mWaitingPromise = !mAudio.mWaitingPromise.IsEmpty();
3289 aInfo.mAudioState.mHasDemuxRequest = mAudio.mDemuxRequest.Exists();
3290 aInfo.mAudioState.mDemuxQueueSize =
3291 uint32_t(mAudio.mQueuedSamples.Length());
3292 aInfo.mAudioState.mHasDecoder = mAudio.mDecodeRequest.Exists();
3293 aInfo.mAudioState.mTimeTreshold =
3294 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().Time().ToSeconds()
3295 : -1.0;
3296 aInfo.mAudioState.mTimeTresholdHasSeeked =
3297 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().mHasSeeked : false;
3298 aInfo.mAudioState.mNumSamplesInput = mAudio.mNumSamplesInput;
3299 aInfo.mAudioState.mNumSamplesOutput = mAudio.mNumSamplesOutput;
3300 aInfo.mAudioState.mQueueSize = size_t(mAudio.mSizeOfQueue);
3301 aInfo.mAudioState.mPending = mAudio.mOutput.Length();
3302 aInfo.mAudioState.mWaitingForData = mAudio.mWaitingForData;
3303 aInfo.mAudioState.mDemuxEOS = mAudio.mDemuxEOS;
3304 aInfo.mAudioState.mDrainState = int32_t(mAudio.mDrainState);
3305 aInfo.mAudioState.mWaitingForKey = mAudio.mWaitingForKey;
3306 aInfo.mAudioState.mLastStreamSourceID = mAudio.mLastStreamSourceID;
3309 CopyUTF8toUTF16(audioDecoderName, aInfo.mAudioDecoderName);
3310 CopyUTF8toUTF16(audioType, aInfo.mAudioType);
3311 aInfo.mAudioChannels = audioInfo.mChannels;
3312 aInfo.mAudioRate = audioInfo.mRate;
3313 aInfo.mAudioFramesDecoded = mAudio.mNumSamplesOutputTotal;
3315 VideoInfo videoInfo;
3316 if (HasVideo()) {
3317 videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
3318 videoDecoderName = mVideo.mDecoder ? mVideo.mDecoder->GetDescriptionName()
3319 : mVideo.mDescription;
3320 videoType = videoInfo.mMimeType;
3321 aInfo.mVideoState.mNeedInput = NeedInput(mVideo);
3322 aInfo.mVideoState.mHasPromise = mVideo.HasPromise();
3323 aInfo.mVideoState.mWaitingPromise = !mVideo.mWaitingPromise.IsEmpty();
3324 aInfo.mVideoState.mHasDemuxRequest = mVideo.mDemuxRequest.Exists();
3325 aInfo.mVideoState.mDemuxQueueSize =
3326 uint32_t(mVideo.mQueuedSamples.Length());
3327 aInfo.mVideoState.mHasDecoder = mVideo.mDecodeRequest.Exists();
3328 aInfo.mVideoState.mTimeTreshold =
3329 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().Time().ToSeconds()
3330 : -1.0;
3331 aInfo.mVideoState.mTimeTresholdHasSeeked =
3332 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().mHasSeeked : false;
3333 aInfo.mVideoState.mNumSamplesInput = mVideo.mNumSamplesInput;
3334 aInfo.mVideoState.mNumSamplesOutput = mVideo.mNumSamplesOutput;
3335 aInfo.mVideoState.mQueueSize = size_t(mVideo.mSizeOfQueue);
3336 aInfo.mVideoState.mPending = mVideo.mOutput.Length();
3337 aInfo.mVideoState.mWaitingForData = mVideo.mWaitingForData;
3338 aInfo.mVideoState.mDemuxEOS = mVideo.mDemuxEOS;
3339 aInfo.mVideoState.mDrainState = int32_t(mVideo.mDrainState);
3340 aInfo.mVideoState.mWaitingForKey = mVideo.mWaitingForKey;
3341 aInfo.mVideoState.mLastStreamSourceID = mVideo.mLastStreamSourceID;
3344 CopyUTF8toUTF16(videoDecoderName, aInfo.mVideoDecoderName);
3345 CopyUTF8toUTF16(videoType, aInfo.mVideoType);
3346 aInfo.mVideoWidth =
3347 videoInfo.mDisplay.width < 0 ? 0 : videoInfo.mDisplay.width;
3348 aInfo.mVideoHeight =
3349 videoInfo.mDisplay.height < 0 ? 0 : videoInfo.mDisplay.height;
3350 aInfo.mVideoRate = mVideo.mMeanRate.Mean();
3351 aInfo.mVideoHardwareAccelerated = VideoIsHardwareAccelerated();
3352 aInfo.mVideoNumSamplesOutputTotal = mVideo.mNumSamplesOutputTotal;
3353 aInfo.mVideoNumSamplesSkippedTotal = mVideo.mNumSamplesSkippedTotal;
3355 // Looking at dropped frames
3356 FrameStatisticsData stats = mFrameStats->GetFrameStatisticsData();
3357 aInfo.mFrameStats.mDroppedDecodedFrames = stats.mDroppedDecodedFrames;
3358 aInfo.mFrameStats.mDroppedSinkFrames = stats.mDroppedSinkFrames;
3359 aInfo.mFrameStats.mDroppedCompositorFrames = stats.mDroppedCompositorFrames;
3362 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode) {
3363 MOZ_ASSERT(OnTaskQueue());
3364 return SetNullDecode(TrackType::kVideoTrack, aIsNullDecode);
3367 void MediaFormatReader::UpdateCompositor(
3368 already_AddRefed<layers::KnowsCompositor> aCompositor) {
3369 MOZ_ASSERT(OnTaskQueue());
3370 mKnowsCompositor = aCompositor;
3373 void MediaFormatReader::SetNullDecode(TrackType aTrack, bool aIsNullDecode) {
3374 MOZ_ASSERT(OnTaskQueue());
3376 auto& decoder = GetDecoderData(aTrack);
3377 if (decoder.mIsNullDecode == aIsNullDecode) {
3378 return;
3381 LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3382 TrackTypeToStr(aTrack), decoder.mIsNullDecode, aIsNullDecode);
3384 decoder.mIsNullDecode = aIsNullDecode;
3385 ShutdownDecoder(aTrack);
3388 void MediaFormatReader::OnFirstDemuxCompleted(
3389 TrackInfo::TrackType aType,
3390 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
3391 AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3392 MEDIA_PLAYBACK);
3393 MOZ_ASSERT(OnTaskQueue());
3395 if (mShutdown) {
3396 return;
3399 auto& decoder = GetDecoderData(aType);
3400 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3401 decoder.mFirstDemuxedSampleTime.emplace(aSamples->GetSamples()[0]->mTime);
3402 MaybeResolveMetadataPromise();
3405 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType,
3406 const MediaResult& aError) {
3407 MOZ_ASSERT(OnTaskQueue());
3409 if (mShutdown) {
3410 return;
3413 auto& decoder = GetDecoderData(aType);
3414 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3415 decoder.mFirstDemuxedSampleTime.emplace(TimeUnit::FromInfinity());
3416 MaybeResolveMetadataPromise();
3419 } // namespace mozilla
3421 #undef NS_DispatchToMainThread
3422 #undef LOGV
3423 #undef LOG