dom/media/MediaFormatReader.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "MediaFormatReader.h"
   8
   9 #include <algorithm>
  10 #include <map>
  11 #include <queue>
  12
  13 #include "AllocationPolicy.h"
  14 #ifdef MOZ_AV1
  15 #  include "AOMDecoder.h"
  16 #endif
  17 #include "DecoderBenchmark.h"
  18 #include "MediaData.h"
  19 #include "MediaDataDecoderProxy.h"
  20 #include "MediaInfo.h"
  21 #include "MP4Decoder.h"
  22 #include "PDMFactory.h"
  23 #include "PerformanceRecorder.h"
  24 #include "VideoFrameContainer.h"
  25 #include "VideoUtils.h"
  26 #include "VPXDecoder.h"
  27 #include "mozilla/AbstractThread.h"
  28 #include "mozilla/CDMProxy.h"
  29 #include "mozilla/ClearOnShutdown.h"
  30 #include "mozilla/NotNull.h"
  31 #include "mozilla/Preferences.h"
  32 #include "mozilla/ProfilerLabels.h"
  33 #include "mozilla/ProfilerMarkers.h"
  34 #include "mozilla/SharedThreadPool.h"
  35 #include "mozilla/StaticPrefs_media.h"
  36 #include "mozilla/TaskQueue.h"
  37 #include "mozilla/Unused.h"
  38 #include "nsContentUtils.h"
  39 #include "nsLiteralString.h"
  40 #include "nsPrintfCString.h"
  41 #include "nsTHashSet.h"
  42
  43 using namespace mozilla::media;
  44
  45 static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
  46 mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
  47
  48 #define LOG(arg, ...)                                                  \
  49   DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
  50             __func__, ##__VA_ARGS__)
  51 #define LOGV(arg, ...)                                                   \
  52   DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
  53             __func__, ##__VA_ARGS__)
  54
  55 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
  56
  57 namespace mozilla {
  58
  59 using MediaDataDecoderID = void*;
  60
  61 /**
  62  * This class tracks shutdown promises to ensure all decoders are shut down
  63  * completely before MFR continues the rest of the shutdown procedure.
  64  */
  65 class MediaFormatReader::ShutdownPromisePool {
  66  public:
  67   ShutdownPromisePool()
  68       : mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}
  69
  70   // Return a promise which will be resolved when all the tracking promises
  71   // are resolved. Note no more promises should be added for tracking once
  72   // this function is called.
  73   RefPtr<ShutdownPromise> Shutdown();
  74
  75   // Track a shutdown promise.
  76   void Track(const RefPtr<ShutdownPromise>& aPromise);
  77
  78   // Shut down a decoder and track its shutdown promise.
  79   void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {
  80     Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());
  81   }
  82
  83  private:
  84   bool mShutdown = false;
  85   const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;
  86   nsTHashSet<RefPtr<ShutdownPromise>> mPromises;
  87 };
  88
  89 RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {
  90   MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
  91   mShutdown = true;
  92   if (mPromises.Count() == 0) {
  93     mOnShutdownComplete->Resolve(true, __func__);
  94   }
  95   return mOnShutdownComplete;
  96 }
  97
  98 void MediaFormatReader::ShutdownPromisePool::Track(
  99     const RefPtr<ShutdownPromise>& aPromise) {
 100   MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
 101   MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));
 102   mPromises.Insert(aPromise);
 103   aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {
 104     MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));
 105     mPromises.Remove(aPromise);
 106     if (mShutdown && mPromises.Count() == 0) {
 107       mOnShutdownComplete->Resolve(true, __func__);
 108     }
 109   });
 110 }
 111
 112 void MediaFormatReader::DecoderData::ShutdownDecoder() {
 113   MOZ_ASSERT(mOwner->OnTaskQueue());
 114
 115   MutexAutoLock lock(mMutex);
 116
 117   if (!mDecoder) {
 118     // No decoder to shut down.
 119     return;
 120   }
 121
 122   if (mFlushing) {
 123     // Flush is is in action. Shutdown will be initiated after flush completes.
 124     MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);
 125     mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));
 126     // The order of decoder creation and shutdown is handled by LocalAllocPolicy
 127     // and ShutdownPromisePool. MFR can now reset these members to a fresh state
 128     // and be ready to create new decoders again without explicitly waiting for
 129     // flush/shutdown to complete.
 130     mShutdownPromise = nullptr;
 131     mFlushing = false;
 132   } else {
 133     // No flush is in action. We can shut down the decoder now.
 134     mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());
 135   }
 136
 137   // mShutdownPromisePool will handle the order of decoder shutdown so
 138   // we can forget mDecoder and be ready to create a new one.
 139   mDecoder = nullptr;
 140   mDescription = "shutdown"_ns;
 141   mHasReportedVideoHardwareSupportTelemtry = false;
 142   mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA
 143                              ? TrackType::kAudioTrack
 144                              : TrackType::kVideoTrack);
 145 }
 146
 147 void MediaFormatReader::DecoderData::Flush() {
 148   AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);
 149   MOZ_ASSERT(mOwner->OnTaskQueue());
 150
 151   if (mFlushing || mFlushed) {
 152     // Flush still pending or already flushed, nothing more to do.
 153     return;
 154   }
 155   mDecodeRequest.DisconnectIfExists();
 156   mDrainRequest.DisconnectIfExists();
 157   mDrainState = DrainState::None;
 158   CancelWaitingForKey();
 159   mOutput.Clear();
 160   mNumSamplesInput = 0;
 161   mNumSamplesOutput = 0;
 162   mSizeOfQueue = 0;
 163   if (mDecoder) {
 164     TrackType type = mType == MediaData::Type::AUDIO_DATA
 165                          ? TrackType::kAudioTrack
 166                          : TrackType::kVideoTrack;
 167     mFlushing = true;
 168     MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);
 169     mShutdownPromise = new SharedShutdownPromiseHolder();
 170     RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;
 171     RefPtr<MediaDataDecoder> d = mDecoder;
 172     DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
 173              "flushing", DDNoValue{});
 174     mDecoder->Flush()->Then(
 175         mOwner->OwnerThread(), __func__,
 176         [type, this, p, d]() {
 177           AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
 178                               MEDIA_PLAYBACK);
 179           DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
 180                    "flushed", DDNoValue{});
 181           if (!p->IsEmpty()) {
 182             // Shutdown happened before flush completes.
 183             // Let's continue to shut down the decoder. Note
 184             // we don't access |this| because this decoder
 185             // is no longer managed by MFR::DecoderData.
 186             d->Shutdown()->ChainTo(p->Steal(), __func__);
 187             return;
 188           }
 189           mFlushing = false;
 190           mShutdownPromise = nullptr;
 191           mOwner->ScheduleUpdate(type);
 192         },
 193         [type, this, p, d](const MediaResult& aError) {
 194           AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
 195                               MEDIA_PLAYBACK);
 196           DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
 197                    "flush_error", aError);
 198           if (!p->IsEmpty()) {
 199             d->Shutdown()->ChainTo(p->Steal(), __func__);
 200             return;
 201           }
 202           mFlushing = false;
 203           mShutdownPromise = nullptr;
 204           mOwner->NotifyError(type, aError);
 205         });
 206   }
 207   mFlushed = true;
 208 }
 209
 210 class MediaFormatReader::DecoderFactory {
 211   using InitPromise = MediaDataDecoder::InitPromise;
 212   using TokenPromise = AllocPolicy::Promise;
 213   using Token = AllocPolicy::Token;
 214   using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise;
 215
 216  public:
 217   explicit DecoderFactory(MediaFormatReader* aOwner)
 218       : mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),
 219         mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),
 220         mOwner(WrapNotNull(aOwner)) {
 221     DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
 222                                          this);
 223     DecoderDoctorLogger::LinkParentAndChild(
 224         aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);
 225   }
 226
 227   ~DecoderFactory() {
 228     DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
 229                                         this);
 230   }
 231
 232   void CreateDecoder(TrackType aTrack);
 233
 234   // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
 235   // pristine state so CreateDecoder() is ready to be called again immediately.
 236   void ShutdownDecoder(TrackType aTrack) {
 237     MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
 238                aTrack == TrackInfo::kVideoTrack);
 239     auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
 240     data.mPolicy->Cancel();
 241     data.mTokenRequest.DisconnectIfExists();
 242     if (data.mLiveToken) {
 243       // We haven't completed creation of the decoder, and it hasn't been
 244       // initialised yet.
 245       data.mLiveToken = nullptr;
 246       // The decoder will be shutdown as soon as it's available and tracked by
 247       // the ShutdownPromisePool.
 248       mOwner->mShutdownPromisePool->Track(data.mCreateDecoderPromise->Then(
 249           mOwner->mTaskQueue, __func__,
 250           [](CreateDecoderPromise::ResolveOrRejectValue&& aResult) {
 251             if (aResult.IsReject()) {
 252               return ShutdownPromise::CreateAndResolve(true, __func__);
 253             }
 254             return aResult.ResolveValue()->Shutdown();
 255           }));
 256       // Free the token to leave room for a new decoder.
 257       data.mToken = nullptr;
 258     }
 259     data.mInitRequest.DisconnectIfExists();
 260     if (data.mDecoder) {
 261       mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());
 262     }
 263     data.mStage = Stage::None;
 264     MOZ_ASSERT(!data.mToken);
 265   }
 266
 267  private:
 268   enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };
 269
 270   struct Data {
 271     Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)
 272         : mOwnerData(aOwnerData),
 273           mTrack(aTrack),
 274           mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}
 275     DecoderData& mOwnerData;
 276     const TrackType mTrack;
 277     RefPtr<SingleAllocPolicy> mPolicy;
 278     Stage mStage = Stage::None;
 279     RefPtr<Token> mToken;
 280     RefPtr<MediaDataDecoder> mDecoder;
 281     MozPromiseRequestHolder<TokenPromise> mTokenRequest;
 282     struct DecoderCancelled : public SupportsWeakPtr {
 283       NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled)
 284      private:
 285       ~DecoderCancelled() = default;
 286     };
 287     // Set when decoder is about to be created. If cleared before the decoder
 288     // creation promise is resolved; it indicates that Shutdown() was called and
 289     // further processing such as initialization should stop.
 290     RefPtr<DecoderCancelled> mLiveToken;
 291     RefPtr<CreateDecoderPromise> mCreateDecoderPromise;
 292     MozPromiseRequestHolder<InitPromise> mInitRequest;
 293   } mAudio, mVideo;
 294
 295   void RunStage(Data& aData);
 296   void DoCreateDecoder(Data& aData);
 297   void DoInitDecoder(Data& aData);
 298
 299   // guaranteed to be valid by the owner.
 300   const NotNull<MediaFormatReader*> mOwner;
 301 };
 302
 303 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {
 304   MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
 305              aTrack == TrackInfo::kVideoTrack);
 306   Data& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
 307   MOZ_DIAGNOSTIC_ASSERT_IF(mOwner->GetDecoderData(data.mTrack).IsEncrypted(),
 308                            mOwner->mCDMProxy);
 309   RunStage(data);
 310 }
 311
 312 void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {
 313   switch (aData.mStage) {
 314     case Stage::None: {
 315       MOZ_DIAGNOSTIC_ASSERT(!aData.mToken);
 316       aData.mPolicy->Alloc()
 317           ->Then(
 318               mOwner->OwnerThread(), __func__,
 319               [this, &aData](RefPtr<Token> aToken) {
 320                 aData.mTokenRequest.Complete();
 321                 aData.mToken = std::move(aToken);
 322                 aData.mStage = Stage::CreateDecoder;
 323                 RunStage(aData);
 324               },
 325               [&aData]() {
 326                 aData.mTokenRequest.Complete();
 327                 aData.mStage = Stage::None;
 328               })
 329           ->Track(aData.mTokenRequest);
 330       aData.mStage = Stage::WaitForToken;
 331       break;
 332     }
 333
 334     case Stage::WaitForToken: {
 335       MOZ_DIAGNOSTIC_ASSERT(!aData.mToken);
 336       MOZ_DIAGNOSTIC_ASSERT(aData.mTokenRequest.Exists());
 337       break;
 338     }
 339
 340     case Stage::CreateDecoder: {
 341       MOZ_DIAGNOSTIC_ASSERT(aData.mToken);
 342       MOZ_DIAGNOSTIC_ASSERT(!aData.mDecoder);
 343       MOZ_DIAGNOSTIC_ASSERT(!aData.mInitRequest.Exists());
 344
 345       DoCreateDecoder(aData);
 346       aData.mStage = Stage::WaitForInit;
 347       break;
 348     }
 349
 350     case Stage::WaitForInit: {
 351       MOZ_DIAGNOSTIC_ASSERT((aData.mDecoder && aData.mInitRequest.Exists()) ||
 352                             aData.mLiveToken);
 353       break;
 354     }
 355   }
 356 }
 357
 358 void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {
 359   AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);
 360   auto& ownerData = aData.mOwnerData;
 361   auto& decoder = mOwner->GetDecoderData(aData.mTrack);
 362
 363   RefPtr<PDMFactory> platform = new PDMFactory();
 364   if (decoder.IsEncrypted()) {
 365     MOZ_DIAGNOSTIC_ASSERT(mOwner->mCDMProxy);
 366     platform->SetCDMProxy(mOwner->mCDMProxy);
 367   }
 368
 369   RefPtr<PlatformDecoderModule::CreateDecoderPromise> p;
 370   MediaFormatReader* owner = mOwner;
 371   auto onWaitingForKeyEvent =
 372       [owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)]() {
 373         RefPtr<MediaFormatReader> mfr(owner);
 374         MOZ_DIAGNOSTIC_ASSERT(mfr, "The MediaFormatReader didn't wait for us");
 375         return mfr ? &mfr->OnTrackWaitingForKeyProducer() : nullptr;
 376       };
 377
 378   switch (aData.mTrack) {
 379     case TrackInfo::kAudioTrack: {
 380       p = platform->CreateDecoder(
 381           {*ownerData.GetCurrentInfo()->GetAsAudioInfo(), mOwner->mCrashHelper,
 382            CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
 383            TrackInfo::kAudioTrack, std::move(onWaitingForKeyEvent),
 384            mOwner->mMediaEngineId, mOwner->mTrackingId});
 385       break;
 386     }
 387
 388     case TrackType::kVideoTrack: {
 389       // Decoders use the layers backend to decide if they can use hardware
 390       // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
 391       using Option = CreateDecoderParams::Option;
 392       using OptionSet = CreateDecoderParams::OptionSet;
 393
 394       p = platform->CreateDecoder(
 395           {*ownerData.GetCurrentInfo()->GetAsVideoInfo(),
 396            mOwner->mKnowsCompositor, mOwner->GetImageContainer(),
 397            mOwner->mCrashHelper,
 398            CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
 399            TrackType::kVideoTrack, std::move(onWaitingForKeyEvent),
 400            CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),
 401            OptionSet(ownerData.mHardwareDecodingDisabled
 402                          ? Option::HardwareDecoderNotAllowed
 403                          : Option::Default),
 404            mOwner->mMediaEngineId, mOwner->mTrackingId});
 405       break;
 406     }
 407
 408     default:
 409       p = PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(
 410           NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 411   }
 412
 413   aData.mLiveToken = MakeRefPtr<Data::DecoderCancelled>();
 414
 415   aData.mCreateDecoderPromise = p->Then(
 416       mOwner->OwnerThread(), __func__,
 417       [this, &aData, &ownerData, live = WeakPtr{aData.mLiveToken},
 418        owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)](
 419           RefPtr<MediaDataDecoder>&& aDecoder) {
 420         if (!live) {
 421           return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder),
 422                                                         __func__);
 423         }
 424         aData.mLiveToken = nullptr;
 425         aData.mDecoder = new MediaDataDecoderProxy(
 426             aDecoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));
 427         aData.mDecoder = new AllocationWrapper(aData.mDecoder.forget(),
 428                                                aData.mToken.forget());
 429         DecoderDoctorLogger::LinkParentAndChild(
 430             aData.mDecoder.get(), "decoder",
 431             "MediaFormatReader::DecoderFactory", this);
 432
 433         DoInitDecoder(aData);
 434
 435         return CreateDecoderPromise::CreateAndResolve(aData.mDecoder, __func__);
 436       },
 437       [this, &aData,
 438        live = WeakPtr{aData.mLiveToken}](const MediaResult& aError) {
 439         NS_WARNING("Error constructing decoders");
 440         if (!live) {
 441           return CreateDecoderPromise::CreateAndReject(aError, __func__);
 442         }
 443         aData.mLiveToken = nullptr;
 444         aData.mToken = nullptr;
 445         aData.mStage = Stage::None;
 446         aData.mOwnerData.mDescription = aError.Description();
 447         DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
 448                  "create_decoder_error", aError);
 449         mOwner->NotifyError(aData.mTrack, aError);
 450
 451         return CreateDecoderPromise::CreateAndReject(aError, __func__);
 452       });
 453 }
 454
 455 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {
 456   AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);
 457   auto& ownerData = aData.mOwnerData;
 458
 459   DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
 460            "initialize_decoder", DDNoValue{});
 461   aData.mDecoder->Init()
 462       ->Then(
 463           mOwner->OwnerThread(), __func__,
 464           [this, &aData, &ownerData](TrackType aTrack) {
 465             AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
 466                                 MEDIA_PLAYBACK);
 467             aData.mInitRequest.Complete();
 468             aData.mStage = Stage::None;
 469             MutexAutoLock lock(ownerData.mMutex);
 470             ownerData.mDecoder = std::move(aData.mDecoder);
 471             ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
 472             DDLOGEX2("MediaFormatReader::DecoderFactory", this,
 473                      DDLogCategory::Log, "decoder_initialized", DDNoValue{});
 474             DecoderDoctorLogger::LinkParentAndChild(
 475                 "MediaFormatReader::DecoderData", &ownerData, "decoder",
 476                 ownerData.mDecoder.get());
 477             mOwner->SetVideoDecodeThreshold();
 478             mOwner->ScheduleUpdate(aTrack);
 479             if (aTrack == TrackInfo::kVideoTrack) {
 480               DecoderBenchmark::CheckVersion(
 481                   ownerData.GetCurrentInfo()->mMimeType);
 482             }
 483             if (aTrack == TrackInfo::kAudioTrack) {
 484               ownerData.mProcessName = ownerData.mDecoder->GetProcessName();
 485               ownerData.mCodecName = ownerData.mDecoder->GetCodecName();
 486             }
 487           },
 488           [this, &aData, &ownerData](const MediaResult& aError) {
 489             AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
 490                                 MEDIA_PLAYBACK);
 491             aData.mInitRequest.Complete();
 492             MOZ_RELEASE_ASSERT(!ownerData.mDecoder,
 493                                "Can't have a decoder already set");
 494             aData.mStage = Stage::None;
 495             mOwner->mShutdownPromisePool->ShutdownDecoder(
 496                 aData.mDecoder.forget());
 497             DDLOGEX2("MediaFormatReader::DecoderFactory", this,
 498                      DDLogCategory::Log, "initialize_decoder_error", aError);
 499             mOwner->NotifyError(aData.mTrack, aError);
 500           })
 501       ->Track(aData.mInitRequest);
 502 }
 503
 504 // DemuxerProxy ensures that the original main demuxer is only ever accessed
 505 // via its own dedicated task queue.
 506 // This ensure that the reader's taskqueue will never blocked while a demuxer
 507 // is itself blocked attempting to access the MediaCache or the MediaResource.
 508 class MediaFormatReader::DemuxerProxy {
 509   using TrackType = TrackInfo::TrackType;
 510   class Wrapper;
 511
 512  public:
 513   explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)
 514       : mTaskQueue(TaskQueue::Create(
 515             GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
 516             "DemuxerProxy::mTaskQueue")),
 517         mData(new Data(aDemuxer)) {
 518     MOZ_COUNT_CTOR(DemuxerProxy);
 519   }
 520
 521   MOZ_COUNTED_DTOR(DemuxerProxy)
 522
 523   RefPtr<ShutdownPromise> Shutdown() {
 524     RefPtr<Data> data = std::move(mData);
 525     return InvokeAsync(mTaskQueue, __func__, [data]() {
 526       // We need to clear our reference to the demuxer now. So that in the event
 527       // the init promise wasn't resolved, such as what can happen with the
 528       // mediasource demuxer that is waiting on more data, it will force the
 529       // init promise to be rejected.
 530       data->mDemuxer = nullptr;
 531       data->mAudioDemuxer = nullptr;
 532       data->mVideoDemuxer = nullptr;
 533       return ShutdownPromise::CreateAndResolve(true, __func__);
 534     });
 535   }
 536
 537   RefPtr<MediaDataDemuxer::InitPromise> Init();
 538
 539   Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {
 540     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 541
 542     switch (aTrack) {
 543       case TrackInfo::kAudioTrack:
 544         return mData->mAudioDemuxer;
 545       case TrackInfo::kVideoTrack:
 546         return mData->mVideoDemuxer;
 547       default:
 548         return nullptr;
 549     }
 550   }
 551
 552   uint32_t GetNumberTracks(TrackType aTrack) const {
 553     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 554
 555     switch (aTrack) {
 556       case TrackInfo::kAudioTrack:
 557         return mData->mNumAudioTrack;
 558       case TrackInfo::kVideoTrack:
 559         return mData->mNumVideoTrack;
 560       default:
 561         return 0;
 562     }
 563   }
 564
 565   bool IsSeekable() const {
 566     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 567
 568     return mData->mSeekable;
 569   }
 570
 571   bool IsSeekableOnlyInBufferedRanges() const {
 572     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 573
 574     return mData->mSeekableOnlyInBufferedRange;
 575   }
 576
 577   UniquePtr<EncryptionInfo> GetCrypto() const {
 578     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 579
 580     if (!mData->mCrypto) {
 581       return nullptr;
 582     }
 583     auto crypto = MakeUnique<EncryptionInfo>();
 584     *crypto = *mData->mCrypto;
 585     return crypto;
 586   }
 587
 588   RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();
 589
 590   bool ShouldComputeStartTime() const {
 591     MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
 592
 593     return mData->mShouldComputeStartTime;
 594   }
 595
 596  private:
 597   const RefPtr<TaskQueue> mTaskQueue;
 598   struct Data {
 599     NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)
 600
 601     explicit Data(MediaDataDemuxer* aDemuxer)
 602         : mInitDone(false), mDemuxer(aDemuxer) {}
 603
 604     Atomic<bool> mInitDone;
 605     // Only ever accessed over mTaskQueue once.
 606     RefPtr<MediaDataDemuxer> mDemuxer;
 607     // Only accessed once InitPromise has been resolved and immutable after.
 608     // So we can safely access them without the use of the mutex.
 609     uint32_t mNumAudioTrack = 0;
 610     RefPtr<Wrapper> mAudioDemuxer;
 611     uint32_t mNumVideoTrack = 0;
 612     RefPtr<Wrapper> mVideoDemuxer;
 613     bool mSeekable = false;
 614     bool mSeekableOnlyInBufferedRange = false;
 615     bool mShouldComputeStartTime = true;
 616     UniquePtr<EncryptionInfo> mCrypto;
 617
 618    private:
 619     ~Data() = default;
 620   };
 621   RefPtr<Data> mData;
 622 };
 623
 624 class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {
 625  public:
 626   Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)
 627       : mMutex("TrackDemuxer Mutex"),
 628         mTaskQueue(aTaskQueue),
 629         mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),
 630         mInfo(aTrackDemuxer->GetInfo()),
 631         mTrackDemuxer(aTrackDemuxer) {
 632     DecoderDoctorLogger::LogConstructionAndBase(
 633         "MediaFormatReader::DemuxerProxy::Wrapper", this,
 634         static_cast<const MediaTrackDemuxer*>(this));
 635     DecoderDoctorLogger::LinkParentAndChild(
 636         "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
 637         aTrackDemuxer);
 638   }
 639
 640   UniquePtr<TrackInfo> GetInfo() const override {
 641     if (!mInfo) {
 642       return nullptr;
 643     }
 644     return mInfo->Clone();
 645   }
 646
 647   RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {
 648     RefPtr<Wrapper> self = this;
 649     return InvokeAsync(
 650                mTaskQueue, __func__,
 651                [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })
 652         ->Then(
 653             mTaskQueue, __func__,
 654             [self](const TimeUnit& aTime) {
 655               self->UpdateRandomAccessPoint();
 656               return SeekPromise::CreateAndResolve(aTime, __func__);
 657             },
 658             [self](const MediaResult& aError) {
 659               self->UpdateRandomAccessPoint();
 660               return SeekPromise::CreateAndReject(aError, __func__);
 661             });
 662   }
 663
 664   RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {
 665     RefPtr<Wrapper> self = this;
 666     return InvokeAsync(mTaskQueue, __func__,
 667                        [self, aNumSamples]() {
 668                          return self->mTrackDemuxer->GetSamples(aNumSamples);
 669                        })
 670         ->Then(
 671             mTaskQueue, __func__,
 672             [self](RefPtr<SamplesHolder> aSamples) {
 673               self->UpdateRandomAccessPoint();
 674               return SamplesPromise::CreateAndResolve(aSamples.forget(),
 675                                                       __func__);
 676             },
 677             [self](const MediaResult& aError) {
 678               self->UpdateRandomAccessPoint();
 679               return SamplesPromise::CreateAndReject(aError, __func__);
 680             });
 681   }
 682
 683   bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }
 684
 685   void Reset() override {
 686     RefPtr<Wrapper> self = this;
 687     nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
 688         "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
 689         [self]() { self->mTrackDemuxer->Reset(); }));
 690     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
 691     Unused << rv;
 692   }
 693
 694   nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {
 695     MutexAutoLock lock(mMutex);
 696     if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {
 697       *aTime = mNextRandomAccessPoint;
 698     }
 699     return mNextRandomAccessPointResult;
 700   }
 701
 702   RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
 703       const TimeUnit& aTimeThreshold) override {
 704     RefPtr<Wrapper> self = this;
 705     return InvokeAsync(
 706                mTaskQueue, __func__,
 707                [self, aTimeThreshold]() {
 708                  return self->mTrackDemuxer->SkipToNextRandomAccessPoint(
 709                      aTimeThreshold);
 710                })
 711         ->Then(
 712             mTaskQueue, __func__,
 713             [self](uint32_t aVal) {
 714               self->UpdateRandomAccessPoint();
 715               return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);
 716             },
 717             [self](const SkipFailureHolder& aError) {
 718               self->UpdateRandomAccessPoint();
 719               return SkipAccessPointPromise::CreateAndReject(aError, __func__);
 720             });
 721   }
 722
 723   TimeIntervals GetBuffered() override {
 724     MutexAutoLock lock(mMutex);
 725     return mBuffered;
 726   }
 727
 728   void BreakCycles() override {}
 729
 730  private:
 731   Mutex mMutex MOZ_UNANNOTATED;
 732   const RefPtr<TaskQueue> mTaskQueue;
 733   const bool mGetSamplesMayBlock;
 734   const UniquePtr<TrackInfo> mInfo;
 735   // mTrackDemuxer is only ever accessed on demuxer's task queue.
 736   RefPtr<MediaTrackDemuxer> mTrackDemuxer;
 737   // All following members are protected by mMutex
 738   nsresult mNextRandomAccessPointResult = NS_OK;
 739   TimeUnit mNextRandomAccessPoint;
 740   TimeIntervals mBuffered;
 741   friend class DemuxerProxy;
 742
 743   ~Wrapper() {
 744     RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);
 745     nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
 746         "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
 747         [trackDemuxer]() { trackDemuxer->BreakCycles(); }));
 748     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
 749     Unused << rv;
 750     DecoderDoctorLogger::LogDestruction(
 751         "MediaFormatReader::DemuxerProxy::Wrapper", this);
 752   }
 753
 754   void UpdateRandomAccessPoint() {
 755     MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 756     if (!mTrackDemuxer) {
 757       // Detached.
 758       return;
 759     }
 760     MutexAutoLock lock(mMutex);
 761     mNextRandomAccessPointResult =
 762         mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);
 763   }
 764
 765   void UpdateBuffered() {
 766     MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 767     if (!mTrackDemuxer) {
 768       // Detached.
 769       return;
 770     }
 771     MutexAutoLock lock(mMutex);
 772     mBuffered = mTrackDemuxer->GetBuffered();
 773   }
 774 };
 775
 776 RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {
 777   AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);
 778   using InitPromise = MediaDataDemuxer::InitPromise;
 779
 780   RefPtr<Data> data = mData;
 781   RefPtr<TaskQueue> taskQueue = mTaskQueue;
 782   return InvokeAsync(mTaskQueue, __func__,
 783                      [data, taskQueue]() {
 784                        if (!data->mDemuxer) {
 785                          return InitPromise::CreateAndReject(
 786                              NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 787                        }
 788                        return data->mDemuxer->Init();
 789                      })
 790       ->Then(
 791           taskQueue, __func__,
 792           [data, taskQueue]() {
 793             AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);
 794             if (!data->mDemuxer) {  // Was shutdown.
 795               return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
 796                                                   __func__);
 797             }
 798             data->mNumAudioTrack =
 799                 data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
 800             if (data->mNumAudioTrack) {
 801               RefPtr<MediaTrackDemuxer> d =
 802                   data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
 803               if (d) {
 804                 RefPtr<Wrapper> wrapper =
 805                     new DemuxerProxy::Wrapper(d, taskQueue);
 806                 wrapper->UpdateBuffered();
 807                 data->mAudioDemuxer = wrapper;
 808                 DecoderDoctorLogger::LinkParentAndChild(
 809                     data->mDemuxer.get(), "decoder factory wrapper",
 810                     "MediaFormatReader::DecoderFactory::Wrapper",
 811                     wrapper.get());
 812               }
 813             }
 814             data->mNumVideoTrack =
 815                 data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
 816             if (data->mNumVideoTrack) {
 817               RefPtr<MediaTrackDemuxer> d =
 818                   data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
 819               if (d) {
 820                 RefPtr<Wrapper> wrapper =
 821                     new DemuxerProxy::Wrapper(d, taskQueue);
 822                 wrapper->UpdateBuffered();
 823                 data->mVideoDemuxer = wrapper;
 824                 DecoderDoctorLogger::LinkParentAndChild(
 825                     data->mDemuxer.get(), "decoder factory wrapper",
 826                     "MediaFormatReader::DecoderFactory::Wrapper",
 827                     wrapper.get());
 828               }
 829             }
 830             data->mCrypto = data->mDemuxer->GetCrypto();
 831             data->mSeekable = data->mDemuxer->IsSeekable();
 832             data->mSeekableOnlyInBufferedRange =
 833                 data->mDemuxer->IsSeekableOnlyInBufferedRanges();
 834             data->mShouldComputeStartTime =
 835                 data->mDemuxer->ShouldComputeStartTime();
 836             data->mInitDone = true;
 837             return InitPromise::CreateAndResolve(NS_OK, __func__);
 838           },
 839           [](const MediaResult& aError) {
 840             return InitPromise::CreateAndReject(aError, __func__);
 841           });
 842 }
 843
 844 RefPtr<MediaFormatReader::NotifyDataArrivedPromise>
 845 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
 846   RefPtr<Data> data = mData;
 847   return InvokeAsync(mTaskQueue, __func__, [data]() {
 848     if (!data->mDemuxer) {
 849       // Was shutdown.
 850       return NotifyDataArrivedPromise::CreateAndReject(
 851           NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 852     }
 853     data->mDemuxer->NotifyDataArrived();
 854     if (data->mAudioDemuxer) {
 855       data->mAudioDemuxer->UpdateBuffered();
 856     }
 857     if (data->mVideoDemuxer) {
 858       data->mVideoDemuxer->UpdateBuffered();
 859     }
 860     return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);
 861   });
 862 }
 863
 864 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
 865                                      MediaDataDemuxer* aDemuxer)
 866     : mTaskQueue(
 867           TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
 868                             "MediaFormatReader::mTaskQueue",
 869                             /* aSupportsTailDispatch = */ true)),
 870       mAudio(this, MediaData::Type::AUDIO_DATA,
 871              StaticPrefs::media_audio_max_decode_error()),
 872       mVideo(this, MediaData::Type::VIDEO_DATA,
 873              StaticPrefs::media_video_max_decode_error()),
 874       mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),
 875       mWatchManager(this, OwnerThread()),
 876       mIsWatchingWorkingInfo(false),
 877       mDemuxer(new DemuxerProxy(aDemuxer)),
 878       mDemuxerInitDone(false),
 879       mPendingNotifyDataArrived(false),
 880       mLastReportedNumDecodedFrames(0),
 881       mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),
 882       mKnowsCompositor(aInit.mKnowsCompositor),
 883       mInitDone(false),
 884       mTrackDemuxersMayBlock(false),
 885       mSeekScheduled(false),
 886       mVideoFrameContainer(aInit.mVideoFrameContainer),
 887       mCrashHelper(aInit.mCrashHelper),
 888       mDecoderFactory(new DecoderFactory(this)),
 889       mShutdownPromisePool(new ShutdownPromisePool()),
 890       mBuffered(mTaskQueue, TimeIntervals(),
 891                 "MediaFormatReader::mBuffered (Canonical)"),
 892       mFrameStats(aInit.mFrameStats),
 893       mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID),
 894       mTrackingId(std::move(aInit.mTrackingId)) {
 895   MOZ_ASSERT(aDemuxer);
 896   MOZ_COUNT_CTOR(MediaFormatReader);
 897   DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
 898               &mAudio);
 899   DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
 900               &mVideo);
 901   DDLINKCHILD("demuxer", aDemuxer);
 902   mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(
 903       mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);
 904 }
 905
 906 MediaFormatReader::~MediaFormatReader() {
 907   MOZ_COUNT_DTOR(MediaFormatReader);
 908   MOZ_ASSERT(mShutdown);
 909 }
 910
 911 RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
 912   MOZ_ASSERT(OnTaskQueue());
 913   LOG("");
 914
 915   mDemuxerInitRequest.DisconnectIfExists();
 916   mNotifyDataArrivedPromise.DisconnectIfExists();
 917   mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 918   mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 919   mSkipRequest.DisconnectIfExists();
 920   mSetCDMPromise.RejectIfExists(
 921       MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
 922                   "MediaFormatReader is shutting down"),
 923       __func__);
 924
 925   if (mIsWatchingWorkingInfo) {
 926     mWatchManager.Unwatch(mWorkingInfoChanged,
 927                           &MediaFormatReader::NotifyTrackInfoUpdated);
 928   }
 929   mWatchManager.Shutdown();
 930
 931   if (mAudio.HasPromise()) {
 932     mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 933   }
 934   if (mVideo.HasPromise()) {
 935     mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 936   }
 937
 938   if (HasAudio()) {
 939     mAudio.ResetDemuxer();
 940     mAudio.mTrackDemuxer->BreakCycles();
 941     {
 942       MutexAutoLock lock(mAudio.mMutex);
 943       mAudio.mTrackDemuxer = nullptr;
 944     }
 945     mAudio.ResetState();
 946     ShutdownDecoder(TrackInfo::kAudioTrack);
 947   }
 948
 949   if (HasVideo()) {
 950     mVideo.ResetDemuxer();
 951     mVideo.mTrackDemuxer->BreakCycles();
 952     {
 953       MutexAutoLock lock(mVideo.mMutex);
 954       mVideo.mTrackDemuxer = nullptr;
 955     }
 956     mVideo.ResetState();
 957     ShutdownDecoder(TrackInfo::kVideoTrack);
 958   }
 959
 960   mShutdownPromisePool->Track(mDemuxer->Shutdown());
 961   mDemuxer = nullptr;
 962
 963   mOnTrackWaitingForKeyListener.Disconnect();
 964
 965   mShutdown = true;
 966   return mShutdownPromisePool->Shutdown()->Then(
 967       OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,
 968       &MediaFormatReader::TearDownDecoders);
 969 }
 970
 971 void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {
 972   LOGV("%s", TrackTypeToStr(aTrack));
 973
 974   // Shut down the pending decoder if any.
 975   mDecoderFactory->ShutdownDecoder(aTrack);
 976
 977   auto& decoder = GetDecoderData(aTrack);
 978   // Flush the decoder if necessary.
 979   decoder.Flush();
 980
 981   // Shut down the decoder if any.
 982   decoder.ShutdownDecoder();
 983 }
 984
 985 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
 986   MOZ_ASSERT(OnTaskQueue());
 987   if (!StaticPrefs::media_mediacapabilities_from_database()) {
 988     return;
 989   }
 990   auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);
 991   if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {
 992     VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());
 993     info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));
 994     mOnStoreDecoderBenchmark.Notify(std::move(info));
 995   }
 996 }
 997
 998 void MediaFormatReader::NotifyTrackInfoUpdated() {
 999   MOZ_ASSERT(OnTaskQueue());
1000   if (mWorkingInfoChanged) {
1001     mWorkingInfoChanged = false;
1002
1003     VideoInfo videoInfo;
1004     AudioInfo audioInfo;
1005     {
1006       MutexAutoLock lock(mVideo.mMutex);
1007       if (HasVideo()) {
1008         videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1009       }
1010     }
1011     {
1012       MutexAutoLock lock(mAudio.mMutex);
1013       if (HasAudio()) {
1014         audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1015       }
1016     }
1017
1018     mTrackInfoUpdatedEvent.Notify(videoInfo, audioInfo);
1019   }
1020 }
1021
1022 RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
1023   if (mAudio.mTaskQueue) {
1024     mAudio.mTaskQueue->BeginShutdown();
1025     mAudio.mTaskQueue->AwaitShutdownAndIdle();
1026     mAudio.mTaskQueue = nullptr;
1027   }
1028   if (mVideo.mTaskQueue) {
1029     mVideo.mTaskQueue->BeginShutdown();
1030     mVideo.mTaskQueue->AwaitShutdownAndIdle();
1031     mVideo.mTaskQueue = nullptr;
1032   }
1033
1034   mDecoderFactory = nullptr;
1035   mVideoFrameContainer = nullptr;
1036
1037   ReleaseResources();
1038   mBuffered.DisconnectAll();
1039   return mTaskQueue->BeginShutdown();
1040 }
1041
1042 nsresult MediaFormatReader::Init() {
1043   MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
1044
1045   mAudio.mTaskQueue =
1046       TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1047                         "MFR::mAudio::mTaskQueue");
1048
1049   mVideo.mTaskQueue =
1050       TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1051                         "MFR::mVideo::mTaskQueue");
1052
1053   return NS_OK;
1054 }
1055
1056 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {
1057   // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
1058   // and would create new one for specific track in the next Update.
1059   MOZ_ASSERT(OnTaskQueue());
1060
1061   if (mSetCDMPromise.IsEmpty()) {
1062     return true;
1063   }
1064
1065   MOZ_ASSERT(mCDMProxy);
1066   if (mSetCDMForTracks.contains(aTrack)) {
1067     mSetCDMForTracks -= aTrack;
1068   }
1069
1070   if (mSetCDMForTracks.isEmpty()) {
1071     LOGV("%s : Done ", __func__);
1072     mSetCDMPromise.Resolve(/* aResolveValue = */ true, __func__);
1073     if (HasAudio()) {
1074       ScheduleUpdate(TrackInfo::kAudioTrack);
1075     }
1076     if (HasVideo()) {
1077       ScheduleUpdate(TrackInfo::kVideoTrack);
1078     }
1079     return true;
1080   }
1081   LOGV("%s : %s track is ready.", __func__, TrackTypeToStr(aTrack));
1082   return false;
1083 }
1084
1085 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack) {
1086   MOZ_ASSERT(OnTaskQueue());
1087   LOGV("%s : %s", __func__, TrackTypeToStr(aTrack));
1088
1089   mSetCDMForTracks += aTrack;
1090   if (mCDMProxy) {
1091     // An old cdm proxy exists, so detaching old cdm proxy by shutting down
1092     // MediaDataDecoder.
1093     ShutdownDecoder(aTrack);
1094   }
1095   ScheduleUpdate(aTrack);
1096 }
1097
1098 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack) {
1099   MOZ_ASSERT(OnTaskQueue());
1100   return GetDecoderData(aTrack).IsEncrypted() &&
1101          mSetCDMForTracks.contains(aTrack) && !mCDMProxy;
1102 }
1103
1104 RefPtr<SetCDMPromise> MediaFormatReader::SetCDMProxy(CDMProxy* aProxy) {
1105   MOZ_ASSERT(OnTaskQueue());
1106   LOGV("SetCDMProxy (%p)", aProxy);
1107
1108   if (mShutdown) {
1109     return SetCDMPromise::CreateAndReject(
1110         MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1111                     "MediaFormatReader is shutting down"),
1112         __func__);
1113   }
1114
1115   mSetCDMPromise.RejectIfExists(
1116       MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1117                   "Another new CDM proxy is being set."),
1118       __func__);
1119
1120   // Shutdown all decoders as switching CDM proxy indicates that it's
1121   // inappropriate for the existing decoders to continue decoding via the old
1122   // CDM proxy.
1123   if (HasAudio()) {
1124     PrepareToSetCDMForTrack(TrackInfo::kAudioTrack);
1125   }
1126   if (HasVideo()) {
1127     PrepareToSetCDMForTrack(TrackInfo::kVideoTrack);
1128   }
1129
1130   mCDMProxy = aProxy;
1131
1132   if (!mInitDone || mSetCDMForTracks.isEmpty() || !mCDMProxy) {
1133     // 1) MFR is not initialized yet or
1134     // 2) Demuxer is initialized without active audio and video or
1135     // 3) A null cdm proxy is set
1136     // the promise can be resolved directly.
1137     mSetCDMForTracks.clear();
1138     return SetCDMPromise::CreateAndResolve(/* aResolveValue = */ true,
1139                                            __func__);
1140   }
1141
1142   RefPtr<SetCDMPromise> p = mSetCDMPromise.Ensure(__func__);
1143   return p;
1144 }
1145
1146 bool MediaFormatReader::IsWaitingOnCDMResource() {
1147   MOZ_ASSERT(OnTaskQueue());
1148   return IsEncrypted() && !mCDMProxy;
1149 }
1150
1151 RefPtr<MediaFormatReader::MetadataPromise>
1152 MediaFormatReader::AsyncReadMetadata() {
1153   AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK);
1154   MOZ_ASSERT(OnTaskQueue());
1155
1156   MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
1157
1158   if (mInitDone) {
1159     // We are returning from dormant.
1160     MetadataHolder metadata;
1161     metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1162     return MetadataPromise::CreateAndResolve(std::move(metadata), __func__);
1163   }
1164
1165   RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
1166
1167   mDemuxer->Init()
1168       ->Then(OwnerThread(), __func__, this,
1169              &MediaFormatReader::OnDemuxerInitDone,
1170              &MediaFormatReader::OnDemuxerInitFailed)
1171       ->Track(mDemuxerInitRequest);
1172   return p;
1173 }
1174
1175 void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
1176   AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK);
1177   MOZ_ASSERT(OnTaskQueue());
1178   mDemuxerInitRequest.Complete();
1179
1180   if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1181     mMetadataPromise.Reject(aResult, __func__);
1182     return;
1183   }
1184
1185   mDemuxerInitDone = true;
1186
1187   UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
1188
1189   RefPtr<PDMFactory> platform;
1190   if (!IsWaitingOnCDMResource()) {
1191     platform = new PDMFactory();
1192   }
1193
1194   // To decode, we need valid video and a place to put it.
1195   bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
1196                      GetImageContainer();
1197
1198   if (videoActive) {
1199     // We currently only handle the first video track.
1200     MutexAutoLock lock(mVideo.mMutex);
1201     mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1202     if (!mVideo.mTrackDemuxer) {
1203       mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1204       return;
1205     }
1206
1207     UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
1208     videoActive = videoInfo && videoInfo->IsValid();
1209     if (videoActive) {
1210       if (platform &&
1211           platform->SupportsMimeType(videoInfo->mMimeType).isEmpty()) {
1212         // We have no decoder for this track. Error.
1213         mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1214         return;
1215       }
1216       mInfo.mVideo = *videoInfo->GetAsVideoInfo();
1217       mVideo.mWorkingInfo = MakeUnique<VideoInfo>(mInfo.mVideo);
1218       for (const MetadataTag& tag : videoInfo->mTags) {
1219         tags->InsertOrUpdate(tag.mKey, tag.mValue);
1220       }
1221       mWorkingInfoChanged = true;
1222       mVideo.mOriginalInfo = std::move(videoInfo);
1223       mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
1224     } else {
1225       mVideo.mTrackDemuxer->BreakCycles();
1226       mVideo.mTrackDemuxer = nullptr;
1227     }
1228   }
1229
1230   bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1231   if (audioActive) {
1232     MutexAutoLock lock(mAudio.mMutex);
1233     mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1234     if (!mAudio.mTrackDemuxer) {
1235       mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1236       return;
1237     }
1238
1239     UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
1240     // We actively ignore audio tracks that we know we can't play.
1241     audioActive = audioInfo && audioInfo->IsValid() &&
1242                   (!platform ||
1243                    !platform->SupportsMimeType(audioInfo->mMimeType).isEmpty());
1244
1245     if (audioActive) {
1246       mInfo.mAudio = *audioInfo->GetAsAudioInfo();
1247       mAudio.mWorkingInfo = MakeUnique<AudioInfo>(mInfo.mAudio);
1248       for (const MetadataTag& tag : audioInfo->mTags) {
1249         tags->InsertOrUpdate(tag.mKey, tag.mValue);
1250       }
1251       mWorkingInfoChanged = true;
1252       mAudio.mOriginalInfo = std::move(audioInfo);
1253       mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
1254     } else {
1255       mAudio.mTrackDemuxer->BreakCycles();
1256       mAudio.mTrackDemuxer = nullptr;
1257     }
1258   }
1259
1260   UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
1261   if (crypto && crypto->IsEncrypted()) {
1262     // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1263     for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
1264       mOnEncrypted.Notify(crypto->mInitDatas[i].mInitData,
1265                           crypto->mInitDatas[i].mType);
1266     }
1267     mInfo.mCrypto = *crypto;
1268   }
1269
1270   auto videoDuration = HasVideo() ? mInfo.mVideo.mDuration : TimeUnit::Zero();
1271   auto audioDuration = HasAudio() ? mInfo.mAudio.mDuration : TimeUnit::Zero();
1272
1273   // If the duration is 0 on both audio and video, it mMetadataDuration is to be
1274   // Nothing(). Duration will use buffered ranges.
1275   if (videoDuration.IsPositive() || audioDuration.IsPositive()) {
1276     auto duration = std::max(videoDuration, audioDuration);
1277     mInfo.mMetadataDuration = Some(duration);
1278   }
1279
1280   mInfo.mMediaSeekable = mDemuxer->IsSeekable();
1281   mInfo.mMediaSeekableOnlyInBufferedRanges =
1282       mDemuxer->IsSeekableOnlyInBufferedRanges();
1283
1284   if (!videoActive && !audioActive) {
1285     mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1286     return;
1287   }
1288
1289   mTags = std::move(tags);
1290   mInitDone = true;
1291
1292   // Try to get the start time.
1293   // For MSE case, the start time of each track is assumed to be 0.
1294   // For others, we must demux the first sample to know the start time for each
1295   // track.
1296   if (!mDemuxer->ShouldComputeStartTime()) {
1297     mAudio.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1298     mVideo.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1299   } else {
1300     if (HasAudio()) {
1301       RequestDemuxSamples(TrackInfo::kAudioTrack);
1302     }
1303
1304     if (HasVideo()) {
1305       RequestDemuxSamples(TrackInfo::kVideoTrack);
1306     }
1307   }
1308
1309   if (aResult != NS_OK) {
1310     mOnDecodeWarning.Notify(aResult);
1311   }
1312
1313   MaybeResolveMetadataPromise();
1314 }
1315
1316 void MediaFormatReader::MaybeResolveMetadataPromise() {
1317   MOZ_ASSERT(OnTaskQueue());
1318
1319   if ((HasAudio() && mAudio.mFirstDemuxedSampleTime.isNothing()) ||
1320       (HasVideo() && mVideo.mFirstDemuxedSampleTime.isNothing())) {
1321     return;
1322   }
1323
1324   TimeUnit startTime =
1325       std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
1326                mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
1327
1328   if (!startTime.IsInfinite()) {
1329     mInfo.mStartTime = startTime;  // mInfo.mStartTime is initialized to 0.
1330   }
1331
1332   MetadataHolder metadata;
1333   metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1334   metadata.mTags = mTags->Count() ? std::move(mTags) : nullptr;
1335
1336   // We now have all the informations required to calculate the initial buffered
1337   // range.
1338   mHasStartTime = true;
1339   UpdateBuffered();
1340
1341   mWatchManager.Watch(mWorkingInfoChanged,
1342                       &MediaFormatReader::NotifyTrackInfoUpdated);
1343   mIsWatchingWorkingInfo = true;
1344
1345   mMetadataPromise.Resolve(std::move(metadata), __func__);
1346 }
1347
1348 bool MediaFormatReader::IsEncrypted() const {
1349   return (HasAudio() && mAudio.GetCurrentInfo()->mCrypto.IsEncrypted()) ||
1350          (HasVideo() && mVideo.GetCurrentInfo()->mCrypto.IsEncrypted());
1351 }
1352
1353 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError) {
1354   mDemuxerInitRequest.Complete();
1355   mMetadataPromise.Reject(aError, __func__);
1356 }
1357
1358 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) {
1359   // Called on the MDSM's TaskQueue.
1360   {
1361     MutexAutoLock lock(mVideo.mMutex);
1362     if (HasVideo()) {
1363       aInfo->mVideo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1364     }
1365   }
1366   {
1367     MutexAutoLock lock(mAudio.mMutex);
1368     if (HasAudio()) {
1369       aInfo->mAudio = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1370       Maybe<nsCString> audioProcessPerCodecName = GetAudioProcessPerCodec();
1371       if (audioProcessPerCodecName.isSome()) {
1372         Telemetry::ScalarAdd(
1373             Telemetry::ScalarID::MEDIA_AUDIO_PROCESS_PER_CODEC_NAME,
1374             NS_ConvertUTF8toUTF16(*audioProcessPerCodecName), 1);
1375       }
1376     }
1377   }
1378 }
1379
1380 MediaFormatReader::DecoderData& MediaFormatReader::GetDecoderData(
1381     TrackType aTrack) {
1382   MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
1383              aTrack == TrackInfo::kVideoTrack);
1384   if (aTrack == TrackInfo::kAudioTrack) {
1385     return mAudio;
1386   }
1387   return mVideo;
1388 }
1389
1390 Maybe<TimeUnit> MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold,
1391                                               bool aRequestNextVideoKeyFrame) {
1392   MOZ_ASSERT(OnTaskQueue());
1393   MOZ_ASSERT(HasVideo());
1394
1395   if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1396     return Nothing();
1397   }
1398
1399   // Ensure we have no pending seek going as skip-to-keyframe could return out
1400   // of date information.
1401   if (mVideo.HasInternalSeekPending()) {
1402     return Nothing();
1403   }
1404
1405   TimeUnit nextKeyframe;
1406   nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
1407   if (NS_FAILED(rv)) {
1408     // Only OggTrackDemuxer with video type gets into here.
1409     // We don't support skip-to-next-frame for this case.
1410     return Nothing();
1411   }
1412
1413   const bool isNextKeyframeValid =
1414       nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
1415   // If we request the next keyframe, only return times greater than
1416   // aTimeThreshold. Otherwise, data will be already behind the threshold and
1417   // will be eventually discarded somewhere in the media pipeline.
1418   if (aRequestNextVideoKeyFrame && isNextKeyframeValid &&
1419       nextKeyframe > aTimeThreshold) {
1420     return Some(nextKeyframe);
1421   }
1422
1423   const bool isNextVideoBehindTheThreshold =
1424       (isNextKeyframeValid && nextKeyframe <= aTimeThreshold) ||
1425       GetInternalSeekTargetEndTime() < aTimeThreshold;
1426   return isNextVideoBehindTheThreshold ? Some(aTimeThreshold) : Nothing();
1427 }
1428
1429 RefPtr<MediaFormatReader::VideoDataPromise> MediaFormatReader::RequestVideoData(
1430     const TimeUnit& aTimeThreshold, bool aRequestNextVideoKeyFrame) {
1431   MOZ_ASSERT(OnTaskQueue());
1432   MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
1433   // Requesting video can be done independently from audio, even during audio
1434   // seeking. But it shouldn't happen if we're doing video seek.
1435   if (!IsAudioOnlySeeking()) {
1436     MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1437                           "No sample requests allowed while seeking");
1438     MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
1439                           mVideo.mTimeThreshold.isSome());
1440     MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1441   }
1442   LOGV("RequestVideoData(%" PRId64 "), requestNextKeyFrame=%d",
1443        aTimeThreshold.ToMicroseconds(), aRequestNextVideoKeyFrame);
1444
1445   if (!HasVideo()) {
1446     LOG("called with no video track");
1447     return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1448                                              __func__);
1449   }
1450
1451   if (IsSeeking()) {
1452     LOG("called mid-seek. Rejecting.");
1453     return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1454                                              __func__);
1455   }
1456
1457   if (mShutdown) {
1458     NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1459     return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1460                                              __func__);
1461   }
1462
1463   if (Maybe<TimeUnit> target =
1464           ShouldSkip(aTimeThreshold, aRequestNextVideoKeyFrame)) {
1465     PROFILER_MARKER_UNTYPED("RequestVideoData SkipVideoDemuxToNextKeyFrame",
1466                             MEDIA_PLAYBACK);
1467     RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1468     SkipVideoDemuxToNextKeyFrame(*target);
1469     return p;
1470   }
1471
1472   RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1473   ScheduleUpdate(TrackInfo::kVideoTrack);
1474
1475   return p;
1476 }
1477
1478 void MediaFormatReader::OnDemuxFailed(TrackType aTrack,
1479                                       const MediaResult& aError) {
1480   AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK);
1481   MOZ_ASSERT(OnTaskQueue());
1482   LOG("Failed to demux %s, failure:%s",
1483       aTrack == TrackType::kVideoTrack ? "video" : "audio",
1484       aError.ErrorName().get());
1485   auto& decoder = GetDecoderData(aTrack);
1486   decoder.mDemuxRequest.Complete();
1487   switch (aError.Code()) {
1488     case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1489       DDLOG(DDLogCategory::Log,
1490             aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1491                                              : "audio_demux_interruption",
1492             aError);
1493       if (!decoder.mWaitingForData) {
1494         decoder.RequestDrain();
1495       }
1496       NotifyEndOfStream(aTrack);
1497       break;
1498     case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1499       DDLOG(DDLogCategory::Log,
1500             aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1501                                              : "audio_demux_interruption",
1502             aError);
1503       if (!decoder.mWaitingForData) {
1504         decoder.RequestDrain();
1505       }
1506       NotifyWaitingForData(aTrack);
1507       break;
1508     case NS_ERROR_DOM_MEDIA_CANCELED:
1509       DDLOG(DDLogCategory::Log,
1510             aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1511                                              : "audio_demux_interruption",
1512             aError);
1513       if (decoder.HasPromise()) {
1514         decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
1515       }
1516       break;
1517     default:
1518       DDLOG(DDLogCategory::Log,
1519             aTrack == TrackType::kVideoTrack ? "video_demux_error"
1520                                              : "audio_demux_error",
1521             aError);
1522       NotifyError(aTrack, aError);
1523       break;
1524   }
1525 }
1526
1527 void MediaFormatReader::DoDemuxVideo() {
1528   AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK);
1529   using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1530
1531   DDLOG(DDLogCategory::Log, "video_demuxing", DDNoValue{});
1532   PerformanceRecorder<PlaybackStage> perfRecorder(
1533       MediaStage::RequestDemux,
1534       mVideo.GetCurrentInfo()->GetAsVideoInfo()->mImage.height);
1535   auto p = mVideo.mTrackDemuxer->GetSamples(1);
1536
1537   RefPtr<MediaFormatReader> self = this;
1538   if (mVideo.mFirstDemuxedSampleTime.isNothing()) {
1539     p = p->Then(
1540         OwnerThread(), __func__,
1541         [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1542           AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1543                               MEDIA_PLAYBACK);
1544           DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxed",
1545                   DDNoValue{});
1546           self->OnFirstDemuxCompleted(TrackInfo::kVideoTrack, aSamples);
1547           return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1548         },
1549         [self](const MediaResult& aError) {
1550           AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1551                               MEDIA_PLAYBACK);
1552           DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxing_error",
1553                   aError);
1554           self->OnFirstDemuxFailed(TrackInfo::kVideoTrack, aError);
1555           return SamplesPromise::CreateAndReject(aError, __func__);
1556         });
1557   }
1558
1559   p->Then(
1560        OwnerThread(), __func__,
1561        [self, perfRecorder(std::move(perfRecorder))](
1562            const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) mutable {
1563          perfRecorder.Record();
1564          self->OnVideoDemuxCompleted(aSamples);
1565        },
1566        [self](const MediaResult& aError) { self->OnVideoDemuxFailed(aError); })
1567       ->Track(mVideo.mDemuxRequest);
1568 }
1569
1570 void MediaFormatReader::OnVideoDemuxCompleted(
1571     const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) {
1572   AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1573                       MEDIA_PLAYBACK);
1574   LOGV("%zu video samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1575        aSamples->GetSamples()[0]->mTrackInfo
1576            ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1577            : 0);
1578   DDLOG(DDLogCategory::Log, "video_demuxed_samples",
1579         uint64_t(aSamples->GetSamples().Length()));
1580   mVideo.mDemuxRequest.Complete();
1581   mVideo.mQueuedSamples.AppendElements(aSamples->GetSamples());
1582   ScheduleUpdate(TrackInfo::kVideoTrack);
1583 }
1584
1585 RefPtr<MediaFormatReader::AudioDataPromise>
1586 MediaFormatReader::RequestAudioData() {
1587   MOZ_ASSERT(OnTaskQueue());
1588   MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
1589   // Requesting audio can be done independently from video, even during video
1590   // seeking. But it shouldn't happen if we're doing audio seek.
1591   if (!IsVideoOnlySeeking()) {
1592     MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1593                           "No sample requests allowed while seeking");
1594     MOZ_DIAGNOSTIC_ASSERT(!mAudio.mSeekRequest.Exists() ||
1595                           mAudio.mTimeThreshold.isSome());
1596     MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1597   }
1598   LOGV("");
1599
1600   if (!HasAudio()) {
1601     LOG("called with no audio track");
1602     return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1603                                              __func__);
1604   }
1605
1606   if (IsSeeking()) {
1607     LOG("called mid-seek. Rejecting.");
1608     return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1609                                              __func__);
1610   }
1611
1612   if (mShutdown) {
1613     NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1614     return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1615                                              __func__);
1616   }
1617
1618   RefPtr<AudioDataPromise> p = mAudio.EnsurePromise(__func__);
1619   ScheduleUpdate(TrackInfo::kAudioTrack);
1620
1621   return p;
1622 }
1623
1624 void MediaFormatReader::DoDemuxAudio() {
1625   AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK);
1626   using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1627
1628   DDLOG(DDLogCategory::Log, "audio_demuxing", DDNoValue{});
1629   PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestDemux);
1630   auto p = mAudio.mTrackDemuxer->GetSamples(1);
1631
1632   RefPtr<MediaFormatReader> self = this;
1633   if (mAudio.mFirstDemuxedSampleTime.isNothing()) {
1634     p = p->Then(
1635         OwnerThread(), __func__,
1636         [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1637           AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1638                               MEDIA_PLAYBACK);
1639           DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxed",
1640                   DDNoValue{});
1641           self->OnFirstDemuxCompleted(TrackInfo::kAudioTrack, aSamples);
1642           return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1643         },
1644         [self](const MediaResult& aError) {
1645           AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1646                               MEDIA_PLAYBACK);
1647           DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxing_error",
1648                   aError);
1649           self->OnFirstDemuxFailed(TrackInfo::kAudioTrack, aError);
1650           return SamplesPromise::CreateAndReject(aError, __func__);
1651         });
1652   }
1653
1654   p->Then(
1655        OwnerThread(), __func__,
1656        [self, perfRecorder(std::move(perfRecorder))](
1657            const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) mutable {
1658          perfRecorder.Record();
1659          self->OnAudioDemuxCompleted(aSamples);
1660        },
1661        [self](const MediaResult& aError) { self->OnAudioDemuxFailed(aError); })
1662       ->Track(mAudio.mDemuxRequest);
1663 }
1664
1665 void MediaFormatReader::OnAudioDemuxCompleted(
1666     const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) {
1667   LOGV("%zu audio samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1668        aSamples->GetSamples()[0]->mTrackInfo
1669            ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1670            : 0);
1671   DDLOG(DDLogCategory::Log, "audio_demuxed_samples",
1672         uint64_t(aSamples->GetSamples().Length()));
1673   mAudio.mDemuxRequest.Complete();
1674   mAudio.mQueuedSamples.AppendElements(aSamples->GetSamples());
1675   ScheduleUpdate(TrackInfo::kAudioTrack);
1676 }
1677
1678 void MediaFormatReader::NotifyNewOutput(
1679     TrackType aTrack, MediaDataDecoder::DecodedData&& aResults) {
1680   AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK);
1681   MOZ_ASSERT(OnTaskQueue());
1682   auto& decoder = GetDecoderData(aTrack);
1683   if (aResults.IsEmpty()) {
1684     DDLOG(DDLogCategory::Log,
1685           aTrack == TrackInfo::kAudioTrack ? "decoded_audio" : "decoded_video",
1686           "no output samples");
1687   } else {
1688     for (auto&& sample : aResults) {
1689       if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1690         switch (sample->mType) {
1691           case MediaData::Type::AUDIO_DATA:
1692             DDLOGPR(DDLogCategory::Log,
1693                     aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1694                                                      : "decoded_got_audio!?",
1695                     "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1696                     ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1697                     ", \"duration_us\":%" PRIi64 ", \"frames\":%" PRIu32
1698                     ", \"channels\":%" PRIu32 ", \"rate\":%" PRIu32
1699                     ", \"bytes\":%zu}",
1700                     sample->mOffset, sample->mTime.ToMicroseconds(),
1701                     sample->mTimecode.ToMicroseconds(),
1702                     sample->mDuration.ToMicroseconds(),
1703                     sample->As<AudioData>()->Frames(),
1704                     sample->As<AudioData>()->mChannels,
1705                     sample->As<AudioData>()->mRate,
1706                     sample->As<AudioData>()->Data().Length());
1707             break;
1708           case MediaData::Type::VIDEO_DATA:
1709             DDLOGPR(DDLogCategory::Log,
1710                     aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1711                                                      : "decoded_got_video!?",
1712                     "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1713                     ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1714                     ", \"duration_us\":%" PRIi64
1715                     ", \"kf\":%s, \"size\":[%" PRIi32 ",%" PRIi32 "]}",
1716                     sample->mOffset, sample->mTime.ToMicroseconds(),
1717                     sample->mTimecode.ToMicroseconds(),
1718                     sample->mDuration.ToMicroseconds(),
1719                     sample->mKeyframe ? "true" : "false",
1720                     sample->As<VideoData>()->mDisplay.width,
1721                     sample->As<VideoData>()->mDisplay.height);
1722             break;
1723           case MediaData::Type::RAW_DATA:
1724             DDLOGPR(DDLogCategory::Log,
1725                     aTrack == TrackInfo::kAudioTrack   ? "decoded_audio"
1726                     : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1727                                                        : "decoded_?",
1728                     "{\"type\":\"RawData\", \"offset\":%" PRIi64
1729                     " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1730                     ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1731                     sample->mOffset, sample->mTime.ToMicroseconds(),
1732                     sample->mTimecode.ToMicroseconds(),
1733                     sample->mDuration.ToMicroseconds(),
1734                     sample->mKeyframe ? "true" : "false");
1735             break;
1736           case MediaData::Type::NULL_DATA:
1737             DDLOGPR(DDLogCategory::Log,
1738                     aTrack == TrackInfo::kAudioTrack   ? "decoded_audio"
1739                     : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1740                                                        : "decoded_?",
1741                     "{\"type\":\"NullData\", \"offset\":%" PRIi64
1742                     " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1743                     ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1744                     sample->mOffset, sample->mTime.ToMicroseconds(),
1745                     sample->mTimecode.ToMicroseconds(),
1746                     sample->mDuration.ToMicroseconds(),
1747                     sample->mKeyframe ? "true" : "false");
1748             break;
1749         }
1750       }
1751       LOGV("Received new %s sample time:%" PRId64 " duration:%" PRId64,
1752            TrackTypeToStr(aTrack), sample->mTime.ToMicroseconds(),
1753            sample->mDuration.ToMicroseconds());
1754       decoder.mOutput.AppendElement(sample);
1755       decoder.mNumSamplesOutput++;
1756       decoder.mNumOfConsecutiveDecodingError = 0;
1757       decoder.mNumOfConsecutiveRDDOrGPUCrashes = 0;
1758       if (aTrack == TrackInfo::kAudioTrack) {
1759         decoder.mNumOfConsecutiveUtilityCrashes = 0;
1760       }
1761     }
1762   }
1763   LOG("Done processing new %s samples", TrackTypeToStr(aTrack));
1764
1765   if (!aResults.IsEmpty()) {
1766     // We have decoded our first frame, we can now starts to skip future errors.
1767     decoder.mFirstFrameTime.reset();
1768   }
1769   ScheduleUpdate(aTrack);
1770 }
1771
1772 void MediaFormatReader::NotifyError(TrackType aTrack,
1773                                     const MediaResult& aError) {
1774   MOZ_ASSERT(OnTaskQueue());
1775   NS_WARNING(aError.Description().get());
1776   LOGV("%s Decoding error", TrackTypeToStr(aTrack));
1777   auto& decoder = GetDecoderData(aTrack);
1778   decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
1779
1780   ScheduleUpdate(aTrack);
1781 }
1782
1783 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack) {
1784   MOZ_ASSERT(OnTaskQueue());
1785   auto& decoder = GetDecoderData(aTrack);
1786   decoder.mWaitingForData = true;
1787   if (decoder.mTimeThreshold) {
1788     decoder.mTimeThreshold.ref().mWaiting = true;
1789   }
1790   ScheduleUpdate(aTrack);
1791 }
1792
1793 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack) {
1794   MOZ_ASSERT(OnTaskQueue());
1795   auto& decoder = GetDecoderData(aTrack);
1796   mOnWaitingForKey.Notify();
1797   if (!decoder.mDecodeRequest.Exists()) {
1798     LOGV("WaitingForKey received while no pending decode. Ignoring");
1799     return;
1800   }
1801   decoder.mWaitingForKey = true;
1802   ScheduleUpdate(aTrack);
1803 }
1804
1805 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack) {
1806   MOZ_ASSERT(OnTaskQueue());
1807   auto& decoder = GetDecoderData(aTrack);
1808   decoder.mDemuxEOS = true;
1809   ScheduleUpdate(aTrack);
1810 }
1811
1812 bool MediaFormatReader::NeedInput(DecoderData& aDecoder) {
1813   // The decoder will not be fed a new raw sample until the current decoding
1814   // requests has completed.
1815   return (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
1816          !aDecoder.HasPendingDrain() && !aDecoder.HasFatalError() &&
1817          !aDecoder.mDemuxRequest.Exists() && !aDecoder.mOutput.Length() &&
1818          !aDecoder.HasInternalSeekPending() &&
1819          !aDecoder.mDecodeRequest.Exists();
1820 }
1821
1822 void MediaFormatReader::ScheduleUpdate(TrackType aTrack) {
1823   MOZ_ASSERT(OnTaskQueue());
1824   if (mShutdown) {
1825     return;
1826   }
1827   auto& decoder = GetDecoderData(aTrack);
1828   MOZ_RELEASE_ASSERT(decoder.GetCurrentInfo(),
1829                      "Can only schedule update when track exists");
1830
1831   if (decoder.mUpdateScheduled) {
1832     return;
1833   }
1834   LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
1835   decoder.mUpdateScheduled = true;
1836   RefPtr<nsIRunnable> task(NewRunnableMethod<TrackType>(
1837       "MediaFormatReader::Update", this, &MediaFormatReader::Update, aTrack));
1838   nsresult rv = OwnerThread()->Dispatch(task.forget());
1839   MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1840   Unused << rv;
1841 }
1842
1843 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) {
1844   MOZ_ASSERT(OnTaskQueue());
1845   auto& decoder = GetDecoderData(aTrack);
1846
1847   if (!decoder.mReceivedNewData) {
1848     return false;
1849   }
1850
1851   // We do not want to clear mWaitingForData while there are pending
1852   // demuxing or seeking operations that could affect the value of this flag.
1853   // This is in order to ensure that we will retry once they complete as we may
1854   // now have new data that could potentially allow those operations to
1855   // successfully complete if tried again.
1856   if (decoder.mSeekRequest.Exists()) {
1857     // Nothing more to do until this operation complete.
1858     return true;
1859   }
1860
1861   if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
1862     LOGV("Skipping in progress, nothing more to do");
1863     return true;
1864   }
1865
1866   if (decoder.mDemuxRequest.Exists()) {
1867     // We may have pending operations to process, so we want to continue
1868     // after UpdateReceivedNewData returns.
1869     return false;
1870   }
1871
1872   if (decoder.HasPendingDrain()) {
1873     // We do not want to clear mWaitingForData or mDemuxEOS while
1874     // a drain is in progress in order to properly complete the operation.
1875     return false;
1876   }
1877
1878   decoder.mReceivedNewData = false;
1879   if (decoder.mTimeThreshold) {
1880     decoder.mTimeThreshold.ref().mWaiting = false;
1881   }
1882   decoder.mWaitingForData = false;
1883
1884   if (decoder.HasFatalError()) {
1885     return false;
1886   }
1887
1888   if (!mSeekPromise.IsEmpty() &&
1889       (!IsVideoOnlySeeking() || aTrack == TrackInfo::kVideoTrack)) {
1890     MOZ_ASSERT(!decoder.HasPromise());
1891     MOZ_DIAGNOSTIC_ASSERT(
1892         (IsVideoOnlySeeking() || !mAudio.mTimeThreshold) &&
1893             !mVideo.mTimeThreshold,
1894         "InternalSeek must have been aborted when Seek was first called");
1895     MOZ_DIAGNOSTIC_ASSERT(
1896         (IsVideoOnlySeeking() || !mAudio.HasWaitingPromise()) &&
1897             !mVideo.HasWaitingPromise(),
1898         "Waiting promises must have been rejected when Seek was first called");
1899     if (mVideo.mSeekRequest.Exists() ||
1900         (!IsVideoOnlySeeking() && mAudio.mSeekRequest.Exists())) {
1901       // Already waiting for a seek to complete. Nothing more to do.
1902       return true;
1903     }
1904     LOG("Attempting Seek");
1905     ScheduleSeek();
1906     return true;
1907   }
1908   if (decoder.HasInternalSeekPending() || decoder.HasWaitingPromise()) {
1909     if (decoder.HasInternalSeekPending()) {
1910       LOG("Attempting Internal Seek");
1911       InternalSeek(aTrack, decoder.mTimeThreshold.ref());
1912     }
1913     if (decoder.HasWaitingPromise() && !decoder.IsWaitingForKey() &&
1914         !decoder.IsWaitingForData()) {
1915       MOZ_ASSERT(!decoder.HasPromise());
1916       LOG("We have new data. Resolving WaitingPromise");
1917       decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
1918     }
1919     return true;
1920   }
1921   return false;
1922 }
1923
1924 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack) {
1925   MOZ_ASSERT(OnTaskQueue());
1926   auto& decoder = GetDecoderData(aTrack);
1927   MOZ_ASSERT(!decoder.mDemuxRequest.Exists());
1928
1929   if (!decoder.mQueuedSamples.IsEmpty()) {
1930     // No need to demux new samples.
1931     return;
1932   }
1933
1934   if (decoder.mDemuxEOS) {
1935     // Nothing left to demux.
1936     // We do not want to attempt to demux while in waiting for data mode
1937     // as it would retrigger an unnecessary drain.
1938     return;
1939   }
1940
1941   LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
1942   if (aTrack == TrackInfo::kVideoTrack) {
1943     DoDemuxVideo();
1944   } else {
1945     DoDemuxAudio();
1946   }
1947 }
1948
1949 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
1950                                              MediaRawData* aSample) {
1951   MOZ_ASSERT(OnTaskQueue());
1952   auto& decoder = GetDecoderData(aTrack);
1953   RefPtr<MediaFormatReader> self = this;
1954   decoder.mFlushed = false;
1955   DDLOGPR(DDLogCategory::Log,
1956           aTrack == TrackInfo::kAudioTrack   ? "decode_audio"
1957           : aTrack == TrackInfo::kVideoTrack ? "decode_video"
1958                                              : "decode_?",
1959           "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1960           ", \"bytes\":%zu, \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1961           ", \"duration_us\":%" PRIi64 ",%s%s}",
1962           aSample->mOffset, aSample->Size(), aSample->mTime.ToMicroseconds(),
1963           aSample->mTimecode.ToMicroseconds(),
1964           aSample->mDuration.ToMicroseconds(), aSample->mKeyframe ? " kf" : "",
1965           aSample->mEOS ? " eos" : "");
1966
1967   const int32_t height =
1968       aTrack == TrackInfo::kVideoTrack
1969           ? decoder.GetCurrentInfo()->GetAsVideoInfo()->mImage.height
1970           : 0;
1971   MediaInfoFlag flag = MediaInfoFlag::None;
1972   flag |=
1973       aSample->mKeyframe ? MediaInfoFlag::KeyFrame : MediaInfoFlag::NonKeyFrame;
1974   if (aTrack == TrackInfo::kVideoTrack) {
1975     flag |= VideoIsHardwareAccelerated() ? MediaInfoFlag::HardwareDecoding
1976                                          : MediaInfoFlag::SoftwareDecoding;
1977     const nsCString& mimeType = decoder.GetCurrentInfo()->mMimeType;
1978     if (MP4Decoder::IsH264(mimeType)) {
1979       flag |= MediaInfoFlag::VIDEO_H264;
1980     } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP8)) {
1981       flag |= MediaInfoFlag::VIDEO_VP8;
1982     } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP9)) {
1983       flag |= MediaInfoFlag::VIDEO_VP9;
1984     }
1985 #ifdef MOZ_AV1
1986     else if (AOMDecoder::IsAV1(mimeType)) {
1987       flag |= MediaInfoFlag::VIDEO_AV1;
1988     }
1989 #endif
1990   }
1991   PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestDecode,
1992                                                   height, flag);
1993   if (mMediaEngineId && aSample->mCrypto.IsEncrypted()) {
1994     aSample->mShouldCopyCryptoToRemoteRawData = true;
1995   }
1996   decoder.mDecoder->Decode(aSample)
1997       ->Then(
1998           mTaskQueue, __func__,
1999           [self, aTrack, &decoder, perfRecorder(std::move(perfRecorder))](
2000               MediaDataDecoder::DecodedData&& aResults) mutable {
2001             perfRecorder.Record();
2002             decoder.mDecodeRequest.Complete();
2003             self->NotifyNewOutput(aTrack, std::move(aResults));
2004           },
2005           [self, aTrack, &decoder](const MediaResult& aError) {
2006             decoder.mDecodeRequest.Complete();
2007             self->NotifyError(aTrack, aError);
2008           })
2009       ->Track(decoder.mDecodeRequest);
2010 }
2011
2012 void MediaFormatReader::HandleDemuxedSamples(
2013     TrackType aTrack, FrameStatistics::AutoNotifyDecoded& aA) {
2014   MOZ_ASSERT(OnTaskQueue());
2015
2016   auto& decoder = GetDecoderData(aTrack);
2017
2018   if (decoder.mFlushing) {
2019     LOGV("Decoder operation in progress, let it complete.");
2020     return;
2021   }
2022
2023   if (decoder.mQueuedSamples.IsEmpty()) {
2024     return;
2025   }
2026
2027   RefPtr<MediaRawData> sample = decoder.mQueuedSamples[0];
2028   const RefPtr<TrackInfoSharedPtr> info = sample->mTrackInfo;
2029
2030   if (info && decoder.mLastStreamSourceID != info->GetID()) {
2031     nsTArray<RefPtr<MediaRawData>> samples;
2032     if (decoder.mDecoder) {
2033       bool recyclable =
2034           StaticPrefs::media_decoder_recycle_enabled() &&
2035           decoder.mDecoder->SupportDecoderRecycling() &&
2036           (*info)->mCrypto.mCryptoScheme ==
2037               decoder.GetCurrentInfo()->mCrypto.mCryptoScheme &&
2038           (*info)->mMimeType == decoder.GetCurrentInfo()->mMimeType;
2039       if (!recyclable && decoder.mTimeThreshold.isNothing() &&
2040           (decoder.mNextStreamSourceID.isNothing() ||
2041            decoder.mNextStreamSourceID.ref() != info->GetID())) {
2042         LOG("%s stream id has changed from:%d to:%d, draining decoder.",
2043             TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
2044         decoder.RequestDrain();
2045         decoder.mNextStreamSourceID = Some(info->GetID());
2046         ScheduleUpdate(aTrack);
2047         return;
2048       }
2049
2050       // If flushing is required, it will clear our array of queued samples.
2051       // So we may need to make a copy.
2052       samples = decoder.mQueuedSamples.Clone();
2053       if (!recyclable) {
2054         LOG("Decoder does not support recycling, recreate decoder.");
2055         ShutdownDecoder(aTrack);
2056         // We're going to be using a new decoder following the change of content
2057         // We can attempt to use hardware decoding again.
2058         decoder.mHardwareDecodingDisabled = false;
2059       } else if (decoder.HasWaitingPromise()) {
2060         decoder.Flush();
2061       }
2062     }
2063
2064     nsPrintfCString markerString(
2065         "%s stream id changed from:%" PRIu32 " to:%" PRIu32,
2066         TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
2067     PROFILER_MARKER_TEXT("StreamID Change", MEDIA_PLAYBACK, {}, markerString);
2068     LOG("%s", markerString.get());
2069
2070     if (aTrack == TrackInfo::kVideoTrack) {
2071       // We are about to create a new decoder thus the benchmark,
2072       // up to this point, is stored.
2073       NotifyDecoderBenchmarkStore();
2074     }
2075     decoder.mNextStreamSourceID.reset();
2076     decoder.mLastStreamSourceID = info->GetID();
2077     decoder.mInfo = info;
2078     {
2079       MutexAutoLock lock(decoder.mMutex);
2080       if (aTrack == TrackInfo::kAudioTrack) {
2081         decoder.mWorkingInfo = MakeUnique<AudioInfo>(*info->GetAsAudioInfo());
2082       } else if (aTrack == TrackInfo::kVideoTrack) {
2083         decoder.mWorkingInfo = MakeUnique<VideoInfo>(*info->GetAsVideoInfo());
2084       }
2085       mWorkingInfoChanged = true;
2086     }
2087
2088     decoder.mMeanRate.Reset();
2089
2090     if (sample->mKeyframe) {
2091       if (samples.Length()) {
2092         decoder.mQueuedSamples = std::move(samples);
2093       }
2094     } else {
2095       auto time = TimeInterval(sample->mTime, sample->GetEndTime());
2096       InternalSeekTarget seekTarget =
2097           decoder.mTimeThreshold.refOr(InternalSeekTarget(time, false));
2098       LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64,
2099           sample->mTime.ToMicroseconds());
2100       InternalSeek(aTrack, seekTarget);
2101       return;
2102     }
2103   }
2104
2105   // Calculate the average frame rate. The first frame will be accounted
2106   // for twice.
2107   decoder.mMeanRate.Update(sample->mDuration);
2108
2109   if (!decoder.mDecoder) {
2110     // In Clear Lead situation, the `mInfo` could change from unencrypted to
2111     // encrypted so we need to ensure the CDM proxy is ready before creating a
2112     // decoder.
2113     if (decoder.IsEncrypted() &&
2114         (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack))) {
2115       return;
2116     }
2117     mDecoderFactory->CreateDecoder(aTrack);
2118     return;
2119   }
2120
2121   LOGV("Input:%" PRId64 " (dts:%" PRId64 " kf:%d)",
2122        sample->mTime.ToMicroseconds(), sample->mTimecode.ToMicroseconds(),
2123        sample->mKeyframe);
2124   decoder.mNumSamplesInput++;
2125   decoder.mSizeOfQueue++;
2126   if (aTrack == TrackInfo::kVideoTrack) {
2127     aA.mStats.mParsedFrames++;
2128   }
2129
2130   DecodeDemuxedSamples(aTrack, sample);
2131
2132   decoder.mQueuedSamples.RemoveElementAt(0);
2133 }
2134
2135 media::TimeUnit MediaFormatReader::GetInternalSeekTargetEndTime() const {
2136   MOZ_ASSERT(OnTaskQueue());
2137   return mVideo.mTimeThreshold ? mVideo.mTimeThreshold->EndTime()
2138                                : TimeUnit::FromInfinity();
2139 }
2140
2141 void MediaFormatReader::InternalSeek(TrackType aTrack,
2142                                      const InternalSeekTarget& aTarget) {
2143   MOZ_ASSERT(OnTaskQueue());
2144   LOG("%s internal seek to %f", TrackTypeToStr(aTrack),
2145       aTarget.Time().ToSeconds());
2146
2147   auto& decoder = GetDecoderData(aTrack);
2148   decoder.Flush();
2149   decoder.ResetDemuxer();
2150   decoder.mTimeThreshold = Some(aTarget);
2151   DDLOG(DDLogCategory::Log, "seeking", DDNoValue{});
2152   RefPtr<MediaFormatReader> self = this;
2153   decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().Time())
2154       ->Then(
2155           OwnerThread(), __func__,
2156           [self, aTrack](TimeUnit aTime) {
2157             DDLOGEX(self.get(), DDLogCategory::Log, "seeked", DDNoValue{});
2158             auto& decoder = self->GetDecoderData(aTrack);
2159             decoder.mSeekRequest.Complete();
2160             MOZ_ASSERT(decoder.mTimeThreshold,
2161                        "Seek promise must be disconnected when "
2162                        "timethreshold is reset");
2163             decoder.mTimeThreshold.ref().mHasSeeked = true;
2164             self->SetVideoDecodeThreshold();
2165             self->ScheduleUpdate(aTrack);
2166           },
2167           [self, aTrack](const MediaResult& aError) {
2168             auto& decoder = self->GetDecoderData(aTrack);
2169             decoder.mSeekRequest.Complete();
2170             switch (aError.Code()) {
2171               case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2172                 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2173                         aError);
2174                 self->NotifyWaitingForData(aTrack);
2175                 break;
2176               case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2177                 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2178                         aError);
2179                 decoder.mTimeThreshold.reset();
2180                 self->NotifyEndOfStream(aTrack);
2181                 break;
2182               case NS_ERROR_DOM_MEDIA_CANCELED:
2183                 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2184                         aError);
2185                 decoder.mTimeThreshold.reset();
2186                 break;
2187               default:
2188                 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_error",
2189                         aError);
2190                 decoder.mTimeThreshold.reset();
2191                 self->NotifyError(aTrack, aError);
2192                 break;
2193             }
2194           })
2195       ->Track(decoder.mSeekRequest);
2196 }
2197
2198 void MediaFormatReader::DrainDecoder(TrackType aTrack) {
2199   AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK);
2200   MOZ_ASSERT(OnTaskQueue());
2201
2202   auto& decoder = GetDecoderData(aTrack);
2203   if (decoder.mDrainState == DrainState::Draining) {
2204     return;
2205   }
2206   if (!decoder.mDecoder ||
2207       (decoder.mDrainState != DrainState::PartialDrainPending &&
2208        decoder.mNumSamplesInput == decoder.mNumSamplesOutput)) {
2209     // No frames to drain.
2210     LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack));
2211     decoder.mDrainState = DrainState::DrainAborted;
2212     ScheduleUpdate(aTrack);
2213     return;
2214   }
2215
2216   decoder.mDrainState = DrainState::Draining;
2217
2218   DDLOG(DDLogCategory::Log, "draining", DDNoValue{});
2219   RefPtr<MediaFormatReader> self = this;
2220   decoder.mDecoder->Drain()
2221       ->Then(
2222           mTaskQueue, __func__,
2223           [self, aTrack, &decoder](MediaDataDecoder::DecodedData&& aResults) {
2224             decoder.mDrainRequest.Complete();
2225             DDLOGEX(self.get(), DDLogCategory::Log, "drained", DDNoValue{});
2226             if (aResults.IsEmpty()) {
2227               decoder.mDrainState = DrainState::DrainCompleted;
2228             } else {
2229               self->NotifyNewOutput(aTrack, std::move(aResults));
2230               // Let's see if we have any more data available to drain.
2231               decoder.mDrainState = DrainState::PartialDrainPending;
2232             }
2233             self->ScheduleUpdate(aTrack);
2234           },
2235           [self, aTrack, &decoder](const MediaResult& aError) {
2236             decoder.mDrainRequest.Complete();
2237             DDLOGEX(self.get(), DDLogCategory::Log, "draining_error", aError);
2238             self->NotifyError(aTrack, aError);
2239           })
2240       ->Track(decoder.mDrainRequest);
2241   LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack));
2242 }
2243
2244 void MediaFormatReader::Update(TrackType aTrack) {
2245   AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK);
2246   MOZ_ASSERT(OnTaskQueue());
2247
2248   if (mShutdown) {
2249     return;
2250   }
2251
2252   LOGV("Processing update for %s", TrackTypeToStr(aTrack));
2253
2254   bool needOutput = false;
2255   auto& decoder = GetDecoderData(aTrack);
2256   decoder.mUpdateScheduled = false;
2257
2258   if (!mInitDone) {
2259     return;
2260   }
2261
2262   if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
2263     LOGV("Skipping in progress, nothing more to do");
2264     return;
2265   }
2266
2267   if (UpdateReceivedNewData(aTrack)) {
2268     LOGV("Nothing more to do");
2269     return;
2270   }
2271
2272   if (decoder.mSeekRequest.Exists()) {
2273     LOGV("Seeking hasn't completed, nothing more to do");
2274     return;
2275   }
2276
2277   MOZ_DIAGNOSTIC_ASSERT(
2278       !decoder.HasInternalSeekPending() ||
2279           (!decoder.mOutput.Length() && !decoder.mQueuedSamples.Length()),
2280       "No frames can be demuxed or decoded while an internal seek is pending");
2281
2282   // Record number of frames decoded and parsed. Automatically update the
2283   // stats counters using the AutoNotifyDecoded stack-based class.
2284   FrameStatistics::AutoNotifyDecoded a(mFrameStats);
2285
2286   // Drop any frames found prior our internal seek target.
2287   while (decoder.mTimeThreshold && decoder.mOutput.Length()) {
2288     RefPtr<MediaData>& output = decoder.mOutput[0];
2289     InternalSeekTarget target = decoder.mTimeThreshold.ref();
2290     auto time = output->mTime;
2291     if (time >= target.Time()) {
2292       // We have reached our internal seek target.
2293       decoder.mTimeThreshold.reset();
2294       // We might have dropped some keyframes.
2295       mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2296     }
2297     if (time < target.Time() || (target.mDropTarget && target.Contains(time))) {
2298       LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2299            TrackTypeToStr(aTrack), output->mTime.ToSeconds(),
2300            target.Time().ToSeconds(), output->mKeyframe);
2301       decoder.mOutput.RemoveElementAt(0);
2302       decoder.mSizeOfQueue -= 1;
2303     }
2304   }
2305
2306   while (decoder.mOutput.Length() &&
2307          decoder.mOutput[0]->mType == MediaData::Type::NULL_DATA) {
2308     LOGV("Dropping null data. Time: %" PRId64,
2309          decoder.mOutput[0]->mTime.ToMicroseconds());
2310     decoder.mOutput.RemoveElementAt(0);
2311     decoder.mSizeOfQueue -= 1;
2312   }
2313
2314   if (decoder.HasPromise()) {
2315     needOutput = true;
2316     if (decoder.mOutput.Length()) {
2317       RefPtr<MediaData> output = decoder.mOutput[0];
2318       decoder.mOutput.RemoveElementAt(0);
2319       decoder.mSizeOfQueue -= 1;
2320       decoder.mLastDecodedSampleTime =
2321           Some(TimeInterval(output->mTime, output->GetEndTime()));
2322       decoder.mNumSamplesOutputTotal++;
2323       ReturnOutput(output, aTrack);
2324       // We have a decoded sample ready to be returned.
2325       if (aTrack == TrackType::kVideoTrack) {
2326         uint64_t delta =
2327             decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
2328         a.mStats.mDecodedFrames = static_cast<uint32_t>(delta);
2329         mLastReportedNumDecodedFrames = decoder.mNumSamplesOutputTotal;
2330         if (output->mKeyframe) {
2331           if (mPreviousDecodedKeyframeTime_us <
2332               output->mTime.ToMicroseconds()) {
2333             // There is a previous keyframe -> Record inter-keyframe stats.
2334             uint64_t segment_us = output->mTime.ToMicroseconds() -
2335                                   mPreviousDecodedKeyframeTime_us;
2336             a.mStats.mInterKeyframeSum_us += segment_us;
2337             a.mStats.mInterKeyframeCount += 1;
2338             if (a.mStats.mInterKeyFrameMax_us < segment_us) {
2339               a.mStats.mInterKeyFrameMax_us = segment_us;
2340             }
2341           }
2342           mPreviousDecodedKeyframeTime_us = output->mTime.ToMicroseconds();
2343         }
2344         bool wasHardwareAccelerated = mVideo.mIsHardwareAccelerated;
2345         nsCString error;
2346         mVideo.mIsHardwareAccelerated =
2347             mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
2348         VideoData* videoData = output->As<VideoData>();
2349         if (!mVideo.mHasReportedVideoHardwareSupportTelemtry ||
2350             wasHardwareAccelerated != mVideo.mIsHardwareAccelerated) {
2351           mVideo.mHasReportedVideoHardwareSupportTelemtry = true;
2352           Telemetry::ScalarSet(
2353               Telemetry::ScalarID::MEDIA_VIDEO_HARDWARE_DECODING_SUPPORT,
2354               NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2355               !!mVideo.mIsHardwareAccelerated);
2356           static constexpr gfx::IntSize HD_VIDEO_SIZE{1280, 720};
2357           if (videoData->mDisplay.width >= HD_VIDEO_SIZE.Width() &&
2358               videoData->mDisplay.height >= HD_VIDEO_SIZE.Height()) {
2359             Telemetry::ScalarSet(
2360                 Telemetry::ScalarID::MEDIA_VIDEO_HD_HARDWARE_DECODING_SUPPORT,
2361                 NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2362                 !!mVideo.mIsHardwareAccelerated);
2363           }
2364         }
2365 #ifdef XP_WIN
2366         // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2367         // of GPU RAM used.
2368         mVideo.mIsHardwareAccelerated =
2369             mVideo.mIsHardwareAccelerated ||
2370             (videoData->mImage &&
2371              videoData->mImage->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE);
2372 #endif
2373       }
2374     } else if (decoder.HasFatalError()) {
2375       nsCString mimeType = decoder.GetCurrentInfo()->mMimeType;
2376       if (!mimeType.IsEmpty()) {
2377         Telemetry::ScalarAdd(
2378             Telemetry::ScalarID::MEDIA_DECODE_ERROR_PER_MIME_TYPE,
2379             NS_ConvertUTF8toUTF16(mimeType), 1 /* error count */);
2380       }
2381       LOG("Rejecting %s promise for %s : DECODE_ERROR", TrackTypeToStr(aTrack),
2382           mimeType.get());
2383       decoder.RejectPromise(decoder.mError.ref(), __func__);
2384       return;
2385     } else if (decoder.HasCompletedDrain()) {
2386       if (decoder.mDemuxEOS) {
2387         LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2388         if (aTrack == TrackInfo::kVideoTrack) {
2389           // End of video, store the benchmark of the decoder.
2390           NotifyDecoderBenchmarkStore();
2391         }
2392         decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2393       } else if (decoder.mWaitingForData) {
2394         if (decoder.mDrainState == DrainState::DrainCompleted &&
2395             decoder.mLastDecodedSampleTime && !decoder.mNextStreamSourceID) {
2396           // We have completed draining the decoder following WaitingForData.
2397           // Set up the internal seek machinery to be able to resume from the
2398           // last sample decoded.
2399           LOG("Seeking to last sample time: %" PRId64,
2400               decoder.mLastDecodedSampleTime.ref().mStart.ToMicroseconds());
2401           InternalSeek(aTrack, InternalSeekTarget(
2402                                    decoder.mLastDecodedSampleTime.ref(), true));
2403         }
2404         if (!decoder.mReceivedNewData) {
2405           LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack));
2406           decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2407         }
2408       }
2409
2410       decoder.mDrainState = DrainState::None;
2411
2412       // Now that draining has completed, we check if we have received
2413       // new data again as the result may now be different from the earlier
2414       // run.
2415       if (UpdateReceivedNewData(aTrack) || decoder.mSeekRequest.Exists()) {
2416         LOGV("Nothing more to do");
2417         return;
2418       }
2419     } else if (decoder.mDemuxEOS && !decoder.HasPendingDrain() &&
2420                decoder.mQueuedSamples.IsEmpty()) {
2421       // It is possible to transition from WAITING_FOR_DATA directly to EOS
2422       // state during the internal seek; in which case no draining would occur.
2423       // There is no more samples left to be decoded and we are already in
2424       // EOS state. We can immediately reject the data promise.
2425       LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2426       decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2427     } else if (decoder.mWaitingForKey) {
2428       LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2429           TrackTypeToStr(aTrack));
2430       decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2431     } else if (IsDecoderWaitingForCDM(aTrack)) {
2432       // Rejecting the promise could lead to entering buffering state for MDSM,
2433       // once a qualified(with the same key system and sessions created by the
2434       // same InitData) new cdm proxy is set, decoding can be resumed.
2435       LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2436           TrackTypeToStr(aTrack));
2437       decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2438     }
2439   }
2440
2441   if (decoder.mDrainState == DrainState::DrainRequested ||
2442       decoder.mDrainState == DrainState::PartialDrainPending) {
2443     if (decoder.mOutput.IsEmpty()) {
2444       DrainDecoder(aTrack);
2445     }
2446     return;
2447   }
2448
2449   if (decoder.mError && !decoder.HasFatalError()) {
2450     MOZ_RELEASE_ASSERT(!decoder.HasInternalSeekPending(),
2451                        "No error can occur while an internal seek is pending");
2452
2453     nsCString error;
2454     bool firstFrameDecodingFailedWithHardware =
2455         decoder.mFirstFrameTime &&
2456         decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2457         decoder.mDecoder && decoder.mDecoder->IsHardwareAccelerated(error) &&
2458         !decoder.mHardwareDecodingDisabled;
2459     bool needsNewDecoder =
2460         decoder.mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER ||
2461         firstFrameDecodingFailedWithHardware;
2462     // Limit number of process restarts after crash
2463     if ((decoder.mError.ref() ==
2464              NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR &&
2465          decoder.mNumOfConsecutiveRDDOrGPUCrashes++ <
2466              decoder.mMaxConsecutiveRDDOrGPUCrashes) ||
2467         (decoder.mError.ref() ==
2468              NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR &&
2469          decoder.mNumOfConsecutiveUtilityCrashes++ <
2470              decoder.mMaxConsecutiveUtilityCrashes)) {
2471       needsNewDecoder = true;
2472     }
2473     // For MF CDM crash, it needs to be handled differently. We need to shutdown
2474     // current decoder and report that error to the state machine in order to
2475     // let it to determine if playback can keep going or not.
2476     if (decoder.mError.ref() ==
2477         NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_MF_CDM_ERR) {
2478       LOG("Error: notify MF CDM crash and shutdown %s decoder",
2479           TrackTypeToStr(aTrack));
2480       ShutdownDecoder(aTrack);
2481       decoder.RejectPromise(decoder.mError.ref(), __func__);
2482       decoder.mError.reset();
2483       return;
2484     }
2485 #ifdef XP_LINUX
2486     // We failed to decode on Linux with HW decoder,
2487     // give it another try without HW decoder.
2488     if (decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2489         decoder.mDecoder->IsHardwareAccelerated(error)) {
2490       LOG("Error: %s decode error, disable HW acceleration",
2491           TrackTypeToStr(aTrack));
2492       needsNewDecoder = true;
2493       decoder.mHardwareDecodingDisabled = true;
2494     }
2495     // RDD process crashed on Linux, give it another try without HW decoder.
2496     if (decoder.mError.ref() ==
2497         NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR) {
2498       LOG("Error: %s remote decoder crashed, disable HW acceleration",
2499           TrackTypeToStr(aTrack));
2500       decoder.mHardwareDecodingDisabled = true;
2501     }
2502 #endif
2503     // We don't want to expose crash error so switch to
2504     // NS_ERROR_DOM_MEDIA_DECODE_ERR.
2505     if (decoder.mError.ref() ==
2506             NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR ||
2507         decoder.mError.ref() ==
2508             NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR) {
2509       decoder.mError = Some(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
2510                                         RESULT_DETAIL("Unable to decode")));
2511     }
2512     if (!needsNewDecoder && ++decoder.mNumOfConsecutiveDecodingError >
2513                                 decoder.mMaxConsecutiveDecodingError) {
2514       DDLOG(DDLogCategory::Log, "too_many_decode_errors", decoder.mError.ref());
2515       NotifyError(aTrack, decoder.mError.ref());
2516       return;
2517     }
2518
2519     if (firstFrameDecodingFailedWithHardware) {
2520       decoder.mHardwareDecodingDisabled = true;
2521     }
2522     decoder.mError.reset();
2523
2524     LOG("%s decoded error count %d RDD crashes count %d",
2525         TrackTypeToStr(aTrack), decoder.mNumOfConsecutiveDecodingError,
2526         decoder.mNumOfConsecutiveRDDOrGPUCrashes);
2527
2528     if (needsNewDecoder) {
2529       LOG("Error: %s needs a new decoder", TrackTypeToStr(aTrack));
2530       ShutdownDecoder(aTrack);
2531     }
2532     if (decoder.mFirstFrameTime) {
2533       TimeInterval seekInterval = TimeInterval(decoder.mFirstFrameTime.ref(),
2534                                                decoder.mFirstFrameTime.ref());
2535       InternalSeek(aTrack, InternalSeekTarget(seekInterval, false));
2536       return;
2537     }
2538
2539     TimeUnit nextKeyframe;
2540     if (aTrack == TrackType::kVideoTrack &&
2541         NS_SUCCEEDED(
2542             decoder.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe)) &&
2543         !nextKeyframe.IsInfinite()) {
2544       SkipVideoDemuxToNextKeyFrame(
2545           decoder.mLastDecodedSampleTime.refOr(TimeInterval()).Length());
2546     } else if (aTrack == TrackType::kAudioTrack) {
2547       decoder.Flush();
2548     } else {
2549       DDLOG(DDLogCategory::Log, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR);
2550       // We can't recover from this error.
2551       NotifyError(aTrack, NS_ERROR_DOM_MEDIA_FATAL_ERR);
2552     }
2553     return;
2554   }
2555
2556   bool needInput = NeedInput(decoder);
2557
2558   LOGV("Update(%s) ni=%d no=%d in:%" PRIu64 " out:%" PRIu64
2559        " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2560        "ds:%d sid:%u waitcdm:%d",
2561        TrackTypeToStr(aTrack), needInput, needOutput, decoder.mNumSamplesInput,
2562        decoder.mNumSamplesOutput, uint32_t(size_t(decoder.mSizeOfQueue)),
2563        decoder.mDecodeRequest.Exists(), decoder.mFlushing,
2564        decoder.mDescription.get(), uint32_t(decoder.mOutput.Length()),
2565        decoder.mWaitingForData, decoder.mDemuxEOS, int32_t(decoder.mDrainState),
2566        decoder.mLastStreamSourceID, IsDecoderWaitingForCDM(aTrack));
2567
2568   if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack)) {
2569     // If the content is encrypted, MFR won't start to create decoder until
2570     // CDMProxy is set.
2571     return;
2572   }
2573
2574   if ((decoder.IsWaitingForData() &&
2575        (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) ||
2576       (decoder.IsWaitingForKey())) {
2577     // Nothing more we can do at present.
2578     LOGV("Still waiting for data or key. data(%d)/key(%d)",
2579          decoder.mWaitingForData, decoder.mWaitingForKey);
2580     return;
2581   }
2582
2583   if (decoder.CancelWaitingForKey()) {
2584     LOGV("No longer waiting for key. Resolving waiting promise");
2585     return;
2586   }
2587
2588   if (!needInput) {
2589     LOGV("No need for additional input (pending:%u)",
2590          uint32_t(decoder.mOutput.Length()));
2591     return;
2592   }
2593
2594   // Demux samples if we don't have some.
2595   RequestDemuxSamples(aTrack);
2596
2597   HandleDemuxedSamples(aTrack, a);
2598 }
2599
2600 void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
2601   AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK);
2602   MOZ_ASSERT(GetDecoderData(aTrack).HasPromise());
2603   MOZ_DIAGNOSTIC_ASSERT(aData->mType != MediaData::Type::NULL_DATA);
2604   LOG("Resolved data promise for %s [%" PRId64 ", %" PRId64 "]",
2605       TrackTypeToStr(aTrack), aData->mTime.ToMicroseconds(),
2606       aData->GetEndTime().ToMicroseconds());
2607
2608   if (aTrack == TrackInfo::kAudioTrack) {
2609     AudioData* audioData = aData->As<AudioData>();
2610
2611     if (audioData->mChannels != mInfo.mAudio.mChannels ||
2612         audioData->mRate != mInfo.mAudio.mRate) {
2613       LOG("change of audio format (rate:%d->%d). "
2614           "This is an unsupported configuration",
2615           mInfo.mAudio.mRate, audioData->mRate);
2616       mInfo.mAudio.mRate = audioData->mRate;
2617       mInfo.mAudio.mChannels = audioData->mChannels;
2618       MutexAutoLock lock(mAudio.mMutex);
2619       mAudio.mWorkingInfo->GetAsAudioInfo()->mRate = audioData->mRate;
2620       mAudio.mWorkingInfo->GetAsAudioInfo()->mChannels = audioData->mChannels;
2621       mWorkingInfoChanged = true;
2622     }
2623     mAudio.ResolvePromise(audioData, __func__);
2624   } else if (aTrack == TrackInfo::kVideoTrack) {
2625     VideoData* videoData = aData->As<VideoData>();
2626
2627     if (videoData->mDisplay != mInfo.mVideo.mDisplay) {
2628       LOG("change of video display size (%dx%d->%dx%d)",
2629           mInfo.mVideo.mDisplay.width, mInfo.mVideo.mDisplay.height,
2630           videoData->mDisplay.width, videoData->mDisplay.height);
2631       mInfo.mVideo.mDisplay = videoData->mDisplay;
2632       MutexAutoLock lock(mVideo.mMutex);
2633       mVideo.mWorkingInfo->GetAsVideoInfo()->mDisplay = videoData->mDisplay;
2634       mWorkingInfoChanged = true;
2635     }
2636
2637     mozilla::gfx::ColorDepth colorDepth = videoData->GetColorDepth();
2638     if (colorDepth != mInfo.mVideo.mColorDepth) {
2639       LOG("change of video color depth (enum %u -> enum %u)",
2640           (unsigned)mInfo.mVideo.mColorDepth, (unsigned)colorDepth);
2641       mInfo.mVideo.mColorDepth = colorDepth;
2642       MutexAutoLock lock(mVideo.mMutex);
2643       mVideo.mWorkingInfo->GetAsVideoInfo()->mColorDepth = colorDepth;
2644       mWorkingInfoChanged = true;
2645     }
2646
2647     TimeUnit nextKeyframe;
2648     if (!mVideo.HasInternalSeekPending() &&
2649         NS_SUCCEEDED(
2650             mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe))) {
2651       videoData->SetNextKeyFrameTime(nextKeyframe);
2652     }
2653
2654     mVideo.ResolvePromise(videoData, __func__);
2655   }
2656 }
2657
2658 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2659   return SizeOfQueue(TrackInfo::kVideoTrack);
2660 }
2661
2662 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2663   return SizeOfQueue(TrackInfo::kAudioTrack);
2664 }
2665
2666 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack) {
2667   auto& decoder = GetDecoderData(aTrack);
2668   return decoder.mSizeOfQueue;
2669 }
2670
2671 RefPtr<MediaFormatReader::WaitForDataPromise> MediaFormatReader::WaitForData(
2672     MediaData::Type aType) {
2673   MOZ_ASSERT(OnTaskQueue());
2674   TrackType trackType = aType == MediaData::Type::VIDEO_DATA
2675                             ? TrackType::kVideoTrack
2676                             : TrackType::kAudioTrack;
2677   auto& decoder = GetDecoderData(trackType);
2678   if (!decoder.IsWaitingForData() && !decoder.IsWaitingForKey()) {
2679     // We aren't waiting for anything.
2680     return WaitForDataPromise::CreateAndResolve(decoder.mType, __func__);
2681   }
2682   RefPtr<WaitForDataPromise> p = decoder.mWaitingPromise.Ensure(__func__);
2683   ScheduleUpdate(trackType);
2684   return p;
2685 }
2686
2687 nsresult MediaFormatReader::ResetDecode(const TrackSet& aTracks) {
2688   AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK);
2689   MOZ_ASSERT(OnTaskQueue());
2690   LOGV("");
2691
2692   mSeekPromise.RejectIfExists(NS_OK, __func__);
2693   mSkipRequest.DisconnectIfExists();
2694
2695   // Do the same for any data wait promises.
2696   if (aTracks.contains(TrackInfo::kAudioTrack)) {
2697     mAudio.mWaitingPromise.RejectIfExists(
2698         WaitForDataRejectValue(MediaData::Type::AUDIO_DATA,
2699                                WaitForDataRejectValue::CANCELED),
2700         __func__);
2701   }
2702
2703   if (aTracks.contains(TrackInfo::kVideoTrack)) {
2704     mVideo.mWaitingPromise.RejectIfExists(
2705         WaitForDataRejectValue(MediaData::Type::VIDEO_DATA,
2706                                WaitForDataRejectValue::CANCELED),
2707         __func__);
2708   }
2709
2710   // Reset miscellaneous seeking state.
2711   mPendingSeekTime.reset();
2712
2713   if (HasVideo() && aTracks.contains(TrackInfo::kVideoTrack)) {
2714     mVideo.ResetDemuxer();
2715     mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2716     Reset(TrackInfo::kVideoTrack);
2717     if (mVideo.HasPromise()) {
2718       mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2719     }
2720   }
2721
2722   if (HasAudio() && aTracks.contains(TrackInfo::kAudioTrack)) {
2723     mAudio.ResetDemuxer();
2724     mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2725     Reset(TrackInfo::kAudioTrack);
2726     if (mAudio.HasPromise()) {
2727       mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2728     }
2729   }
2730
2731   return NS_OK;
2732 }
2733
2734 void MediaFormatReader::Reset(TrackType aTrack) {
2735   MOZ_ASSERT(OnTaskQueue());
2736   LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack));
2737
2738   auto& decoder = GetDecoderData(aTrack);
2739
2740   decoder.ResetState();
2741   decoder.Flush();
2742
2743   LOG("Reset(%s) END", TrackTypeToStr(aTrack));
2744 }
2745
2746 void MediaFormatReader::DropDecodedSamples(TrackType aTrack) {
2747   MOZ_ASSERT(OnTaskQueue());
2748   auto& decoder = GetDecoderData(aTrack);
2749   size_t lengthDecodedQueue = decoder.mOutput.Length();
2750   if (lengthDecodedQueue && decoder.mTimeThreshold.isSome()) {
2751     auto time = decoder.mOutput.LastElement()->mTime;
2752     if (time >= decoder.mTimeThreshold.ref().Time()) {
2753       // We would have reached our internal seek target.
2754       decoder.mTimeThreshold.reset();
2755     }
2756   }
2757   decoder.mOutput.Clear();
2758   decoder.mSizeOfQueue -= lengthDecodedQueue;
2759   if (aTrack == TrackInfo::kVideoTrack && mFrameStats) {
2760     mFrameStats->Accumulate({0, 0, 0, lengthDecodedQueue, 0, 0});
2761   }
2762 }
2763
2764 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold) {
2765   AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2766                       MEDIA_PLAYBACK);
2767   MOZ_ASSERT(OnTaskQueue());
2768   LOG("Skipping up to %" PRId64, aTimeThreshold.ToMicroseconds());
2769
2770   // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2771   // As such we can drop all already decoded samples and discard all pending
2772   // samples.
2773   DropDecodedSamples(TrackInfo::kVideoTrack);
2774
2775   mVideo.mTrackDemuxer->SkipToNextRandomAccessPoint(aTimeThreshold)
2776       ->Then(OwnerThread(), __func__, this,
2777              &MediaFormatReader::OnVideoSkipCompleted,
2778              &MediaFormatReader::OnVideoSkipFailed)
2779       ->Track(mSkipRequest);
2780 }
2781
2782 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped) {
2783   PROFILER_MARKER_UNTYPED("SkippedVideoDecode", MEDIA_PLAYBACK);
2784   MOZ_ASSERT(OnTaskQueue());
2785
2786   // Some frames may have been output by the decoder since we initiated the
2787   // videoskip process and we know they would be late.
2788   DropDecodedSamples(TrackInfo::kVideoTrack);
2789   // Report the pending frames as dropped.
2790   if (mFrameStats) {
2791     uint32_t droppedDecoderCount = SizeOfVideoQueueInFrames();
2792     mFrameStats->Accumulate({0, 0, 0, droppedDecoderCount, 0, 0});
2793   }
2794
2795   // Cancel any pending demux request and pending demuxed samples.
2796   mVideo.mDemuxRequest.DisconnectIfExists();
2797   Reset(TrackType::kVideoTrack);
2798
2799   if (mFrameStats) {
2800     mFrameStats->Accumulate({aSkipped, 0, 0, aSkipped, 0, 0});
2801   }
2802
2803   mVideo.mNumSamplesSkippedTotal += aSkipped;
2804 }
2805
2806 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped) {
2807   AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2808                       MEDIA_PLAYBACK);
2809   MOZ_ASSERT(OnTaskQueue());
2810   LOG("Skipping succeeded, skipped %u frames", aSkipped);
2811   mSkipRequest.Complete();
2812
2813   DDLOG(DDLogCategory::Log, "video_skipped", DDNoValue());
2814
2815   VideoSkipReset(aSkipped);
2816
2817   ScheduleUpdate(TrackInfo::kVideoTrack);
2818 }
2819
2820 void MediaFormatReader::OnVideoSkipFailed(
2821     MediaTrackDemuxer::SkipFailureHolder aFailure) {
2822   AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK);
2823   MOZ_ASSERT(OnTaskQueue());
2824   LOG("Skipping failed, skipped %u frames", aFailure.mSkipped);
2825   mSkipRequest.Complete();
2826
2827   switch (aFailure.mFailure.Code()) {
2828     case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2829     case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2830       DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2831             aFailure.mFailure);
2832       // Some frames may have been output by the decoder since we initiated the
2833       // videoskip process and we know they would be late.
2834       DropDecodedSamples(TrackInfo::kVideoTrack);
2835       // We can't complete the skip operation, will just service a video frame
2836       // normally.
2837       ScheduleUpdate(TrackInfo::kVideoTrack);
2838       break;
2839     case NS_ERROR_DOM_MEDIA_CANCELED:
2840       DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2841             aFailure.mFailure);
2842       if (mVideo.HasPromise()) {
2843         mVideo.RejectPromise(aFailure.mFailure, __func__);
2844       }
2845       break;
2846     default:
2847       DDLOG(DDLogCategory::Log, "video_skipping_error", aFailure.mFailure);
2848       NotifyError(TrackType::kVideoTrack, aFailure.mFailure);
2849       break;
2850   }
2851 }
2852
2853 RefPtr<MediaFormatReader::SeekPromise> MediaFormatReader::Seek(
2854     const SeekTarget& aTarget) {
2855   AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK);
2856   MOZ_ASSERT(OnTaskQueue());
2857
2858   LOG("aTarget=(%" PRId64 "), track=%s", aTarget.GetTime().ToMicroseconds(),
2859       SeekTarget::TrackToStr(aTarget.GetTrack()));
2860
2861   MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty());
2862   MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime.isNothing());
2863   // Should reset data request, and no pending internal seek.
2864   if (aTarget.IsAllTracks()) {
2865     MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2866     MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise());
2867     MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2868     MOZ_DIAGNOSTIC_ASSERT(mAudio.mTimeThreshold.isNothing());
2869   } else if (aTarget.IsVideoOnly()) {
2870     MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2871     MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2872   } else if (aTarget.IsAudioOnly()) {
2873     MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise());
2874     MOZ_DIAGNOSTIC_ASSERT(mAudio.mTimeThreshold.isNothing());
2875   }
2876
2877   if (!mInfo.mMediaSeekable && !mInfo.mMediaSeekableOnlyInBufferedRanges) {
2878     LOG("Seek() END (Unseekable)");
2879     return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2880   }
2881
2882   if (mShutdown) {
2883     return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2884   }
2885
2886   SetSeekTarget(aTarget);
2887
2888   RefPtr<SeekPromise> p = mSeekPromise.Ensure(__func__);
2889
2890   ScheduleSeek();
2891
2892   return p;
2893 }
2894
2895 void MediaFormatReader::SetSeekTarget(const SeekTarget& aTarget) {
2896   MOZ_ASSERT(OnTaskQueue());
2897
2898   mOriginalSeekTarget = aTarget;
2899   mFallbackSeekTime = mPendingSeekTime = Some(aTarget.GetTime());
2900 }
2901
2902 void MediaFormatReader::ScheduleSeek() {
2903   if (mSeekScheduled) {
2904     return;
2905   }
2906   mSeekScheduled = true;
2907   nsresult rv = OwnerThread()->Dispatch(NewRunnableMethod(
2908       "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek));
2909   MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
2910   Unused << rv;
2911 }
2912
2913 void MediaFormatReader::AttemptSeek() {
2914   AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK);
2915   MOZ_ASSERT(OnTaskQueue());
2916
2917   mSeekScheduled = false;
2918
2919   if (mPendingSeekTime.isNothing()) {
2920     LOGV("AttemptSeek, no pending seek time?");
2921     return;
2922   }
2923
2924   // Only reset the demuxers targeted by this SeekTarget, to avoid A/V sync
2925   // issues.
2926   const bool isSeekingAudio = HasAudio() && !mOriginalSeekTarget.IsVideoOnly();
2927   const bool isSeekingVideo = HasVideo() && !mOriginalSeekTarget.IsAudioOnly();
2928   LOG("AttemptSeek, seekingAudio=%d, seekingVideo=%d", isSeekingAudio,
2929       isSeekingVideo);
2930   if (isSeekingVideo) {
2931     mVideo.ResetDemuxer();
2932     mVideo.ResetState();
2933   }
2934   if (isSeekingAudio) {
2935     mAudio.ResetDemuxer();
2936     mAudio.ResetState();
2937   }
2938
2939   // If seeking both tracks, seek the video track, and then the audio track when
2940   // the video track seek has completed. Otherwise, only seek a specific track.
2941   if (isSeekingVideo) {
2942     DoVideoSeek();
2943   } else if (isSeekingAudio) {
2944     DoAudioSeek();
2945   } else {
2946     MOZ_CRASH();
2947   }
2948 }
2949
2950 void MediaFormatReader::OnSeekFailed(TrackType aTrack,
2951                                      const MediaResult& aError) {
2952   AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK);
2953   MOZ_ASSERT(OnTaskQueue());
2954   LOGV("%s failure:%s", TrackTypeToStr(aTrack), aError.ErrorName().get());
2955   if (aTrack == TrackType::kVideoTrack) {
2956     mVideo.mSeekRequest.Complete();
2957   } else {
2958     mAudio.mSeekRequest.Complete();
2959   }
2960
2961   if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
2962     if (HasVideo() && aTrack == TrackType::kAudioTrack &&
2963         mFallbackSeekTime.isSome() &&
2964         mPendingSeekTime.ref() != mFallbackSeekTime.ref()) {
2965       // We have failed to seek audio where video seeked to earlier.
2966       // Attempt to seek instead to the closest point that we know we have in
2967       // order to limit A/V sync discrepency.
2968
2969       // Ensure we have the most up to date buffered ranges.
2970       UpdateReceivedNewData(TrackType::kAudioTrack);
2971       Maybe<TimeUnit> nextSeekTime;
2972       // Find closest buffered time found after video seeked time.
2973       for (const auto& timeRange : mAudio.mTimeRanges) {
2974         if (timeRange.mStart >= mPendingSeekTime.ref()) {
2975           nextSeekTime.emplace(timeRange.mStart);
2976           break;
2977         }
2978       }
2979       if (nextSeekTime.isNothing() ||
2980           nextSeekTime.ref() > mFallbackSeekTime.ref()) {
2981         nextSeekTime = Some(mFallbackSeekTime.ref());
2982         LOG("Unable to seek audio to video seek time. A/V sync may be broken");
2983       } else {
2984         mFallbackSeekTime.reset();
2985       }
2986       mPendingSeekTime = nextSeekTime;
2987       DoAudioSeek();
2988       return;
2989     }
2990     NotifyWaitingForData(aTrack);
2991   }
2992   MOZ_ASSERT(!mVideo.mSeekRequest.Exists() && !mAudio.mSeekRequest.Exists());
2993   mPendingSeekTime.reset();
2994
2995   auto type = aTrack == TrackType::kAudioTrack ? MediaData::Type::AUDIO_DATA
2996                                                : MediaData::Type::VIDEO_DATA;
2997   mSeekPromise.RejectIfExists(SeekRejectValue(type, aError), __func__);
2998 }
2999
3000 void MediaFormatReader::DoVideoSeek() {
3001   AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK);
3002   MOZ_ASSERT(mPendingSeekTime.isSome());
3003   LOGV("Seeking video to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
3004   MOZ_DIAGNOSTIC_ASSERT(!IsAudioOnlySeeking());
3005   MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists());
3006   auto seekTime = mPendingSeekTime.ref();
3007   mVideo.mTrackDemuxer->Seek(seekTime)
3008       ->Then(OwnerThread(), __func__, this,
3009              &MediaFormatReader::OnVideoSeekCompleted,
3010              &MediaFormatReader::OnVideoSeekFailed)
3011       ->Track(mVideo.mSeekRequest);
3012 }
3013
3014 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime) {
3015   AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
3016                       MEDIA_PLAYBACK);
3017   MOZ_ASSERT(OnTaskQueue());
3018   LOGV("Video seeked to %" PRId64, aTime.ToMicroseconds());
3019   mVideo.mSeekRequest.Complete();
3020
3021   mVideo.mFirstFrameTime = Some(aTime);
3022   mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
3023
3024   SetVideoDecodeThreshold();
3025
3026   if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
3027     MOZ_ASSERT(mPendingSeekTime.isSome());
3028     if (mOriginalSeekTarget.IsFast()) {
3029       // We are performing a fast seek. We need to seek audio to where the
3030       // video seeked to, to ensure proper A/V sync once playback resume.
3031       mPendingSeekTime = Some(aTime);
3032     }
3033     DoAudioSeek();
3034   } else {
3035     mPendingSeekTime.reset();
3036     mSeekPromise.ResolveIfExists(aTime, __func__);
3037   }
3038 }
3039
3040 void MediaFormatReader::OnVideoSeekFailed(const MediaResult& aError) {
3041   AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK);
3042   mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
3043   OnSeekFailed(TrackType::kVideoTrack, aError);
3044 }
3045
3046 void MediaFormatReader::SetVideoDecodeThreshold() {
3047   MOZ_ASSERT(OnTaskQueue());
3048
3049   if (!HasVideo() || !mVideo.mDecoder) {
3050     return;
3051   }
3052
3053   if (!mVideo.mTimeThreshold && !IsSeeking()) {
3054     return;
3055   }
3056
3057   TimeUnit threshold;
3058   if (mVideo.mTimeThreshold) {
3059     // For internalSeek.
3060     threshold = mVideo.mTimeThreshold.ref().Time();
3061   } else if (IsSeeking()) {
3062     // If IsSeeking() is true, then video seek must have completed already.
3063     TimeUnit keyframe;
3064     if (NS_FAILED(mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&keyframe))) {
3065       return;
3066     }
3067
3068     // If the key frame is invalid/infinite, it means the target position is
3069     // closing to end of stream. We don't want to skip any frame at this point.
3070     threshold = keyframe.IsValid() && !keyframe.IsInfinite()
3071                     ? mOriginalSeekTarget.GetTime()
3072                     : TimeUnit::Invalid();
3073   } else {
3074     return;
3075   }
3076
3077   if (threshold.IsValid()) {
3078     LOG("Set seek threshold to %" PRId64, threshold.ToMicroseconds());
3079   } else {
3080     LOG("Resetting seek threshold");
3081   }
3082   mVideo.mDecoder->SetSeekThreshold(threshold);
3083 }
3084
3085 void MediaFormatReader::DoAudioSeek() {
3086   AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK);
3087   MOZ_ASSERT(mPendingSeekTime.isSome());
3088   LOGV("Seeking audio to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
3089   MOZ_DIAGNOSTIC_ASSERT(!IsVideoOnlySeeking());
3090   MOZ_DIAGNOSTIC_ASSERT(!mAudio.mSeekRequest.Exists());
3091   auto seekTime = mPendingSeekTime.ref();
3092   mAudio.mTrackDemuxer->Seek(seekTime)
3093       ->Then(OwnerThread(), __func__, this,
3094              &MediaFormatReader::OnAudioSeekCompleted,
3095              &MediaFormatReader::OnAudioSeekFailed)
3096       ->Track(mAudio.mSeekRequest);
3097 }
3098
3099 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime) {
3100   MOZ_ASSERT(OnTaskQueue());
3101   AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
3102                       MEDIA_PLAYBACK);
3103   LOGV("Audio seeked to %" PRId64, aTime.ToMicroseconds());
3104   mAudio.mSeekRequest.Complete();
3105   mAudio.mFirstFrameTime = Some(aTime);
3106   mPendingSeekTime.reset();
3107   mSeekPromise.ResolveIfExists(aTime, __func__);
3108 }
3109
3110 void MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError) {
3111   AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK);
3112   OnSeekFailed(TrackType::kAudioTrack, aError);
3113 }
3114
3115 void MediaFormatReader::ReleaseResources() {
3116   LOGV("");
3117   if (mShutdown) {
3118     return;
3119   }
3120   ShutdownDecoder(TrackInfo::kAudioTrack);
3121   ShutdownDecoder(TrackInfo::kVideoTrack);
3122 }
3123
3124 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
3125   return mVideo.mIsHardwareAccelerated;
3126 }
3127
3128 void MediaFormatReader::NotifyTrackDemuxers() {
3129   MOZ_ASSERT(OnTaskQueue());
3130
3131   LOGV("");
3132
3133   if (!mInitDone) {
3134     return;
3135   }
3136
3137   if (HasVideo()) {
3138     mVideo.mReceivedNewData = true;
3139     ScheduleUpdate(TrackType::kVideoTrack);
3140   }
3141   if (HasAudio()) {
3142     mAudio.mReceivedNewData = true;
3143     ScheduleUpdate(TrackType::kAudioTrack);
3144   }
3145 }
3146
3147 void MediaFormatReader::NotifyDataArrived() {
3148   AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK);
3149   MOZ_ASSERT(OnTaskQueue());
3150
3151   if (mShutdown || !mDemuxer || !mDemuxerInitDone) {
3152     return;
3153   }
3154
3155   if (mNotifyDataArrivedPromise.Exists()) {
3156     // Already one in progress. Set the dirty flag so we can process it later.
3157     mPendingNotifyDataArrived = true;
3158     return;
3159   }
3160
3161   RefPtr<MediaFormatReader> self = this;
3162   mDemuxer->NotifyDataArrived()
3163       ->Then(
3164           OwnerThread(), __func__,
3165           [self]() {
3166             AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
3167                                 MEDIA_PLAYBACK);
3168             self->mNotifyDataArrivedPromise.Complete();
3169             self->UpdateBuffered();
3170             self->NotifyTrackDemuxers();
3171             if (self->mPendingNotifyDataArrived) {
3172               self->mPendingNotifyDataArrived = false;
3173               self->NotifyDataArrived();
3174             }
3175           },
3176           [self]() { self->mNotifyDataArrivedPromise.Complete(); })
3177       ->Track(mNotifyDataArrivedPromise);
3178 }
3179
3180 void MediaFormatReader::UpdateMediaEngineId(uint64_t aMediaEngineId) {
3181   LOG("Update external media engine Id %" PRIu64, aMediaEngineId);
3182   mMediaEngineId = Some(aMediaEngineId);
3183 }
3184
3185 void MediaFormatReader::UpdateBuffered() {
3186   AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK);
3187   MOZ_ASSERT(OnTaskQueue());
3188
3189   if (mShutdown) {
3190     return;
3191   }
3192
3193   if (!mInitDone || !mHasStartTime) {
3194     mBuffered = TimeIntervals();
3195     return;
3196   }
3197
3198   if (HasVideo()) {
3199     mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered();
3200     bool hasLastEnd;
3201     auto lastEnd = mVideo.mTimeRanges.GetEnd(&hasLastEnd);
3202     if (hasLastEnd) {
3203       if (mVideo.mLastTimeRangesEnd &&
3204           mVideo.mLastTimeRangesEnd.ref() < lastEnd) {
3205         // New data was added after our previous end, we can clear the EOS flag.
3206         mVideo.mDemuxEOS = false;
3207         ScheduleUpdate(TrackInfo::kVideoTrack);
3208       }
3209       mVideo.mLastTimeRangesEnd = Some(lastEnd);
3210     }
3211   }
3212   if (HasAudio()) {
3213     mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered();
3214     bool hasLastEnd;
3215     auto lastEnd = mAudio.mTimeRanges.GetEnd(&hasLastEnd);
3216     if (hasLastEnd) {
3217       if (mAudio.mLastTimeRangesEnd &&
3218           mAudio.mLastTimeRangesEnd.ref() < lastEnd) {
3219         // New data was added after our previous end, we can clear the EOS flag.
3220         mAudio.mDemuxEOS = false;
3221         ScheduleUpdate(TrackInfo::kAudioTrack);
3222       }
3223       mAudio.mLastTimeRangesEnd = Some(lastEnd);
3224     }
3225   }
3226
3227   media::TimeIntervals intervals;
3228   if (HasAudio() && HasVideo()) {
3229     intervals = media::Intersection(mVideo.mTimeRanges, mAudio.mTimeRanges);
3230   } else if (HasAudio()) {
3231     intervals = mAudio.mTimeRanges;
3232   } else if (HasVideo()) {
3233     intervals = mVideo.mTimeRanges;
3234   }
3235
3236   if (intervals.IsEmpty() || intervals.GetStart() == TimeUnit::Zero()) {
3237     // IntervalSet already starts at 0 or is empty, nothing to shift.
3238     mBuffered = intervals;
3239   } else {
3240     mBuffered = intervals.Shift(TimeUnit::Zero() - mInfo.mStartTime);
3241   }
3242 }
3243
3244 layers::ImageContainer* MediaFormatReader::GetImageContainer() {
3245   return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
3246                               : nullptr;
3247 }
3248
3249 RefPtr<GenericPromise> MediaFormatReader::RequestDebugInfo(
3250     dom::MediaFormatReaderDebugInfo& aInfo) {
3251   if (!OnTaskQueue()) {
3252     // Run the request on the task queue if it's not already.
3253     return InvokeAsync(mTaskQueue, __func__,
3254                        [this, self = RefPtr{this}, &aInfo] {
3255                          return RequestDebugInfo(aInfo);
3256                        });
3257   }
3258   GetDebugInfo(aInfo);
3259   return GenericPromise::CreateAndResolve(true, __func__);
3260 }
3261
3262 Maybe<nsCString> MediaFormatReader::GetAudioProcessPerCodec() {
3263   if (mAudio.mDescription == "uninitialized"_ns) {
3264     return Nothing();
3265   }
3266
3267   MOZ_ASSERT(mAudio.mProcessName.Length() > 0,
3268              "Should have had a process name");
3269   MOZ_ASSERT(mAudio.mCodecName.Length() > 0, "Should have had a codec name");
3270
3271   nsCString processName = mAudio.mProcessName;
3272   nsCString audioProcessPerCodecName(processName + ","_ns + mAudio.mCodecName);
3273   if (processName != "utility"_ns) {
3274     if (!StaticPrefs::media_rdd_process_enabled()) {
3275       audioProcessPerCodecName += ",rdd-disabled"_ns;
3276     }
3277     if (!StaticPrefs::media_utility_process_enabled()) {
3278       audioProcessPerCodecName += ",utility-disabled"_ns;
3279     }
3280     if (StaticPrefs::media_allow_audio_non_utility()) {
3281       audioProcessPerCodecName += ",allow-non-utility"_ns;
3282     }
3283   }
3284   return Some(audioProcessPerCodecName);
3285 }
3286
3287 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo) {
3288   MOZ_ASSERT(OnTaskQueue(),
3289              "Don't call this off the task queue, it's going to touch a lot of "
3290              "data members");
3291   nsCString result;
3292   nsAutoCString audioDecoderName("unavailable");
3293   nsAutoCString videoDecoderName = audioDecoderName;
3294   nsAutoCString audioType("none");
3295   nsAutoCString videoType("none");
3296
3297   AudioInfo audioInfo;
3298   if (HasAudio()) {
3299     audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
3300     audioDecoderName = mAudio.mDecoder ? mAudio.mDecoder->GetDescriptionName()
3301                                        : mAudio.mDescription;
3302     audioType = audioInfo.mMimeType;
3303     aInfo.mAudioState.mNeedInput = NeedInput(mAudio);
3304     aInfo.mAudioState.mHasPromise = mAudio.HasPromise();
3305     aInfo.mAudioState.mWaitingPromise = !mAudio.mWaitingPromise.IsEmpty();
3306     aInfo.mAudioState.mHasDemuxRequest = mAudio.mDemuxRequest.Exists();
3307     aInfo.mAudioState.mDemuxQueueSize =
3308         AssertedCast<int>(mAudio.mQueuedSamples.Length());
3309     aInfo.mAudioState.mHasDecoder = mAudio.mDecodeRequest.Exists();
3310     aInfo.mAudioState.mTimeTreshold =
3311         mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().Time().ToSeconds()
3312                               : -1.0;
3313     aInfo.mAudioState.mTimeTresholdHasSeeked =
3314         mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().mHasSeeked : false;
3315     aInfo.mAudioState.mNumSamplesInput =
3316         AssertedCast<int64_t>(mAudio.mNumSamplesInput);
3317     aInfo.mAudioState.mNumSamplesOutput =
3318         AssertedCast<int64_t>(mAudio.mNumSamplesOutput);
3319     aInfo.mAudioState.mQueueSize =
3320         AssertedCast<int32_t>(size_t(mAudio.mSizeOfQueue));
3321     aInfo.mAudioState.mPending = AssertedCast<int>(mAudio.mOutput.Length());
3322     aInfo.mAudioState.mWaitingForData = mAudio.mWaitingForData;
3323     aInfo.mAudioState.mDemuxEOS = mAudio.mDemuxEOS;
3324     aInfo.mAudioState.mDrainState = int32_t(mAudio.mDrainState);
3325     aInfo.mAudioState.mWaitingForKey = mAudio.mWaitingForKey;
3326     aInfo.mAudioState.mLastStreamSourceID =
3327         AssertedCast<int64_t>(mAudio.mLastStreamSourceID);
3328   }
3329
3330   CopyUTF8toUTF16(audioDecoderName, aInfo.mAudioDecoderName);
3331   CopyUTF8toUTF16(audioType, aInfo.mAudioType);
3332   aInfo.mAudioChannels = AssertedCast<int32_t>(audioInfo.mChannels);
3333   aInfo.mAudioRate = audioInfo.mRate;
3334   aInfo.mAudioFramesDecoded =
3335       AssertedCast<int64_t>(mAudio.mNumSamplesOutputTotal);
3336
3337   VideoInfo videoInfo;
3338   if (HasVideo()) {
3339     videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
3340     videoDecoderName = mVideo.mDecoder ? mVideo.mDecoder->GetDescriptionName()
3341                                        : mVideo.mDescription;
3342     videoType = videoInfo.mMimeType;
3343     aInfo.mVideoState.mNeedInput = NeedInput(mVideo);
3344     aInfo.mVideoState.mHasPromise = mVideo.HasPromise();
3345     aInfo.mVideoState.mWaitingPromise = !mVideo.mWaitingPromise.IsEmpty();
3346     aInfo.mVideoState.mHasDemuxRequest = mVideo.mDemuxRequest.Exists();
3347     aInfo.mVideoState.mDemuxQueueSize =
3348         AssertedCast<int32_t>(mVideo.mQueuedSamples.Length());
3349     aInfo.mVideoState.mHasDecoder = mVideo.mDecodeRequest.Exists();
3350     aInfo.mVideoState.mTimeTreshold =
3351         mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().Time().ToSeconds()
3352                               : -1.0;
3353     aInfo.mVideoState.mTimeTresholdHasSeeked =
3354         mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().mHasSeeked : false;
3355     aInfo.mVideoState.mNumSamplesInput =
3356         AssertedCast<int64_t>(mVideo.mNumSamplesInput);
3357     aInfo.mVideoState.mNumSamplesOutput =
3358         AssertedCast<int64_t>(mVideo.mNumSamplesOutput);
3359     aInfo.mVideoState.mQueueSize =
3360         AssertedCast<int32_t>(size_t(mVideo.mSizeOfQueue));
3361     aInfo.mVideoState.mPending = AssertedCast<int32_t>(mVideo.mOutput.Length());
3362     aInfo.mVideoState.mWaitingForData = mVideo.mWaitingForData;
3363     aInfo.mVideoState.mDemuxEOS = mVideo.mDemuxEOS;
3364     aInfo.mVideoState.mDrainState = int32_t(mVideo.mDrainState);
3365     aInfo.mVideoState.mWaitingForKey = mVideo.mWaitingForKey;
3366     aInfo.mVideoState.mLastStreamSourceID =
3367         AssertedCast<int64_t>(mVideo.mLastStreamSourceID);
3368   }
3369
3370   CopyUTF8toUTF16(videoDecoderName, aInfo.mVideoDecoderName);
3371   CopyUTF8toUTF16(videoType, aInfo.mVideoType);
3372   aInfo.mVideoWidth =
3373       videoInfo.mDisplay.width < 0 ? 0 : videoInfo.mDisplay.width;
3374   aInfo.mVideoHeight =
3375       videoInfo.mDisplay.height < 0 ? 0 : videoInfo.mDisplay.height;
3376   aInfo.mVideoRate = mVideo.mMeanRate.Mean();
3377   aInfo.mVideoHardwareAccelerated = VideoIsHardwareAccelerated();
3378   aInfo.mVideoNumSamplesOutputTotal =
3379       AssertedCast<int64_t>(mVideo.mNumSamplesOutputTotal);
3380   aInfo.mVideoNumSamplesSkippedTotal =
3381       AssertedCast<int64_t>(mVideo.mNumSamplesSkippedTotal);
3382
3383   // Looking at dropped frames
3384   FrameStatisticsData stats = mFrameStats->GetFrameStatisticsData();
3385   aInfo.mFrameStats.mDroppedDecodedFrames =
3386       AssertedCast<int64_t>(stats.mDroppedDecodedFrames);
3387   aInfo.mFrameStats.mDroppedSinkFrames =
3388       AssertedCast<int64_t>(stats.mDroppedSinkFrames);
3389   aInfo.mFrameStats.mDroppedCompositorFrames =
3390       AssertedCast<int64_t>(stats.mDroppedCompositorFrames);
3391 }
3392
3393 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode) {
3394   MOZ_ASSERT(OnTaskQueue());
3395   return SetNullDecode(TrackType::kVideoTrack, aIsNullDecode);
3396 }
3397
3398 void MediaFormatReader::UpdateCompositor(
3399     already_AddRefed<layers::KnowsCompositor> aCompositor) {
3400   MOZ_ASSERT(OnTaskQueue());
3401   mKnowsCompositor = aCompositor;
3402 }
3403
3404 void MediaFormatReader::SetNullDecode(TrackType aTrack, bool aIsNullDecode) {
3405   MOZ_ASSERT(OnTaskQueue());
3406
3407   auto& decoder = GetDecoderData(aTrack);
3408   if (decoder.mIsNullDecode == aIsNullDecode) {
3409     return;
3410   }
3411
3412   LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3413       TrackTypeToStr(aTrack), decoder.mIsNullDecode, aIsNullDecode);
3414
3415   decoder.mIsNullDecode = aIsNullDecode;
3416   ShutdownDecoder(aTrack);
3417 }
3418
3419 void MediaFormatReader::OnFirstDemuxCompleted(
3420     TrackInfo::TrackType aType,
3421     const RefPtr<MediaTrackDemuxer::SamplesHolder>& aSamples) {
3422   AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3423                       MEDIA_PLAYBACK);
3424   MOZ_ASSERT(OnTaskQueue());
3425
3426   if (mShutdown) {
3427     return;
3428   }
3429
3430   auto& decoder = GetDecoderData(aType);
3431   MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3432   decoder.mFirstDemuxedSampleTime.emplace(aSamples->GetSamples()[0]->mTime);
3433   MaybeResolveMetadataPromise();
3434 }
3435
3436 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType,
3437                                            const MediaResult& aError) {
3438   MOZ_ASSERT(OnTaskQueue());
3439
3440   if (mShutdown) {
3441     return;
3442   }
3443
3444   auto& decoder = GetDecoderData(aType);
3445   MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3446   decoder.mFirstDemuxedSampleTime.emplace(TimeUnit::FromInfinity());
3447   MaybeResolveMetadataPromise();
3448 }
3449
3450 }  // namespace mozilla
3451
3452 #undef NS_DispatchToMainThread
3453 #undef LOGV
3454 #undef LOG