dom/media/MediaTrackGraph.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "MediaTrackGraphImpl.h"
   7 #include "mozilla/MathAlgorithms.h"
   8 #include "mozilla/Unused.h"
   9
  10 #include "AudioSegment.h"
  11 #include "CrossGraphPort.h"
  12 #include "VideoSegment.h"
  13 #include "nsContentUtils.h"
  14 #include "nsPrintfCString.h"
  15 #include "nsServiceManagerUtils.h"
  16 #include "prerror.h"
  17 #include "mozilla/Logging.h"
  18 #include "mozilla/Attributes.h"
  19 #include "ForwardedInputTrack.h"
  20 #include "ImageContainer.h"
  21 #include "AudioCaptureTrack.h"
  22 #include "AudioNodeTrack.h"
  23 #include "AudioNodeExternalInputTrack.h"
  24 #if defined(MOZ_WEBRTC)
  25 #  include "MediaEngineWebRTCAudio.h"
  26 #endif  // MOZ_WEBRTC
  27 #include "MediaTrackListener.h"
  28 #include "mozilla/dom/BaseAudioContextBinding.h"
  29 #include "mozilla/dom/WorkletThread.h"
  30 #include "mozilla/media/MediaUtils.h"
  31 #include <algorithm>
  32 #include "GeckoProfiler.h"
  33 #include "VideoFrameContainer.h"
  34 #include "mozilla/AbstractThread.h"
  35 #include "mozilla/StaticPrefs_dom.h"
  36 #include "mozilla/StaticPrefs_media.h"
  37 #include "transport/runnable_utils.h"
  38 #include "VideoUtils.h"
  39 #include "GraphRunner.h"
  40 #include "Tracing.h"
  41 #include "UnderrunHandler.h"
  42 #include "mozilla/CycleCollectedJSRuntime.h"
  43 #include "mozilla/Preferences.h"
  44
  45 #include "webaudio/blink/DenormalDisabler.h"
  46 #include "webaudio/blink/HRTFDatabaseLoader.h"
  47
  48 using namespace mozilla::layers;
  49 using namespace mozilla::dom;
  50 using namespace mozilla::gfx;
  51 using namespace mozilla::media;
  52
  53 namespace mozilla {
  54
  55 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
  56 #ifdef LOG
  57 #  undef LOG
  58 #endif  // LOG
  59 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
  60
  61 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
  62   return mNativeInputTrack.get();
  63 }
  64
  65 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
  66     CubebUtils::AudioDeviceID aID) {
  67   if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
  68     return mNativeInputTrack.get();
  69   }
  70   for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
  71     if (t->mDeviceId == aID) {
  72       return t.get();
  73     }
  74   }
  75   return nullptr;
  76 }
  77
  78 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
  79   if (mNonNativeInputTracks.IsEmpty()) {
  80     return nullptr;
  81   }
  82   return mNonNativeInputTracks[0].get();
  83 }
  84
  85 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
  86   if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
  87     MOZ_ASSERT(!mNativeInputTrack);
  88     mNativeInputTrack = native;
  89   } else {
  90     NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
  91     MOZ_ASSERT(nonNative);
  92     struct DeviceTrackComparator {
  93      public:
  94       bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
  95                   CubebUtils::AudioDeviceID aDeviceId) const {
  96         return aTrack->mDeviceId == aDeviceId;
  97       }
  98     };
  99     MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
 100                                                DeviceTrackComparator()));
 101     mNonNativeInputTracks.AppendElement(nonNative);
 102   }
 103 }
 104
 105 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
 106   if (aTrack->AsNativeInputTrack()) {
 107     MOZ_ASSERT(mNativeInputTrack);
 108     MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
 109     mNativeInputTrack = nullptr;
 110   } else {
 111     NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
 112     MOZ_ASSERT(nonNative);
 113     DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
 114     MOZ_ASSERT(removed);
 115   }
 116 }
 117
 118 namespace {
 119 /**
 120  * A hash table containing the graph instances, one per Window ID,
 121  * sample rate, and device ID combination.
 122  */
 123 class GraphKey final {
 124  public:
 125   GraphKey(uint64_t aWindowID, TrackRate aSampleRate,
 126            CubebUtils::AudioDeviceID aOutputDeviceID)
 127       : mWindowID(aWindowID),
 128         mSampleRate(aSampleRate),
 129         mOutputDeviceID(aOutputDeviceID) {}
 130   GraphKey(const GraphKey&) = default;
 131   ~GraphKey() = default;
 132   bool operator==(const GraphKey& b) const {
 133     return mWindowID == b.mWindowID && mSampleRate == b.mSampleRate &&
 134            mOutputDeviceID == b.mOutputDeviceID;
 135   }
 136   PLDHashNumber Hash() const {
 137     return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
 138   }
 139
 140  private:
 141   uint64_t mWindowID;
 142   TrackRate mSampleRate;
 143   CubebUtils::AudioDeviceID mOutputDeviceID;
 144 };
 145 nsTHashMap<nsGenericHashKey<GraphKey>, MediaTrackGraphImpl*> gGraphs;
 146 }  // anonymous namespace
 147
 148 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode,
 149                                 MediaSegment* aSegment,
 150                                 MediaSegment* aRawSegment) {
 151   if (aDisabledMode == DisabledTrackMode::ENABLED) {
 152     return;
 153   }
 154   if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
 155     aSegment->ReplaceWithDisabled();
 156     if (aRawSegment) {
 157       aRawSegment->ReplaceWithDisabled();
 158     }
 159   } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
 160     aSegment->ReplaceWithNull();
 161     if (aRawSegment) {
 162       aRawSegment->ReplaceWithNull();
 163     }
 164   } else {
 165     MOZ_CRASH("Unsupported mode");
 166   }
 167 }
 168
 169 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
 170   MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
 171              "All tracks should have been destroyed by messages from the main "
 172              "thread");
 173   LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
 174   LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
 175 }
 176
 177 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
 178   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 179   aTrack->mStartTime = mProcessedTime;
 180
 181   if (aTrack->IsSuspended()) {
 182     mSuspendedTracks.AppendElement(aTrack);
 183     LOG(LogLevel::Debug,
 184         ("%p: Adding media track %p, in the suspended track array", this,
 185          aTrack));
 186   } else {
 187     mTracks.AppendElement(aTrack);
 188     LOG(LogLevel::Debug, ("%p:  Adding media track %p, count %zu", this, aTrack,
 189                           mTracks.Length()));
 190   }
 191
 192   SetTrackOrderDirty();
 193 }
 194
 195 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
 196   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 197   // Remove references in mTrackUpdates before we allow aTrack to die.
 198   // Pending updates are not needed (since the main thread has already given
 199   // up the track) so we will just drop them.
 200   {
 201     MonitorAutoLock lock(mMonitor);
 202     for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
 203       if (mTrackUpdates[i].mTrack == aTrack) {
 204         mTrackUpdates[i].mTrack = nullptr;
 205       }
 206     }
 207   }
 208
 209   // Ensure that mFirstCycleBreaker is updated when necessary.
 210   SetTrackOrderDirty();
 211
 212   UnregisterAllAudioOutputs(aTrack);
 213
 214   if (aTrack->IsSuspended()) {
 215     mSuspendedTracks.RemoveElement(aTrack);
 216   } else {
 217     mTracks.RemoveElement(aTrack);
 218   }
 219
 220   LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
 221                         mTracks.Length()));
 222
 223   NS_RELEASE(aTrack);  // probably destroying it
 224 }
 225
 226 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
 227     const MediaTrack* aTrack, GraphTime aTime) const {
 228   MOZ_ASSERT(
 229       aTime <= mStateComputedTime,
 230       "Don't ask about times where we haven't made blocking decisions yet");
 231   return std::max<TrackTime>(
 232       0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
 233 }
 234
 235 GraphTime MediaTrackGraphImpl::IterationEnd() const {
 236   MOZ_ASSERT(OnGraphThread());
 237   return mIterationEndTime;
 238 }
 239
 240 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
 241     GraphTime aPrevCurrentTime) {
 242   MOZ_ASSERT(OnGraphThread());
 243   for (MediaTrack* track : AllTracks()) {
 244     // Shouldn't have already notified of ended *and* have output!
 245     MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
 246                   !track->mNotifiedEnded);
 247
 248     // Calculate blocked time and fire Blocked/Unblocked events
 249     GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
 250     NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
 251     track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
 252                                                  blockedTime);
 253     LOG(LogLevel::Verbose,
 254         ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
 255          MediaTimeToSeconds(track->mStartTime),
 256          MediaTimeToSeconds(blockedTime)));
 257     track->mStartBlocking = mStateComputedTime;
 258
 259     TrackTime trackCurrentTime =
 260         track->GraphTimeToTrackTime(mStateComputedTime);
 261     if (track->mEnded) {
 262       MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
 263       if (!track->mNotifiedEnded) {
 264         // Playout of this track ended and listeners have not been notified.
 265         track->mNotifiedEnded = true;
 266         SetTrackOrderDirty();
 267         for (const auto& listener : track->mTrackListeners) {
 268           listener->NotifyOutput(this, track->GetEnd());
 269           listener->NotifyEnded(this);
 270         }
 271       }
 272     } else {
 273       for (const auto& listener : track->mTrackListeners) {
 274         listener->NotifyOutput(this, trackCurrentTime);
 275       }
 276     }
 277   }
 278 }
 279
 280 template <typename C, typename Chunk>
 281 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
 282                                                           C& aSegment,
 283                                                           TrackTime aStart,
 284                                                           TrackTime aEnd) {
 285   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 286   MOZ_ASSERT(aTrack);
 287
 288   TrackTime offset = 0;
 289   for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
 290        chunk.Next()) {
 291     if (offset >= aEnd) {
 292       break;
 293     }
 294     offset += chunk->GetDuration();
 295     if (chunk->IsNull() || offset < aStart) {
 296       continue;
 297     }
 298     const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
 299     if (principalHandle != aSegment.GetLastPrincipalHandle()) {
 300       aSegment.SetLastPrincipalHandle(principalHandle);
 301       LOG(LogLevel::Debug,
 302           ("%p: MediaTrack %p, principalHandle "
 303            "changed in %sChunk with duration %lld",
 304            this, aTrack,
 305            aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
 306            (long long)chunk->GetDuration()));
 307       for (const auto& listener : aTrack->mTrackListeners) {
 308         listener->NotifyPrincipalHandleChanged(this, principalHandle);
 309       }
 310     }
 311   }
 312 }
 313
 314 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
 315   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 316   for (MediaTrack* track : AllTracks()) {
 317     TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
 318     TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
 319     if (!track->mSegment) {
 320       continue;
 321     }
 322     if (track->mType == MediaSegment::AUDIO) {
 323       ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
 324           track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
 325     } else if (track->mType == MediaSegment::VIDEO) {
 326       ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
 327           track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
 328     } else {
 329       MOZ_CRASH("Unknown track type");
 330     }
 331   }
 332 }
 333
 334 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
 335                                             GraphTime aEndBlockingDecisions) {
 336   // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
 337   // underrun currently, since we'll always be able to produce data for them
 338   // unless they block on some other track.
 339   if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
 340     return aEndBlockingDecisions;
 341   }
 342   // This track isn't ended or suspended. We don't need to call
 343   // TrackTimeToGraphTime since an underrun is the only thing that can block
 344   // it.
 345   GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
 346 #ifdef DEBUG
 347   if (bufferEnd < mProcessedTime) {
 348     LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
 349                           "bufferEnd %f < mProcessedTime %f (%" PRId64
 350                           " < %" PRId64 "), TrackTime %" PRId64,
 351                           this, aTrack, MediaTimeToSeconds(bufferEnd),
 352                           MediaTimeToSeconds(mProcessedTime), bufferEnd,
 353                           mProcessedTime, aTrack->GetEnd()));
 354     NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
 355   }
 356 #endif
 357   return std::min(bufferEnd, aEndBlockingDecisions);
 358 }
 359
 360 namespace {
 361 // Value of mCycleMarker for unvisited tracks in cycle detection.
 362 const uint32_t NOT_VISITED = UINT32_MAX;
 363 // Value of mCycleMarker for ordered tracks in muted cycles.
 364 const uint32_t IN_MUTED_CYCLE = 1;
 365 }  // namespace
 366
 367 bool MediaTrackGraphImpl::AudioTrackPresent() {
 368   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 369
 370   bool audioTrackPresent = false;
 371   for (MediaTrack* track : mTracks) {
 372     if (track->AsAudioNodeTrack()) {
 373       audioTrackPresent = true;
 374       break;
 375     }
 376
 377     if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
 378       audioTrackPresent = true;
 379       break;
 380     }
 381   }
 382
 383   // We may not have audio input device when we only have AudioNodeTracks. But
 384   // if audioTrackPresent is false, we must have no input device.
 385   MOZ_DIAGNOSTIC_ASSERT_IF(
 386       !audioTrackPresent,
 387       !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
 388
 389   return audioTrackPresent;
 390 }
 391
 392 void MediaTrackGraphImpl::CheckDriver() {
 393   MOZ_ASSERT(OnGraphThread());
 394   // An offline graph has only one driver.
 395   // Otherwise, if a switch is already pending, let that happen.
 396   if (!mRealtime || Switching()) {
 397     return;
 398   }
 399
 400   AudioCallbackDriver* audioCallbackDriver =
 401       CurrentDriver()->AsAudioCallbackDriver();
 402   if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
 403     for (PendingResumeOperation& op : mPendingResumeOperations) {
 404       op.Apply(this);
 405     }
 406     mPendingResumeOperations.Clear();
 407   }
 408
 409   // Note that this looks for any audio tracks, input or output, and switches
 410   // to a SystemClockDriver if there are none active or no resume operations
 411   // to make any active.
 412   bool needAudioCallbackDriver =
 413       !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
 414   if (!needAudioCallbackDriver) {
 415     if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
 416       SwitchAtNextIteration(
 417           new SystemClockDriver(this, CurrentDriver(), mSampleRate));
 418     }
 419     return;
 420   }
 421
 422   NativeInputTrack* native =
 423       mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
 424   CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
 425   uint32_t inputChannelCount =
 426       native ? AudioInputChannelCount(native->mDeviceId) : 0;
 427   AudioInputType inputPreference =
 428       native ? AudioInputDevicePreference(native->mDeviceId)
 429              : AudioInputType::Unknown;
 430
 431   uint32_t graphOutputChannelCount = AudioOutputChannelCount();
 432   if (!audioCallbackDriver) {
 433     if (graphOutputChannelCount > 0) {
 434       AudioCallbackDriver* driver = new AudioCallbackDriver(
 435           this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
 436           inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
 437       SwitchAtNextIteration(driver);
 438     }
 439     return;
 440   }
 441
 442   // Check if this graph should switch to a different number of output channels.
 443   // Generally, a driver switch is explicitly made by an event (e.g., setting
 444   // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
 445   // directly playing back via another HTMLMediaElement, the number of channels
 446   // of the media determines how many channels to output, and it can change
 447   // dynamically.
 448   if (graphOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
 449     AudioCallbackDriver* driver = new AudioCallbackDriver(
 450         this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
 451         inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
 452     SwitchAtNextIteration(driver);
 453   }
 454 }
 455
 456 void MediaTrackGraphImpl::UpdateTrackOrder() {
 457   if (!mTrackOrderDirty) {
 458     return;
 459   }
 460
 461   mTrackOrderDirty = false;
 462
 463   // The algorithm for finding cycles is based on Tim Leslie's iterative
 464   // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
 465   // connected components (SCC) algorithm.  There are variations (a) to
 466   // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
 467   // re-run the algorithm over SCCs with breaks at DelayNodes.
 468   //
 469   // [1] http://www.timl.id.au/?p=327
 470   // [2]
 471   // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
 472   // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
 473   //
 474   // There are two stacks.  One for the depth-first search (DFS),
 475   mozilla::LinkedList<MediaTrack> dfsStack;
 476   // and another for tracks popped from the DFS stack, but still being
 477   // considered as part of SCCs involving tracks on the stack.
 478   mozilla::LinkedList<MediaTrack> sccStack;
 479
 480   // An index into mTracks for the next track found with no unsatisfied
 481   // upstream dependencies.
 482   uint32_t orderedTrackCount = 0;
 483
 484   for (uint32_t i = 0; i < mTracks.Length(); ++i) {
 485     MediaTrack* t = mTracks[i];
 486     ProcessedMediaTrack* pt = t->AsProcessedTrack();
 487     if (pt) {
 488       // The dfsStack initially contains a list of all processed tracks in
 489       // unchanged order.
 490       dfsStack.insertBack(t);
 491       pt->mCycleMarker = NOT_VISITED;
 492     } else {
 493       // SourceMediaTracks have no inputs and so can be ordered now.
 494       mTracks[orderedTrackCount] = t;
 495       ++orderedTrackCount;
 496     }
 497   }
 498
 499   // mNextStackMarker corresponds to "index" in Tarjan's algorithm.  It is a
 500   // counter to label mCycleMarker on the next visited track in the DFS
 501   // uniquely in the set of visited tracks that are still being considered.
 502   //
 503   // In this implementation, the counter descends so that the values are
 504   // strictly greater than the values that mCycleMarker takes when the track
 505   // has been ordered (0 or IN_MUTED_CYCLE).
 506   //
 507   // Each new track labelled, as the DFS searches upstream, receives a value
 508   // less than those used for all other tracks being considered.
 509   uint32_t nextStackMarker = NOT_VISITED - 1;
 510   // Reset list of DelayNodes in cycles stored at the tail of mTracks.
 511   mFirstCycleBreaker = mTracks.Length();
 512
 513   // Rearrange dfsStack order as required to DFS upstream and pop tracks
 514   // in processing order to place in mTracks.
 515   while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
 516     const auto& inputs = pt->mInputs;
 517     MOZ_ASSERT(pt->AsProcessedTrack());
 518     if (pt->mCycleMarker == NOT_VISITED) {
 519       // Record the position on the visited stack, so that any searches
 520       // finding this track again know how much of the stack is in the cycle.
 521       pt->mCycleMarker = nextStackMarker;
 522       --nextStackMarker;
 523       // Not-visited input tracks should be processed first.
 524       // SourceMediaTracks have already been ordered.
 525       for (uint32_t i = inputs.Length(); i--;) {
 526         if (inputs[i]->GetSource()->IsSuspended()) {
 527           continue;
 528         }
 529         auto input = inputs[i]->GetSource()->AsProcessedTrack();
 530         if (input && input->mCycleMarker == NOT_VISITED) {
 531           // It can be that this track has an input which is from a suspended
 532           // AudioContext.
 533           if (input->isInList()) {
 534             input->remove();
 535             dfsStack.insertFront(input);
 536           }
 537         }
 538       }
 539       continue;
 540     }
 541
 542     // Returning from DFS.  Pop from dfsStack.
 543     pt->remove();
 544
 545     // cycleStackMarker keeps track of the highest marker value on any
 546     // upstream track, if any, found receiving input, directly or indirectly,
 547     // from the visited stack (and so from |ps|, making a cycle).  In a
 548     // variation from Tarjan's SCC algorithm, this does not include |ps|
 549     // unless it is part of the cycle.
 550     uint32_t cycleStackMarker = 0;
 551     for (uint32_t i = inputs.Length(); i--;) {
 552       if (inputs[i]->GetSource()->IsSuspended()) {
 553         continue;
 554       }
 555       auto input = inputs[i]->GetSource()->AsProcessedTrack();
 556       if (input) {
 557         cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
 558       }
 559     }
 560
 561     if (cycleStackMarker <= IN_MUTED_CYCLE) {
 562       // All inputs have been ordered and their stack markers have been removed.
 563       // This track is not part of a cycle.  It can be processed next.
 564       pt->mCycleMarker = 0;
 565       mTracks[orderedTrackCount] = pt;
 566       ++orderedTrackCount;
 567       continue;
 568     }
 569
 570     // A cycle has been found.  Record this track for ordering when all
 571     // tracks in this SCC have been popped from the DFS stack.
 572     sccStack.insertFront(pt);
 573
 574     if (cycleStackMarker > pt->mCycleMarker) {
 575       // Cycles have been found that involve tracks that remain on the stack.
 576       // Leave mCycleMarker indicating the most downstream (last) track on
 577       // the stack known to be part of this SCC.  In this way, any searches on
 578       // other paths that find |ps| will know (without having to traverse from
 579       // this track again) that they are part of this SCC (i.e. part of an
 580       // intersecting cycle).
 581       pt->mCycleMarker = cycleStackMarker;
 582       continue;
 583     }
 584
 585     // |pit| is the root of an SCC involving no other tracks on dfsStack, the
 586     // complete SCC has been recorded, and tracks in this SCC are part of at
 587     // least one cycle.
 588     MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
 589     // If there are DelayNodes in this SCC, then they may break the cycles.
 590     bool haveDelayNode = false;
 591     auto next = sccStack.getFirst();
 592     // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
 593     // (There may be other tracks later in sccStack from other incompletely
 594     // searched SCCs, involving tracks still on dfsStack.)
 595     //
 596     // DelayNodes in cycles must behave differently from those not in cycles,
 597     // so all DelayNodes in the SCC must be identified.
 598     while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
 599                        cycleStackMarker) {
 600       auto nt = next->AsAudioNodeTrack();
 601       // Get next before perhaps removing from list below.
 602       next = next->getNext();
 603       if (nt && nt->Engine()->AsDelayNodeEngine()) {
 604         haveDelayNode = true;
 605         // DelayNodes break cycles by producing their output in a
 606         // preprocessing phase; they do not need to be ordered before their
 607         // consumers.  Order them at the tail of mTracks so that they can be
 608         // handled specially.  Do so now, so that DFS ignores them.
 609         nt->remove();
 610         nt->mCycleMarker = 0;
 611         --mFirstCycleBreaker;
 612         mTracks[mFirstCycleBreaker] = nt;
 613       }
 614     }
 615     auto after_scc = next;
 616     while ((next = sccStack.getFirst()) != after_scc) {
 617       next->remove();
 618       auto removed = static_cast<ProcessedMediaTrack*>(next);
 619       if (haveDelayNode) {
 620         // Return tracks to the DFS stack again (to order and detect cycles
 621         // without delayNodes).  Any of these tracks that are still inputs
 622         // for tracks on the visited stack must be returned to the front of
 623         // the stack to be ordered before their dependents.  We know that none
 624         // of these tracks need input from tracks on the visited stack, so
 625         // they can all be searched and ordered before the current stack head
 626         // is popped.
 627         removed->mCycleMarker = NOT_VISITED;
 628         dfsStack.insertFront(removed);
 629       } else {
 630         // Tracks in cycles without any DelayNodes must be muted, and so do
 631         // not need input and can be ordered now.  They must be ordered before
 632         // their consumers so that their muted output is available.
 633         removed->mCycleMarker = IN_MUTED_CYCLE;
 634         mTracks[orderedTrackCount] = removed;
 635         ++orderedTrackCount;
 636       }
 637     }
 638   }
 639
 640   MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
 641 }
 642
 643 TrackTime MediaTrackGraphImpl::PlayAudio(AudioMixer* aMixer,
 644                                          const TrackKeyAndVolume& aTkv,
 645                                          GraphTime aPlayedTime) {
 646   MOZ_ASSERT(OnGraphThread());
 647   MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
 648   MOZ_ASSERT(aMixer, "Can only play audio if there's a mixer");
 649
 650   TrackTime ticksWritten = 0;
 651
 652   ticksWritten = 0;
 653   MediaTrack* track = aTkv.mTrack;
 654   AudioSegment* audio = track->GetData<AudioSegment>();
 655   AudioSegment output;
 656
 657   TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
 658
 659   // We don't update Track->mTracksStartTime here to account for time spent
 660   // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
 661   // the blocked period has completed. But we do need to make sure we play
 662   // from the right offsets in the track buffer, even if we've already
 663   // written silence for some amount of blocked time after the current time.
 664   GraphTime t = aPlayedTime;
 665   while (t < mStateComputedTime) {
 666     bool blocked = t >= track->mStartBlocking;
 667     GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
 668     NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
 669
 670     // Check how many ticks of sound we can provide if we are blocked some
 671     // time in the middle of this cycle.
 672     TrackTime toWrite = end - t;
 673
 674     if (blocked) {
 675       output.InsertNullDataAtStart(toWrite);
 676       ticksWritten += toWrite;
 677       LOG(LogLevel::Verbose,
 678           ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
 679            "%f to %f (%" PRId64 " to %" PRId64 ")",
 680            this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
 681            offset, offset + toWrite));
 682     } else {
 683       TrackTime endTicksNeeded = offset + toWrite;
 684       TrackTime endTicksAvailable = audio->GetDuration();
 685
 686       if (endTicksNeeded <= endTicksAvailable) {
 687         LOG(LogLevel::Verbose,
 688             ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
 689              "(samples %" PRId64 " to %" PRId64 ")",
 690              this, track, toWrite, MediaTimeToSeconds(t),
 691              MediaTimeToSeconds(end), offset, endTicksNeeded));
 692         output.AppendSlice(*audio, offset, endTicksNeeded);
 693         ticksWritten += toWrite;
 694         offset = endTicksNeeded;
 695       } else {
 696         // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
 697         // ended."); If we are at the end of the track, maybe write the
 698         // remaining samples, and pad with/output silence.
 699         if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
 700           output.AppendSlice(*audio, offset, endTicksAvailable);
 701
 702           LOG(LogLevel::Verbose,
 703               ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
 704                "(samples %" PRId64 " to %" PRId64 ")",
 705                this, track, toWrite, MediaTimeToSeconds(t),
 706                MediaTimeToSeconds(end), offset, endTicksNeeded));
 707           uint32_t available = endTicksAvailable - offset;
 708           ticksWritten += available;
 709           toWrite -= available;
 710           offset = endTicksAvailable;
 711         }
 712         output.AppendNullData(toWrite);
 713         LOG(LogLevel::Verbose,
 714             ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
 715              "%f (samples %" PRId64 " to %" PRId64 ")",
 716              this, track, toWrite, MediaTimeToSeconds(t),
 717              MediaTimeToSeconds(end), offset, endTicksNeeded));
 718         ticksWritten += toWrite;
 719       }
 720       output.ApplyVolume(mGlobalVolume * aTkv.mVolume);
 721     }
 722     t = end;
 723
 724     uint32_t outputChannels;
 725     // Use the number of channel the driver expects: this is the number of
 726     // channel that can be output by the underlying system level audio stream.
 727     // Fall back to something sensible if this graph is being driven by a normal
 728     // thread (this can happen when there are no output devices, etc.).
 729     if (CurrentDriver()->AsAudioCallbackDriver()) {
 730       outputChannels =
 731           CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
 732     } else {
 733       outputChannels = AudioOutputChannelCount();
 734     }
 735     output.WriteTo(*aMixer, outputChannels, mSampleRate);
 736   }
 737   return ticksWritten;
 738 }
 739
 740 DeviceInputTrack* MediaTrackGraphImpl::GetDeviceInputTrackMainThread(
 741     CubebUtils::AudioDeviceID aID) {
 742   MOZ_ASSERT(NS_IsMainThread());
 743   return mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
 744 }
 745
 746 NativeInputTrack* MediaTrackGraphImpl::GetNativeInputTrackMainThread() {
 747   MOZ_ASSERT(NS_IsMainThread());
 748   return mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
 749 }
 750
 751 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
 752   MOZ_ASSERT(OnGraphThread());
 753   LOG(LogLevel::Debug,
 754       ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
 755
 756   mDeviceInputTrackManagerGraphThread.Add(aTrack);
 757
 758   if (aTrack->AsNativeInputTrack()) {
 759     // Switch Drivers since we're adding input (to input-only or full-duplex)
 760     AudioCallbackDriver* driver = new AudioCallbackDriver(
 761         this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
 762         AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID,
 763         aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId));
 764     LOG(LogLevel::Debug,
 765         ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
 766          this, driver));
 767     SwitchAtNextIteration(driver);
 768   } else {
 769     NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
 770     MOZ_ASSERT(nonNative);
 771     // Start non-native input right away.
 772     nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
 773         MakeRefPtr<AudioInputSourceListener>(nonNative),
 774         nonNative->GenerateSourceId(), nonNative->mDeviceId,
 775         AudioInputChannelCount(nonNative->mDeviceId),
 776         AudioInputDevicePreference(nonNative->mDeviceId) ==
 777             AudioInputType::Voice,
 778         nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate(),
 779         StaticPrefs::media_clockdrift_buffering()));
 780   }
 781 }
 782
 783 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
 784   MOZ_ASSERT(NS_IsMainThread());
 785   MOZ_ASSERT(aTrack);
 786
 787   LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
 788                         aTrack, aTrack->mDeviceId));
 789
 790   class Message : public ControlMessage {
 791    public:
 792     Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
 793         : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
 794     void Run() override {
 795       TRACE("MTG::OpenAudioInputImpl ControlMessage");
 796       mGraph->OpenAudioInputImpl(mInputTrack);
 797     }
 798     MediaTrackGraphImpl* mGraph;
 799     DeviceInputTrack* mInputTrack;
 800   };
 801
 802   mDeviceInputTrackManagerMainThread.Add(aTrack);
 803
 804   this->AppendMessage(MakeUnique<Message>(this, aTrack));
 805 }
 806
 807 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
 808   MOZ_ASSERT(OnGraphThread());
 809
 810   LOG(LogLevel::Debug,
 811       ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
 812
 813   if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
 814     nonNative->StopAudio();
 815     mDeviceInputTrackManagerGraphThread.Remove(aTrack);
 816     return;
 817   }
 818
 819   MOZ_ASSERT(aTrack->AsNativeInputTrack());
 820
 821   mDeviceInputTrackManagerGraphThread.Remove(aTrack);
 822
 823   // Switch Drivers since we're adding or removing an input (to nothing/system
 824   // or output only)
 825   bool audioTrackPresent = AudioTrackPresent();
 826
 827   GraphDriver* driver;
 828   if (audioTrackPresent) {
 829     // We still have audio output
 830     LOG(LogLevel::Debug,
 831         ("%p: CloseInput: output present (AudioCallback)", this));
 832
 833     driver = new AudioCallbackDriver(
 834         this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
 835         AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID, nullptr,
 836         AudioInputDevicePreference(aTrack->mDeviceId));
 837     SwitchAtNextIteration(driver);
 838   } else if (CurrentDriver()->AsAudioCallbackDriver()) {
 839     LOG(LogLevel::Debug,
 840         ("%p: CloseInput: no output present (SystemClockCallback)", this));
 841
 842     driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
 843     SwitchAtNextIteration(driver);
 844   }  // else SystemClockDriver->SystemClockDriver, no switch
 845 }
 846
 847 void MediaTrackGraphImpl::RegisterAudioOutput(MediaTrack* aTrack, void* aKey) {
 848   MOZ_ASSERT(OnGraphThread());
 849   MOZ_ASSERT(!mAudioOutputs.Contains(TrackAndKey{aTrack, aKey}));
 850
 851   TrackKeyAndVolume* tkv = mAudioOutputs.AppendElement();
 852   tkv->mTrack = aTrack;
 853   tkv->mKey = aKey;
 854   tkv->mVolume = 1.0;
 855
 856   if (!CurrentDriver()->AsAudioCallbackDriver() && !Switching()) {
 857     NativeInputTrack* native =
 858         mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
 859     CubebUtils::AudioDeviceID inputDevice =
 860         native ? native->mDeviceId : nullptr;
 861     uint32_t inputChannelCount =
 862         native ? AudioInputChannelCount(native->mDeviceId) : 0;
 863     AudioInputType inputPreference =
 864         native ? AudioInputDevicePreference(native->mDeviceId)
 865                : AudioInputType::Unknown;
 866
 867     AudioCallbackDriver* driver = new AudioCallbackDriver(
 868         this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
 869         inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
 870     SwitchAtNextIteration(driver);
 871   }
 872 }
 873
 874 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
 875   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 876
 877   mAudioOutputs.RemoveElementsBy([aTrack](const TrackKeyAndVolume& aTkv) {
 878     return aTkv.mTrack == aTrack;
 879   });
 880 }
 881
 882 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
 883                                                 void* aKey) {
 884   MOZ_ASSERT(OnGraphThreadOrNotRunning());
 885
 886   DebugOnly<bool> removed =
 887       mAudioOutputs.RemoveElement(TrackAndKey{aTrack, aKey});
 888   MOZ_ASSERT(removed, "Audio output not found");
 889 }
 890
 891 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
 892   MOZ_ASSERT(NS_IsMainThread());
 893   MOZ_ASSERT(aTrack);
 894
 895   LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
 896                         this, aTrack, aTrack->mDeviceId));
 897
 898   class Message : public ControlMessage {
 899    public:
 900     Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
 901         : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
 902     void Run() override {
 903       TRACE("MTG::CloseAudioInputImpl ControlMessage");
 904       mGraph->CloseAudioInputImpl(mInputTrack);
 905     }
 906     MediaTrackGraphImpl* mGraph;
 907     DeviceInputTrack* mInputTrack;
 908   };
 909
 910   // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
 911   // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
 912   // for more details.
 913   mDeviceInputTrackManagerMainThread.Remove(aTrack);
 914
 915   this->AppendMessage(MakeUnique<Message>(this, aTrack));
 916
 917   if (aTrack->AsNativeInputTrack()) {
 918     LOG(LogLevel::Debug,
 919         ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
 920     SetNewNativeInput();
 921   }
 922 }
 923
 924 // All AudioInput listeners get the same speaker data (at least for now).
 925 void MediaTrackGraphImpl::NotifyOutputData(AudioDataValue* aBuffer,
 926                                            size_t aFrames, TrackRate aRate,
 927                                            uint32_t aChannels) {
 928   if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
 929     return;
 930   }
 931
 932 #if defined(MOZ_WEBRTC)
 933   for (const auto& track : mTracks) {
 934     if (const auto& t = track->AsAudioProcessingTrack()) {
 935       t->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
 936     }
 937   }
 938 #endif
 939 }
 940
 941 void MediaTrackGraphImpl::NotifyInputStopped() {
 942   NativeInputTrack* native =
 943       mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
 944   if (!native) {
 945     return;
 946   }
 947   native->NotifyInputStopped(this);
 948 }
 949
 950 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
 951                                           size_t aFrames, TrackRate aRate,
 952                                           uint32_t aChannels,
 953                                           uint32_t aAlreadyBuffered) {
 954   // Either we have an audio input device, or we just removed the audio input
 955   // this iteration, and we're switching back to an output-only driver next
 956   // iteration.
 957   NativeInputTrack* native =
 958       mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
 959   MOZ_ASSERT(native || Switching());
 960   if (!native) {
 961     return;
 962   }
 963   native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
 964                           aAlreadyBuffered);
 965 }
 966
 967 void MediaTrackGraphImpl::DeviceChangedImpl() {
 968   MOZ_ASSERT(OnGraphThread());
 969   NativeInputTrack* native =
 970       mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
 971   if (!native) {
 972     return;
 973   }
 974   native->DeviceChanged(this);
 975 }
 976
 977 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
 978   MOZ_ASSERT(OnGraphThread());
 979   mMaxOutputChannelCount = aMaxChannelCount;
 980 }
 981
 982 void MediaTrackGraphImpl::DeviceChanged() {
 983   // This is safe to be called from any thread: this message comes from an
 984   // underlying platform API, and we don't have much guarantees. If it is not
 985   // called from the main thread (and it probably will rarely be), it will post
 986   // itself to the main thread, and the actual device change message will be ran
 987   // and acted upon on the graph thread.
 988   if (!NS_IsMainThread()) {
 989     RefPtr<nsIRunnable> runnable = WrapRunnable(
 990         RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
 991     mMainThread->Dispatch(runnable.forget());
 992     return;
 993   }
 994
 995   class Message : public ControlMessage {
 996    public:
 997     explicit Message(MediaTrackGraph* aGraph)
 998         : ControlMessage(nullptr),
 999           mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
1000     void Run() override {
1001       TRACE("MTG::DeviceChangeImpl ControlMessage");
1002       mGraphImpl->DeviceChangedImpl();
1003     }
1004     // We know that this is valid, because the graph can't shutdown if it has
1005     // messages.
1006     MediaTrackGraphImpl* mGraphImpl;
1007   };
1008
1009   if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
1010     // This is a special case where the origin of this event cannot control the
1011     // lifetime of the graph, because the graph is controling the lifetime of
1012     // the AudioCallbackDriver where the event originated.
1013     // We know the graph is soon going away, so there's no need to notify about
1014     // this device change.
1015     return;
1016   }
1017
1018   // Reset the latency, it will get fetched again next time it's queried.
1019   MOZ_ASSERT(NS_IsMainThread());
1020   mAudioOutputLatency = 0.0;
1021
1022   // Dispatch to the bg thread to do the (potentially expensive) query of the
1023   // maximum channel count, and then dispatch back to the main thread, then to
1024   // the graph, with the new info.
1025   RefPtr<MediaTrackGraphImpl> self = this;
1026   NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1027       "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
1028         uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
1029         self->Dispatch(NS_NewRunnableFunction(
1030             "MaxChannelCountUpdateToMainThread",
1031             [self{self}, maxChannelCount]() {
1032               class MessageToGraph : public ControlMessage {
1033                public:
1034                 explicit MessageToGraph(MediaTrackGraph* aGraph,
1035                                         uint32_t aMaxChannelCount)
1036                     : ControlMessage(nullptr),
1037                       mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
1038                       mMaxChannelCount(aMaxChannelCount) {}
1039                 void Run() override {
1040                   TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1041                   mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
1042                 }
1043                 MediaTrackGraphImpl* mGraphImpl;
1044                 uint32_t mMaxChannelCount;
1045               };
1046               self->AppendMessage(
1047                   MakeUnique<MessageToGraph>(self, maxChannelCount));
1048             }));
1049       }));
1050
1051   AppendMessage(MakeUnique<Message>(this));
1052 }
1053
1054 static const char* GetAudioInputTypeString(const AudioInputType& aType) {
1055   return aType == AudioInputType::Voice ? "Voice" : "Unknown";
1056 }
1057
1058 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1059   MOZ_ASSERT(OnGraphThread());
1060
1061   LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
1062
1063   DeviceInputTrack* track =
1064       mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
1065   if (!track) {
1066     LOG(LogLevel::Debug,
1067         ("%p: No DeviceInputTrack for this device. Ignore", this));
1068     return;
1069   }
1070
1071   bool needToSwitch = false;
1072
1073   if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
1074     if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
1075       LOG(LogLevel::Debug,
1076           ("%p: %u-channel non-native input device %p (track %p) is "
1077            "re-configured to %d-channel",
1078            this, nonNative->NumberOfChannels(), aID, track,
1079            AudioInputChannelCount(aID)));
1080       needToSwitch = true;
1081     }
1082     if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
1083       LOG(LogLevel::Debug,
1084           ("%p: %s-type non-native input device %p (track %p) is re-configured "
1085            "to %s-type",
1086            this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
1087            track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1088       needToSwitch = true;
1089     }
1090
1091     if (needToSwitch) {
1092       nonNative->StopAudio();
1093       nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
1094           MakeRefPtr<AudioInputSourceListener>(nonNative),
1095           nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
1096           AudioInputDevicePreference(aID) == AudioInputType::Voice,
1097           nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate(),
1098           StaticPrefs::media_clockdrift_buffering()));
1099     }
1100
1101     return;
1102   }
1103
1104   MOZ_ASSERT(track->AsNativeInputTrack());
1105
1106   if (AudioCallbackDriver* audioCallbackDriver =
1107           CurrentDriver()->AsAudioCallbackDriver()) {
1108     if (audioCallbackDriver->InputChannelCount() !=
1109         AudioInputChannelCount(aID)) {
1110       LOG(LogLevel::Debug,
1111           ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1112            "re-configured to %d-channel",
1113            this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
1114            AudioInputChannelCount(aID)));
1115       needToSwitch = true;
1116     }
1117     if (audioCallbackDriver->InputDevicePreference() !=
1118         AudioInputDevicePreference(aID)) {
1119       LOG(LogLevel::Debug,
1120           ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1121            "re-configured to %s-type",
1122            this,
1123            GetAudioInputTypeString(
1124                audioCallbackDriver->InputDevicePreference()),
1125            audioCallbackDriver,
1126            GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1127       needToSwitch = true;
1128     }
1129   } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1130     // We're already in the process of switching to a audio callback driver,
1131     // which will happen at the next iteration.
1132     // However, maybe it's not the correct number of channels. Re-query the
1133     // correct channel amount at this time.
1134     needToSwitch = true;
1135   }
1136
1137   if (needToSwitch) {
1138     AudioCallbackDriver* newDriver = new AudioCallbackDriver(
1139         this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
1140         AudioInputChannelCount(aID), mOutputDeviceID, aID,
1141         AudioInputDevicePreference(aID));
1142     SwitchAtNextIteration(newDriver);
1143   }
1144 }
1145
1146 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1147   // either we're on the right thread (and calling CurrentDriver() is safe),
1148   // or we're going to fail the assert anyway, so don't cross-check
1149   // via CurrentDriver().
1150   return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
1151 }
1152
1153 bool MediaTrackGraphImpl::OnGraphThread() const {
1154   // we're on the right thread (and calling mDriver is safe),
1155   MOZ_ASSERT(mDriver);
1156   if (mGraphRunner && mGraphRunner->OnThread()) {
1157     return true;
1158   }
1159   return mDriver->OnThread();
1160 }
1161
1162 bool MediaTrackGraphImpl::Destroyed() const {
1163   MOZ_ASSERT(NS_IsMainThread());
1164   return !mSelfRef;
1165 }
1166
1167 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1168   MOZ_ASSERT(OnGraphThreadOrNotRunning());
1169   if (mRealtime) {
1170     return true;
1171   }
1172
1173   TimeStamp now = TimeStamp::Now();
1174   // For offline graphs, update now if it has been long enough since the last
1175   // update, or if it has reached the end.
1176   if ((now - mLastMainThreadUpdate).ToMilliseconds() >
1177           CurrentDriver()->IterationDuration() ||
1178       mStateComputedTime >= mEndTime) {
1179     mLastMainThreadUpdate = now;
1180     return true;
1181   }
1182   return false;
1183 }
1184
1185 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
1186   MOZ_ASSERT(OnGraphThreadOrNotRunning());
1187   mMonitor.AssertCurrentThreadOwns();
1188
1189   // We don't want to frequently update the main thread about timing update
1190   // when we are not running in realtime.
1191   if (aFinalUpdate || ShouldUpdateMainThread()) {
1192     // Strip updates that will be obsoleted below, so as to keep the length of
1193     // mTrackUpdates sane.
1194     size_t keptUpdateCount = 0;
1195     for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
1196       MediaTrack* track = mTrackUpdates[i].mTrack;
1197       // RemoveTrackGraphThread() clears mTrack in updates for
1198       // tracks that are removed from the graph.
1199       MOZ_ASSERT(!track || track->GraphImpl() == this);
1200       if (!track || track->MainThreadNeedsUpdates()) {
1201         // Discard this update as it has either been cleared when the track
1202         // was destroyed or there will be a newer update below.
1203         continue;
1204       }
1205       if (keptUpdateCount != i) {
1206         mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
1207         MOZ_ASSERT(!mTrackUpdates[i].mTrack);
1208       }
1209       ++keptUpdateCount;
1210     }
1211     mTrackUpdates.TruncateLength(keptUpdateCount);
1212
1213     mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
1214                               mSuspendedTracks.Length());
1215     for (MediaTrack* track : AllTracks()) {
1216       if (!track->MainThreadNeedsUpdates()) {
1217         continue;
1218       }
1219       TrackUpdate* update = mTrackUpdates.AppendElement();
1220       update->mTrack = track;
1221       // No blocking to worry about here, since we've passed
1222       // UpdateCurrentTimeForTracks.
1223       update->mNextMainThreadCurrentTime =
1224           track->GraphTimeToTrackTime(mProcessedTime);
1225       update->mNextMainThreadEnded = track->mNotifiedEnded;
1226     }
1227     mNextMainThreadGraphTime = mProcessedTime;
1228     if (!mPendingUpdateRunnables.IsEmpty()) {
1229       mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
1230     }
1231   }
1232
1233   // If this is the final update, then a stable state event will soon be
1234   // posted just before this thread finishes, and so there is no need to also
1235   // post here.
1236   if (!aFinalUpdate &&
1237       // Don't send the message to the main thread if it's not going to have
1238       // any work to do.
1239       !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
1240     EnsureStableStateEventPosted();
1241   }
1242 }
1243
1244 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
1245   if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
1246     return aTime;
1247   }
1248   return RoundUpToNextAudioBlock(aTime);
1249 }
1250
1251 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
1252   uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
1253   uint64_t nextBlock = block + 1;
1254   GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1255   return nextTime;
1256 }
1257
1258 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1259     uint32_t aTrackIndex, TrackRate aSampleRate) {
1260   MOZ_ASSERT(OnGraphThread());
1261   MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
1262              "Cycle breaker is not AudioNodeTrack?");
1263
1264   while (mProcessedTime < mStateComputedTime) {
1265     // Microtask checkpoints are in between render quanta.
1266     nsAutoMicroTask mt;
1267
1268     GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
1269     for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
1270       auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
1271       MOZ_ASSERT(nt->AsAudioNodeTrack());
1272       nt->ProduceOutputBeforeInput(mProcessedTime);
1273     }
1274     for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
1275       ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
1276       if (pt) {
1277         pt->ProcessInput(
1278             mProcessedTime, next,
1279             (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
1280       }
1281     }
1282     mProcessedTime = next;
1283   }
1284   NS_ASSERTION(mProcessedTime == mStateComputedTime,
1285                "Something went wrong with rounding to block boundaries");
1286 }
1287
1288 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1289     UniquePtr<ControlMessage> aMessage) {
1290   MOZ_ASSERT(OnGraphThread());
1291
1292   if (mFrontMessageQueue.IsEmpty()) {
1293     mFrontMessageQueue.AppendElement();
1294   }
1295
1296   // Only one block is used for messages from the graph thread.
1297   MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1298   mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
1299 }
1300
1301 void MediaTrackGraphImpl::RunMessagesInQueue() {
1302   TRACE("MTG::RunMessagesInQueue");
1303   MOZ_ASSERT(OnGraphThread());
1304   // Calculate independent action times for each batch of messages (each
1305   // batch corresponding to an event loop task). This isolates the performance
1306   // of different scripts to some extent.
1307   for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1308     nsTArray<UniquePtr<ControlMessage>>& messages =
1309         mFrontMessageQueue[i].mMessages;
1310
1311     for (uint32_t j = 0; j < messages.Length(); ++j) {
1312       TRACE("ControlMessage::Run");
1313       messages[j]->Run();
1314     }
1315   }
1316   mFrontMessageQueue.Clear();
1317 }
1318
1319 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
1320   TRACE("MTG::UpdateGraph");
1321   MOZ_ASSERT(OnGraphThread());
1322   MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1323   // The next state computed time can be the same as the previous: it
1324   // means the driver would have been blocking indefinitly, but the graph has
1325   // been woken up right after having been to sleep.
1326   MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1327
1328   CheckDriver();
1329   UpdateTrackOrder();
1330
1331   // Always do another iteration if there are tracks waiting to resume.
1332   bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
1333
1334   for (MediaTrack* track : mTracks) {
1335     if (SourceMediaTrack* is = track->AsSourceTrack()) {
1336       ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
1337       is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
1338     }
1339     if (track->mEnded) {
1340       // The track's not suspended, and since it's ended, underruns won't
1341       // stop it playing out. So there's no blocking other than what we impose
1342       // here.
1343       GraphTime endTime = track->GetEnd() + track->mStartTime;
1344       if (endTime <= mStateComputedTime) {
1345         LOG(LogLevel::Verbose,
1346             ("%p: MediaTrack %p is blocked due to being ended", this, track));
1347         track->mStartBlocking = mStateComputedTime;
1348       } else {
1349         LOG(LogLevel::Verbose,
1350             ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1351              "time %f, end at %f)",
1352              this, track, MediaTimeToSeconds(mStateComputedTime),
1353              MediaTimeToSeconds(endTime)));
1354         // Data can't be added to a ended track, so underruns are irrelevant.
1355         MOZ_ASSERT(endTime <= aEndBlockingDecisions);
1356         track->mStartBlocking = endTime;
1357       }
1358     } else {
1359       track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
1360
1361 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1362       if (SourceMediaTrack* s = track->AsSourceTrack()) {
1363         if (s->Ended()) {
1364           continue;
1365         }
1366         {
1367           MutexAutoLock lock(s->mMutex);
1368           if (!s->mUpdateTrack->mPullingEnabled) {
1369             // The invariant that data must be provided is only enforced when
1370             // pulling.
1371             continue;
1372           }
1373         }
1374         if (track->GetEnd() <
1375             track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
1376           LOG(LogLevel::Error,
1377               ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1378                "but wasn't fed "
1379                "enough data. TrackListeners=%zu. Track-end=%f, "
1380                "Iteration-end=%f",
1381                this, track,
1382                (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
1383                track->mTrackListeners.Length(),
1384                MediaTimeToSeconds(track->GetEnd()),
1385                MediaTimeToSeconds(
1386                    track->GraphTimeToTrackTime(aEndBlockingDecisions))));
1387           MOZ_DIAGNOSTIC_ASSERT(false,
1388                                 "A non-ended SourceMediaTrack wasn't fed "
1389                                 "enough data by NotifyPull");
1390         }
1391       }
1392 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1393     }
1394   }
1395
1396   for (MediaTrack* track : mSuspendedTracks) {
1397     track->mStartBlocking = mStateComputedTime;
1398   }
1399
1400   // If the loop is woken up so soon that IterationEnd() barely advances or
1401   // if an offline graph is not currently rendering, we end up having
1402   // aEndBlockingDecisions == mStateComputedTime.
1403   // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1404   // empty, Process() will not find any unblocked track and so will not
1405   // ensure another iteration.  If the graph should be rendering, then ensure
1406   // another iteration to render.
1407   if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
1408                               mStateComputedTime < mEndTime)) {
1409     EnsureNextIteration();
1410   }
1411 }
1412
1413 void MediaTrackGraphImpl::Process(AudioMixer* aMixer) {
1414   TRACE("MTG::Process");
1415   MOZ_ASSERT(OnGraphThread());
1416   // Play track contents.
1417   bool allBlockedForever = true;
1418   // True when we've done ProcessInput for all processed tracks.
1419   bool doneAllProducing = false;
1420   const GraphTime oldProcessedTime = mProcessedTime;
1421
1422   // Figure out what each track wants to do
1423   for (uint32_t i = 0; i < mTracks.Length(); ++i) {
1424     MediaTrack* track = mTracks[i];
1425     if (!doneAllProducing) {
1426       ProcessedMediaTrack* pt = track->AsProcessedTrack();
1427       if (pt) {
1428         AudioNodeTrack* n = track->AsAudioNodeTrack();
1429         if (n) {
1430 #ifdef DEBUG
1431           // Verify that the sampling rate for all of the following tracks is
1432           // the same
1433           for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
1434             AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
1435             if (nextTrack) {
1436               MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
1437                          "All AudioNodeTracks in the graph must have the same "
1438                          "sampling rate");
1439             }
1440           }
1441 #endif
1442           // Since an AudioNodeTrack is present, go ahead and
1443           // produce audio block by block for all the rest of the tracks.
1444           ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
1445           doneAllProducing = true;
1446         } else {
1447           pt->ProcessInput(mProcessedTime, mStateComputedTime,
1448                            ProcessedMediaTrack::ALLOW_END);
1449           // Assert that a live track produced enough data
1450           MOZ_ASSERT_IF(!track->mEnded,
1451                         track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1452                                                track, mStateComputedTime));
1453         }
1454       }
1455     }
1456     if (track->mStartBlocking > oldProcessedTime) {
1457       allBlockedForever = false;
1458     }
1459   }
1460   mProcessedTime = mStateComputedTime;
1461
1462   if (aMixer) {
1463     MOZ_ASSERT(mRealtime, "If there's a mixer, this graph must be realtime");
1464     aMixer->StartMixing();
1465     // This is the number of frames that are written to the output buffer, for
1466     // this iteration.
1467     TrackTime ticksPlayed = 0;
1468     for (auto& t : mAudioOutputs) {
1469       TrackTime ticksPlayedForThisTrack =
1470           PlayAudio(aMixer, t, oldProcessedTime);
1471       if (ticksPlayed == 0) {
1472         ticksPlayed = ticksPlayedForThisTrack;
1473       } else {
1474         MOZ_ASSERT(
1475             !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
1476             "Each track should have the same number of frames.");
1477       }
1478     }
1479
1480     if (ticksPlayed == 0) {
1481       // Nothing was played, so the mixer doesn't know how many frames were
1482       // processed. We still tell it so AudioCallbackDriver knows how much has
1483       // been processed. (bug 1406027)
1484       aMixer->Mix(
1485           nullptr,
1486           CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(),
1487           mStateComputedTime - oldProcessedTime, mSampleRate);
1488     }
1489     aMixer->FinishMixing();
1490   }
1491
1492   if (!allBlockedForever) {
1493     EnsureNextIteration();
1494   }
1495 }
1496
1497 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1498   MOZ_ASSERT(OnGraphThread());
1499   if (mForceShutDownReceived) {
1500     for (MediaTrack* track : AllTracks()) {
1501       track->OnGraphThreadDone();
1502     }
1503   }
1504   {
1505     MonitorAutoLock lock(mMonitor);
1506     bool finalUpdate =
1507         mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
1508     PrepareUpdatesToMainThreadState(finalUpdate);
1509     if (!finalUpdate) {
1510       SwapMessageQueues();
1511       return true;
1512     }
1513     // The JSContext will not be used again.
1514     // Clear main thread access while under monitor.
1515     mJSContext = nullptr;
1516   }
1517   dom::WorkletThread::DeleteCycleCollectedJSContext();
1518   // Enter shutdown mode when this iteration is completed.
1519   // No need to Destroy tracks here. The main-thread owner of each
1520   // track is responsible for calling Destroy on them.
1521   return false;
1522 }
1523
1524 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime,
1525                                        GraphTime aIterationEnd,
1526                                        AudioMixer* aMixer) -> IterationResult {
1527   if (mGraphRunner) {
1528     return mGraphRunner->OneIteration(aStateTime, aIterationEnd, aMixer);
1529   }
1530
1531   return OneIterationImpl(aStateTime, aIterationEnd, aMixer);
1532 }
1533
1534 auto MediaTrackGraphImpl::OneIterationImpl(GraphTime aStateTime,
1535                                            GraphTime aIterationEnd,
1536                                            AudioMixer* aMixer)
1537     -> IterationResult {
1538   TRACE("MTG::OneIterationImpl");
1539
1540   mIterationEndTime = aIterationEnd;
1541
1542   if (SoftRealTimeLimitReached()) {
1543     TRACE("MTG::Demoting real-time thread!");
1544     DemoteThreadFromRealTime();
1545   }
1546
1547   // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1548   // thread, and so the monitor need not be held to check mLifecycleState.
1549   // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1550   // graphs that have not started.
1551
1552   // While changes occur on mainthread, this assert confirms that
1553   // this code shouldn't run if mainthread might be changing the state (to
1554   // > LIFECYCLE_RUNNING)
1555
1556   // Ignore mutex warning: static during execution of the graph
1557   MOZ_PUSH_IGNORE_THREAD_SAFETY
1558   MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
1559   MOZ_POP_THREAD_SAFETY
1560
1561   MOZ_ASSERT(OnGraphThread());
1562
1563   WebCore::DenormalDisabler disabler;
1564
1565   // Process graph message from the main thread for this iteration.
1566   RunMessagesInQueue();
1567
1568   // Process MessagePort events.
1569   // These require a single thread, which has an nsThread with an event queue.
1570   if (mGraphRunner || !mRealtime) {
1571     TRACE("MTG::MessagePort events");
1572     NS_ProcessPendingEvents(nullptr);
1573   }
1574
1575   GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
1576   UpdateGraph(stateTime);
1577
1578   mStateComputedTime = stateTime;
1579
1580   GraphTime oldProcessedTime = mProcessedTime;
1581   Process(aMixer);
1582   MOZ_ASSERT(mProcessedTime == stateTime);
1583
1584   UpdateCurrentTimeForTracks(oldProcessedTime);
1585
1586   ProcessChunkMetadata(oldProcessedTime);
1587
1588   // Process graph messages queued from RunMessageAfterProcessing() on this
1589   // thread during the iteration.
1590   RunMessagesInQueue();
1591
1592   if (!UpdateMainThreadState()) {
1593     if (Switching()) {
1594       // We'll never get to do this switch. Clear mNextDriver to break the
1595       // ref-cycle graph->nextDriver->currentDriver->graph.
1596       SwitchAtNextIteration(nullptr);
1597     }
1598     return IterationResult::CreateStop(
1599         NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1600                           &MediaTrackGraphImpl::SignalMainThreadCleanup));
1601   }
1602
1603   if (Switching()) {
1604     RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
1605     return IterationResult::CreateSwitchDriver(
1606         nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
1607                         "MediaTrackGraphImpl::SetCurrentDriver", this,
1608                         &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
1609   }
1610
1611   return IterationResult::CreateStillProcessing();
1612 }
1613
1614 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
1615   MOZ_ASSERT(NS_IsMainThread());
1616   mMonitor.AssertCurrentThreadOwns();
1617
1618   MediaTrack* track = aUpdate->mTrack;
1619   if (!track) return;
1620   track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1621   track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
1622
1623   if (track->ShouldNotifyTrackEnded()) {
1624     track->NotifyMainThreadListeners();
1625   }
1626 }
1627
1628 void MediaTrackGraphImpl::ForceShutDown() {
1629   MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1630   LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
1631
1632   if (mShutdownBlocker) {
1633     // Avoid waiting forever for a graph to shut down
1634     // synchronously.  Reports are that some 3rd-party audio drivers
1635     // occasionally hang in shutdown (both for us and Chrome).
1636     NS_NewTimerWithCallback(
1637         getter_AddRefs(mShutdownTimer), this,
1638         MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1639         nsITimer::TYPE_ONE_SHOT);
1640   }
1641
1642   class Message final : public ControlMessage {
1643    public:
1644     explicit Message(MediaTrackGraphImpl* aGraph)
1645         : ControlMessage(nullptr), mGraph(aGraph) {}
1646     void Run() override {
1647       TRACE("MTG::ForceShutdown ControlMessage");
1648       mGraph->mForceShutDownReceived = true;
1649     }
1650     // The graph owns this message.
1651     MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
1652   };
1653
1654   if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
1655     // If both the track and port counts are zero, the regular shutdown
1656     // sequence will progress shortly to shutdown threads and destroy the graph.
1657     AppendMessage(MakeUnique<Message>(this));
1658     InterruptJS();
1659   }
1660 }
1661
1662 NS_IMETHODIMP
1663 MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
1664   MOZ_ASSERT(NS_IsMainThread());
1665   NS_ASSERTION(!mShutdownBlocker,
1666                "MediaTrackGraph took too long to shut down!");
1667   // Sigh, graph took too long to shut down.  Stop blocking system
1668   // shutdown and hope all is well.
1669   RemoveShutdownBlocker();
1670   return NS_OK;
1671 }
1672
1673 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1674   MOZ_ASSERT(NS_IsMainThread());
1675   MOZ_ASSERT(!mShutdownBlocker);
1676
1677   class Blocker : public media::ShutdownBlocker {
1678     const RefPtr<MediaTrackGraphImpl> mGraph;
1679
1680    public:
1681     Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
1682         : media::ShutdownBlocker(aName), mGraph(aGraph) {}
1683
1684     NS_IMETHOD
1685     BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
1686       mGraph->ForceShutDown();
1687       return NS_OK;
1688     }
1689   };
1690
1691   nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
1692   if (!barrier) {
1693     // We're already shutting down, we won't be able to add a blocker, bail.
1694     LOG(LogLevel::Error,
1695         ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1696          this));
1697     return false;
1698   }
1699
1700   // Blocker names must be distinct.
1701   nsString blockerName;
1702   blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
1703   mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
1704   nsresult rv = barrier->AddBlocker(mShutdownBlocker,
1705                                     NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
1706                                     __LINE__, u"MediaTrackGraph shutdown"_ns);
1707   MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
1708   return true;
1709 }
1710
1711 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1712   if (!mShutdownBlocker) {
1713     return;
1714   }
1715   media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
1716   mShutdownBlocker = nullptr;
1717 }
1718
1719 NS_IMETHODIMP
1720 MediaTrackGraphImpl::GetName(nsACString& aName) {
1721   aName.AssignLiteral("MediaTrackGraphImpl");
1722   return NS_OK;
1723 }
1724
1725 namespace {
1726
1727 class MediaTrackGraphShutDownRunnable : public Runnable {
1728  public:
1729   explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
1730       : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
1731   // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1732   // See bug 1535398.
1733   MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
1734     TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1735     MOZ_ASSERT(NS_IsMainThread());
1736     MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
1737                "We should know the graph thread control loop isn't running!");
1738
1739     LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
1740
1741     // We've asserted the graph isn't running.  Use mDriver instead of
1742     // CurrentDriver to avoid thread-safety checks
1743 #if 0  // AudioCallbackDrivers are released asynchronously anyways
1744     // XXX a better test would be have setting mGraphDriverRunning make sure
1745     // any current callback has finished and block future ones -- or just
1746     // handle it all in Shutdown()!
1747     if (mGraph->mDriver->AsAudioCallbackDriver()) {
1748       MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1749     }
1750 #endif
1751
1752     for (MediaTrackGraphImpl::PendingResumeOperation& op :
1753          mGraph->mPendingResumeOperations) {
1754       op.Abort();
1755     }
1756
1757     if (mGraph->mGraphRunner) {
1758       RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
1759     }
1760
1761     RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
1762
1763     // Release the driver now so that an AudioCallbackDriver will release its
1764     // SharedThreadPool reference.  Each SharedThreadPool reference must be
1765     // released before SharedThreadPool::SpinUntilEmpty() runs on
1766     // xpcom-shutdown-threads.  Don't wait for GC/CC to release references to
1767     // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1768     // which won't otherwise release its reference on the graph until
1769     // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1770     mGraph->SetCurrentDriver(nullptr);
1771
1772     // Safe to access these without the monitor since the graph isn't running.
1773     // We may be one of several graphs. Drop ticket to eventually unblock
1774     // shutdown.
1775     if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
1776       MOZ_ASSERT(
1777           false,
1778           "AudioCallbackDriver took too long to shut down and we let shutdown"
1779           " continue - freezing and leaking");
1780
1781       // The timer fired, so we may be deeper in shutdown now.  Block any
1782       // further teardown and just leak, for safety.
1783       return NS_OK;
1784     }
1785
1786     // mGraph's thread is not running so it's OK to do whatever here
1787     for (MediaTrack* track : mGraph->AllTracks()) {
1788       // Clean up all MediaSegments since we cannot release Images too
1789       // late during shutdown. Also notify listeners that they were removed
1790       // so they can clean up any gfx resources.
1791       track->RemoveAllResourcesAndListenersImpl();
1792     }
1793
1794 #ifdef DEBUG
1795     {
1796       MonitorAutoLock lock(mGraph->mMonitor);
1797       MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
1798     }
1799 #endif
1800     mGraph->mPendingUpdateRunnables.Clear();
1801
1802     mGraph->RemoveShutdownBlocker();
1803
1804     // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1805     // stage, since completion of that stage requires all tracks to be freed,
1806     // which requires shutdown to proceed.
1807
1808     if (mGraph->IsEmpty()) {
1809       // mGraph is no longer needed, so delete it.
1810       mGraph->Destroy();
1811     } else {
1812       // The graph is not empty.  We must be in a forced shutdown.
1813       // Some later AppendMessage will detect that the graph has
1814       // been emptied, and delete it.
1815       NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
1816       mGraph->LifecycleStateRef() =
1817           MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
1818     }
1819     return NS_OK;
1820   }
1821
1822  private:
1823   RefPtr<MediaTrackGraphImpl> mGraph;
1824 };
1825
1826 class MediaTrackGraphStableStateRunnable : public Runnable {
1827  public:
1828   explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
1829                                               bool aSourceIsMTG)
1830       : Runnable("MediaTrackGraphStableStateRunnable"),
1831         mGraph(aGraph),
1832         mSourceIsMTG(aSourceIsMTG) {}
1833   NS_IMETHOD Run() override {
1834     TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1835     if (mGraph) {
1836       mGraph->RunInStableState(mSourceIsMTG);
1837     }
1838     return NS_OK;
1839   }
1840
1841  private:
1842   RefPtr<MediaTrackGraphImpl> mGraph;
1843   bool mSourceIsMTG;
1844 };
1845
1846 /*
1847  * Control messages forwarded from main thread to graph manager thread
1848  */
1849 class CreateMessage : public ControlMessage {
1850  public:
1851   explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
1852   void Run() override {
1853     TRACE("MTG::AddTrackGraphThread ControlMessage");
1854     mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
1855   }
1856   void RunDuringShutdown() override {
1857     // Make sure to run this message during shutdown too, to make sure
1858     // that we balance the number of tracks registered with the graph
1859     // as they're destroyed during shutdown.
1860     Run();
1861   }
1862 };
1863
1864 }  // namespace
1865
1866 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
1867   MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1868
1869   nsTArray<nsCOMPtr<nsIRunnable>> runnables;
1870   // When we're doing a forced shutdown, pending control messages may be
1871   // run on the main thread via RunDuringShutdown. Those messages must
1872   // run without the graph monitor being held. So, we collect them here.
1873   nsTArray<UniquePtr<ControlMessage>> controlMessagesToRunDuringShutdown;
1874
1875   {
1876     MonitorAutoLock lock(mMonitor);
1877     if (aSourceIsMTG) {
1878       MOZ_ASSERT(mPostedRunInStableStateEvent);
1879       mPostedRunInStableStateEvent = false;
1880     }
1881
1882     // This should be kept in sync with the LifecycleState enum in
1883     // MediaTrackGraphImpl.h
1884     const char* LifecycleState_str[] = {
1885         "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
1886         "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1887         "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1888         "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
1889
1890     if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
1891       LOG(LogLevel::Debug,
1892           ("%p: Running stable state callback. Current state: %s", this,
1893            LifecycleState_str[LifecycleStateRef()]));
1894     }
1895
1896     runnables = std::move(mUpdateRunnables);
1897     for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
1898       TrackUpdate* update = &mTrackUpdates[i];
1899       if (update->mTrack) {
1900         ApplyTrackUpdate(update);
1901       }
1902     }
1903     mTrackUpdates.Clear();
1904
1905     mMainThreadGraphTime = mNextMainThreadGraphTime;
1906
1907     if (mCurrentTaskMessageQueue.IsEmpty()) {
1908       if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1909           IsEmpty()) {
1910         // Complete shutdown. First, ensure that this graph is no longer used.
1911         // A new graph graph will be created if one is needed.
1912         // Asynchronously clean up old graph. We don't want to do this
1913         // synchronously because it spins the event loop waiting for threads
1914         // to shut down, and we don't want to do that in a stable state handler.
1915         LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1916         LOG(LogLevel::Debug,
1917             ("%p: Sending MediaTrackGraphShutDownRunnable", this));
1918         nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1919         mMainThread->Dispatch(event.forget());
1920       }
1921     } else {
1922       if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1923         MessageBlock* block = mBackMessageQueue.AppendElement();
1924         block->mMessages = std::move(mCurrentTaskMessageQueue);
1925         EnsureNextIteration();
1926       }
1927
1928       // If this MediaTrackGraph has entered regular (non-forced) shutdown it
1929       // is not able to process any more messages. Those messages being added to
1930       // the graph in the first place is an error.
1931       MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
1932                                 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
1933                             mForceShutDownReceived);
1934     }
1935
1936     if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
1937       // Start the driver now. We couldn't start it earlier because the graph
1938       // might exit immediately on finding it has no tracks. The first message
1939       // for a new graph must create a track. Ensure that his message runs on
1940       // the first iteration.
1941       MOZ_ASSERT(MessagesQueued());
1942       SwapMessageQueues();
1943
1944       LOG(LogLevel::Debug,
1945           ("%p: Starting a graph with a %s", this,
1946            CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
1947                                                     : "SystemClockDriver"));
1948       LifecycleStateRef() = LIFECYCLE_RUNNING;
1949       mGraphDriverRunning = true;
1950       RefPtr<GraphDriver> driver = CurrentDriver();
1951       driver->Start();
1952       // It's not safe to Shutdown() a thread from StableState, and
1953       // releasing this may shutdown a SystemClockDriver thread.
1954       // Proxy the release to outside of StableState.
1955       NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
1956                              driver.forget(),
1957                              true);  // always proxy
1958     }
1959
1960     if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1961         mForceShutDownReceived) {
1962       // Defer calls to RunDuringShutdown() to happen while mMonitor is not
1963       // held.
1964       for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
1965         MessageBlock& mb = mBackMessageQueue[i];
1966         controlMessagesToRunDuringShutdown.AppendElements(
1967             std::move(mb.mMessages));
1968       }
1969       mBackMessageQueue.Clear();
1970       MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
1971       // Stop MediaTrackGraph threads.
1972       LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1973       nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1974       mMainThread->Dispatch(event.forget());
1975     }
1976
1977     mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
1978   }
1979
1980   // Make sure we get a new current time in the next event loop task
1981   if (!aSourceIsMTG) {
1982     MOZ_ASSERT(mPostedRunInStableState);
1983     mPostedRunInStableState = false;
1984   }
1985
1986   for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
1987     controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
1988   }
1989
1990 #ifdef DEBUG
1991   mCanRunMessagesSynchronously =
1992       !mGraphDriverRunning &&
1993       LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1994 #endif
1995
1996   for (uint32_t i = 0; i < runnables.Length(); ++i) {
1997     runnables[i]->Run();
1998   }
1999 }
2000
2001 void MediaTrackGraphImpl::EnsureRunInStableState() {
2002   MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2003
2004   if (mPostedRunInStableState) return;
2005   mPostedRunInStableState = true;
2006   nsCOMPtr<nsIRunnable> event =
2007       new MediaTrackGraphStableStateRunnable(this, false);
2008   nsContentUtils::RunInStableState(event.forget());
2009 }
2010
2011 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
2012   MOZ_ASSERT(OnGraphThread());
2013   mMonitor.AssertCurrentThreadOwns();
2014
2015   if (mPostedRunInStableStateEvent) return;
2016   mPostedRunInStableStateEvent = true;
2017   nsCOMPtr<nsIRunnable> event =
2018       new MediaTrackGraphStableStateRunnable(this, true);
2019   mMainThread->Dispatch(event.forget());
2020 }
2021
2022 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2023   MOZ_ASSERT(mDriver->OnThread());
2024
2025   MonitorAutoLock lock(mMonitor);
2026   // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2027   // graphs that have not started.
2028   MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
2029   LOG(LogLevel::Debug,
2030       ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2031   LifecycleStateRef() =
2032       MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
2033   EnsureStableStateEventPosted();
2034 }
2035
2036 void MediaTrackGraphImpl::AppendMessage(UniquePtr<ControlMessage> aMessage) {
2037   MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2038   MOZ_RELEASE_ASSERT(!aMessage->GetTrack() ||
2039                      !aMessage->GetTrack()->IsDestroyed());
2040   MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
2041
2042   if (!mGraphDriverRunning &&
2043       LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2044     // The graph control loop is not running and main thread cleanup has
2045     // happened. From now on we can't append messages to
2046     // mCurrentTaskMessageQueue, because that will never be processed again, so
2047     // just RunDuringShutdown this message. This should only happen during
2048     // forced shutdown, or after a non-realtime graph has finished processing.
2049 #ifdef DEBUG
2050     MOZ_ASSERT(mCanRunMessagesSynchronously);
2051     mCanRunMessagesSynchronously = false;
2052 #endif
2053     aMessage->RunDuringShutdown();
2054 #ifdef DEBUG
2055     mCanRunMessagesSynchronously = true;
2056 #endif
2057     if (IsEmpty() &&
2058         LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
2059       Destroy();
2060     }
2061     return;
2062   }
2063
2064   mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
2065   EnsureRunInStableState();
2066 }
2067
2068 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
2069   mMainThread->Dispatch(std::move(aRunnable));
2070 }
2071
2072 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
2073                        MediaSegment* aSegment)
2074     : mSampleRate(aSampleRate),
2075       mType(aType),
2076       mSegment(aSegment),
2077       mStartTime(0),
2078       mForgottenTime(0),
2079       mEnded(false),
2080       mNotifiedEnded(false),
2081       mDisabledMode(DisabledTrackMode::ENABLED),
2082       mStartBlocking(GRAPH_TIME_MAX),
2083       mSuspendedCount(0),
2084       mMainThreadCurrentTime(0),
2085       mMainThreadEnded(false),
2086       mEndedNotificationSent(false),
2087       mMainThreadDestroyed(false),
2088       mGraph(nullptr) {
2089   MOZ_COUNT_CTOR(MediaTrack);
2090   MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
2091 }
2092
2093 MediaTrack::~MediaTrack() {
2094   MOZ_COUNT_DTOR(MediaTrack);
2095   NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
2096   NS_ASSERTION(mMainThreadListeners.IsEmpty(),
2097                "All main thread listeners should have been removed");
2098 }
2099
2100 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
2101   size_t amount = 0;
2102
2103   // Not owned:
2104   // - mGraph - Not reported here
2105   // - mConsumers - elements
2106   // Future:
2107   // - mLastPlayedVideoFrame
2108   // - mTrackListeners - elements
2109
2110   amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2111   amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2112   amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
2113
2114   return amount;
2115 }
2116
2117 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
2118   return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
2119 }
2120
2121 void MediaTrack::IncrementSuspendCount() {
2122   ++mSuspendedCount;
2123   if (mSuspendedCount != 1 || !mGraph) {
2124     MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2125     return;
2126   }
2127   MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning());
2128   for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2129     mConsumers[i]->Suspended();
2130   }
2131   MOZ_ASSERT(mGraph->mTracks.Contains(this));
2132   mGraph->mTracks.RemoveElement(this);
2133   mGraph->mSuspendedTracks.AppendElement(this);
2134   mGraph->SetTrackOrderDirty();
2135 }
2136
2137 void MediaTrack::DecrementSuspendCount() {
2138   MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
2139   --mSuspendedCount;
2140   if (mSuspendedCount != 0 || !mGraph) {
2141     MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2142     return;
2143   }
2144   MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning());
2145   for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2146     mConsumers[i]->Resumed();
2147   }
2148   MOZ_ASSERT(mGraph->mSuspendedTracks.Contains(this));
2149   mGraph->mSuspendedTracks.RemoveElement(this);
2150   mGraph->mTracks.AppendElement(this);
2151   mGraph->SetTrackOrderDirty();
2152 }
2153
2154 void ProcessedMediaTrack::DecrementSuspendCount() {
2155   mCycleMarker = NOT_VISITED;
2156   MediaTrack::DecrementSuspendCount();
2157 }
2158
2159 MediaTrackGraphImpl* MediaTrack::GraphImpl() { return mGraph; }
2160
2161 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const { return mGraph; }
2162
2163 MediaTrackGraph* MediaTrack::Graph() { return mGraph; }
2164
2165 const MediaTrackGraph* MediaTrack::Graph() const { return mGraph; }
2166
2167 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
2168   MOZ_ASSERT(!mGraph, "Should only be called once");
2169   MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
2170   mGraph = aGraph;
2171 }
2172
2173 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
2174   MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
2175   SetGraphImpl(graph);
2176 }
2177
2178 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
2179   NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2180                    aTime <= mStartBlocking,
2181                "Incorrectly ignoring blocking!");
2182   return aTime - mStartTime;
2183 }
2184
2185 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
2186   NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2187                    aTime + mStartTime <= mStartBlocking,
2188                "Incorrectly ignoring blocking!");
2189   return aTime + mStartTime;
2190 }
2191
2192 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
2193   return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
2194 }
2195
2196 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2197   GraphImpl()->AssertOnGraphThreadOrNotRunning();
2198
2199   for (auto& l : mTrackListeners.Clone()) {
2200     l->NotifyRemoved(Graph());
2201   }
2202   mTrackListeners.Clear();
2203
2204   RemoveAllDirectListenersImpl();
2205
2206   if (mSegment) {
2207     mSegment->Clear();
2208   }
2209 }
2210
2211 void MediaTrack::DestroyImpl() {
2212   for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2213     mConsumers[i]->Disconnect();
2214   }
2215   if (mSegment) {
2216     mSegment->Clear();
2217   }
2218   mGraph = nullptr;
2219 }
2220
2221 void MediaTrack::Destroy() {
2222   // Keep this track alive until we leave this method
2223   RefPtr<MediaTrack> kungFuDeathGrip = this;
2224
2225   class Message : public ControlMessage {
2226    public:
2227     explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2228     void RunDuringShutdown() override {
2229       TRACE("MediaTrack::Destroy ControlMessage");
2230       mTrack->RemoveAllResourcesAndListenersImpl();
2231       auto graph = mTrack->GraphImpl();
2232       mTrack->DestroyImpl();
2233       graph->RemoveTrackGraphThread(mTrack);
2234     }
2235     void Run() override {
2236       mTrack->OnGraphThreadDone();
2237       RunDuringShutdown();
2238     }
2239   };
2240   // Keep a reference to the graph, since Message might RunDuringShutdown()
2241   // synchronously and make GraphImpl() invalid.
2242   RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
2243   graph->AppendMessage(MakeUnique<Message>(this));
2244   graph->RemoveTrack(this);
2245   // Message::RunDuringShutdown may have removed this track from the graph,
2246   // but our kungFuDeathGrip above will have kept this track alive if
2247   // necessary.
2248   mMainThreadDestroyed = true;
2249 }
2250
2251 TrackTime MediaTrack::GetEnd() const {
2252   return mSegment ? mSegment->GetDuration() : 0;
2253 }
2254
2255 void MediaTrack::AddAudioOutput(void* aKey) {
2256   class Message : public ControlMessage {
2257    public:
2258     Message(MediaTrack* aTrack, void* aKey)
2259         : ControlMessage(aTrack), mKey(aKey) {}
2260     void Run() override {
2261       TRACE("MediaTrack::AddAudioOutputImpl ControlMessage");
2262       mTrack->AddAudioOutputImpl(mKey);
2263     }
2264     void* mKey;
2265   };
2266   if (mMainThreadDestroyed) {
2267     return;
2268   }
2269   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2270 }
2271
2272 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
2273                                                float aVolume) {
2274   for (auto& tkv : mAudioOutputs) {
2275     if (tkv.mKey == aKey && aTrack == tkv.mTrack) {
2276       tkv.mVolume = aVolume;
2277       return;
2278     }
2279   }
2280   MOZ_CRASH("Audio stream key not found when setting the volume.");
2281 }
2282
2283 void MediaTrack::SetAudioOutputVolumeImpl(void* aKey, float aVolume) {
2284   MOZ_ASSERT(GraphImpl()->OnGraphThread());
2285   GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
2286 }
2287
2288 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
2289   class Message : public ControlMessage {
2290    public:
2291     Message(MediaTrack* aTrack, void* aKey, float aVolume)
2292         : ControlMessage(aTrack), mKey(aKey), mVolume(aVolume) {}
2293     void Run() override {
2294       TRACE("MediaTrack::SetAudioOutputVolumeImpl ControlMessage");
2295       mTrack->SetAudioOutputVolumeImpl(mKey, mVolume);
2296     }
2297     void* mKey;
2298     float mVolume;
2299   };
2300   if (mMainThreadDestroyed) {
2301     return;
2302   }
2303   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey, aVolume));
2304 }
2305
2306 void MediaTrack::AddAudioOutputImpl(void* aKey) {
2307   LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
2308   GraphImpl()->RegisterAudioOutput(this, aKey);
2309 }
2310
2311 void MediaTrack::RemoveAudioOutputImpl(void* aKey) {
2312   LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
2313   GraphImpl()->UnregisterAudioOutput(this, aKey);
2314 }
2315
2316 void MediaTrack::RemoveAudioOutput(void* aKey) {
2317   class Message : public ControlMessage {
2318    public:
2319     explicit Message(MediaTrack* aTrack, void* aKey)
2320         : ControlMessage(aTrack), mKey(aKey) {}
2321     void Run() override {
2322       TRACE("MediaTrack::RemoveAudioOutputImpl ControlMessage");
2323       mTrack->RemoveAudioOutputImpl(mKey);
2324     }
2325     void* mKey;
2326   };
2327   if (mMainThreadDestroyed) {
2328     return;
2329   }
2330   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2331 }
2332
2333 void MediaTrack::Suspend() {
2334   class Message : public ControlMessage {
2335    public:
2336     explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2337     void Run() override {
2338       TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2339       mTrack->IncrementSuspendCount();
2340     }
2341   };
2342
2343   // This can happen if this method has been called asynchronously, and the
2344   // track has been destroyed since then.
2345   if (mMainThreadDestroyed) {
2346     return;
2347   }
2348   GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2349 }
2350
2351 void MediaTrack::Resume() {
2352   class Message : public ControlMessage {
2353    public:
2354     explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2355     void Run() override {
2356       TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2357       mTrack->DecrementSuspendCount();
2358     }
2359   };
2360
2361   // This can happen if this method has been called asynchronously, and the
2362   // track has been destroyed since then.
2363   if (mMainThreadDestroyed) {
2364     return;
2365   }
2366   GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2367 }
2368
2369 void MediaTrack::AddListenerImpl(
2370     already_AddRefed<MediaTrackListener> aListener) {
2371   RefPtr<MediaTrackListener> l(aListener);
2372   mTrackListeners.AppendElement(std::move(l));
2373
2374   PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
2375   mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
2376       Graph(), lastPrincipalHandle);
2377   if (mNotifiedEnded) {
2378     mTrackListeners.LastElement()->NotifyEnded(Graph());
2379   }
2380   if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
2381     mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2382   }
2383 }
2384
2385 void MediaTrack::AddListener(MediaTrackListener* aListener) {
2386   class Message : public ControlMessage {
2387    public:
2388     Message(MediaTrack* aTrack, MediaTrackListener* aListener)
2389         : ControlMessage(aTrack), mListener(aListener) {}
2390     void Run() override {
2391       TRACE("MediaTrack::AddListenerImpl ControlMessage");
2392       mTrack->AddListenerImpl(mListener.forget());
2393     }
2394     RefPtr<MediaTrackListener> mListener;
2395   };
2396   MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
2397   if (mMainThreadDestroyed) {
2398     return;
2399   }
2400   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2401 }
2402
2403 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
2404   for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2405     if (mTrackListeners[i] == aListener) {
2406       mTrackListeners[i]->NotifyRemoved(Graph());
2407       mTrackListeners.RemoveElementAt(i);
2408       return;
2409     }
2410   }
2411 }
2412
2413 RefPtr<GenericPromise> MediaTrack::RemoveListener(
2414     MediaTrackListener* aListener) {
2415   class Message : public ControlMessage {
2416    public:
2417     Message(MediaTrack* aTrack, MediaTrackListener* aListener)
2418         : ControlMessage(aTrack), mListener(aListener) {}
2419     void Run() override {
2420       TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2421       mTrack->RemoveListenerImpl(mListener);
2422       mRemovedPromise.Resolve(true, __func__);
2423     }
2424     void RunDuringShutdown() override {
2425       // During shutdown we still want the listener's NotifyRemoved to be
2426       // called, since not doing that might block shutdown of other modules.
2427       Run();
2428     }
2429     RefPtr<MediaTrackListener> mListener;
2430     MozPromiseHolder<GenericPromise> mRemovedPromise;
2431   };
2432
2433   UniquePtr<Message> message = MakeUnique<Message>(this, aListener);
2434   RefPtr<GenericPromise> p = message->mRemovedPromise.Ensure(__func__);
2435   if (mMainThreadDestroyed) {
2436     message->mRemovedPromise.Reject(NS_ERROR_FAILURE, __func__);
2437     return p;
2438   }
2439   GraphImpl()->AppendMessage(std::move(message));
2440   return p;
2441 }
2442
2443 void MediaTrack::AddDirectListenerImpl(
2444     already_AddRefed<DirectMediaTrackListener> aListener) {
2445   MOZ_ASSERT(mGraph->OnGraphThread());
2446   // Base implementation, for tracks that don't support direct track listeners.
2447   RefPtr<DirectMediaTrackListener> listener = aListener;
2448   listener->NotifyDirectListenerInstalled(
2449       DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
2450 }
2451
2452 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
2453   class Message : public ControlMessage {
2454    public:
2455     Message(MediaTrack* aTrack, DirectMediaTrackListener* aListener)
2456         : ControlMessage(aTrack), mListener(aListener) {}
2457     void Run() override {
2458       TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2459       mTrack->AddDirectListenerImpl(mListener.forget());
2460     }
2461     RefPtr<DirectMediaTrackListener> mListener;
2462   };
2463   if (mMainThreadDestroyed) {
2464     return;
2465   }
2466   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2467 }
2468
2469 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
2470   // Base implementation, the listener was never added so nothing to do.
2471 }
2472
2473 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
2474   class Message : public ControlMessage {
2475    public:
2476     Message(MediaTrack* aTrack, DirectMediaTrackListener* aListener)
2477         : ControlMessage(aTrack), mListener(aListener) {}
2478     void Run() override {
2479       TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2480       mTrack->RemoveDirectListenerImpl(mListener);
2481     }
2482     void RunDuringShutdown() override {
2483       // During shutdown we still want the listener's
2484       // NotifyDirectListenerUninstalled to be called, since not doing that
2485       // might block shutdown of other modules.
2486       Run();
2487     }
2488     RefPtr<DirectMediaTrackListener> mListener;
2489   };
2490   if (mMainThreadDestroyed) {
2491     return;
2492   }
2493   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2494 }
2495
2496 void MediaTrack::RunAfterPendingUpdates(
2497     already_AddRefed<nsIRunnable> aRunnable) {
2498   MOZ_ASSERT(NS_IsMainThread());
2499   MediaTrackGraphImpl* graph = GraphImpl();
2500   nsCOMPtr<nsIRunnable> runnable(aRunnable);
2501
2502   class Message : public ControlMessage {
2503    public:
2504     Message(MediaTrack* aTrack, already_AddRefed<nsIRunnable> aRunnable)
2505         : ControlMessage(aTrack), mRunnable(aRunnable) {}
2506     void Run() override {
2507       TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2508       mTrack->Graph()->DispatchToMainThreadStableState(mRunnable.forget());
2509     }
2510     void RunDuringShutdown() override {
2511       // Don't run mRunnable now as it may call AppendMessage() which would
2512       // assume that there are no remaining controlMessagesToRunDuringShutdown.
2513       MOZ_ASSERT(NS_IsMainThread());
2514       mTrack->GraphImpl()->Dispatch(mRunnable.forget());
2515     }
2516
2517    private:
2518     nsCOMPtr<nsIRunnable> mRunnable;
2519   };
2520
2521   if (mMainThreadDestroyed) {
2522     return;
2523   }
2524   graph->AppendMessage(MakeUnique<Message>(this, runnable.forget()));
2525 }
2526
2527 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2528   MOZ_ASSERT(mGraph->OnGraphThread());
2529   MOZ_DIAGNOSTIC_ASSERT(
2530       aMode == DisabledTrackMode::ENABLED ||
2531           mDisabledMode == DisabledTrackMode::ENABLED,
2532       "Changing disabled track mode for a track is not allowed");
2533   DisabledTrackMode oldMode = CombinedDisabledMode();
2534   mDisabledMode = aMode;
2535   NotifyIfDisabledModeChangedFrom(oldMode);
2536 }
2537
2538 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
2539   class Message : public ControlMessage {
2540    public:
2541     Message(MediaTrack* aTrack, DisabledTrackMode aMode)
2542         : ControlMessage(aTrack), mMode(aMode) {}
2543     void Run() override {
2544       TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2545       mTrack->SetDisabledTrackModeImpl(mMode);
2546     }
2547     DisabledTrackMode mMode;
2548   };
2549   if (mMainThreadDestroyed) {
2550     return;
2551   }
2552   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aMode));
2553 }
2554
2555 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
2556                                      MediaSegment* aRawSegment) {
2557   MOZ_ASSERT(mGraph->OnGraphThread());
2558   mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment);
2559 }
2560
2561 void MediaTrack::AddMainThreadListener(
2562     MainThreadMediaTrackListener* aListener) {
2563   MOZ_ASSERT(NS_IsMainThread());
2564   MOZ_ASSERT(aListener);
2565   MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2566
2567   mMainThreadListeners.AppendElement(aListener);
2568
2569   // If it is not yet time to send the notification, then exit here.
2570   if (!mEndedNotificationSent) {
2571     return;
2572   }
2573
2574   class NotifyRunnable final : public Runnable {
2575    public:
2576     explicit NotifyRunnable(MediaTrack* aTrack)
2577         : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
2578
2579     NS_IMETHOD Run() override {
2580       TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2581       MOZ_ASSERT(NS_IsMainThread());
2582       mTrack->NotifyMainThreadListeners();
2583       return NS_OK;
2584     }
2585
2586    private:
2587     ~NotifyRunnable() = default;
2588
2589     RefPtr<MediaTrack> mTrack;
2590   };
2591
2592   nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2593   GraphImpl()->Dispatch(runnable.forget());
2594 }
2595
2596 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2597                                                        GraphTime aBlockedTime) {
2598   mStartTime += aBlockedTime;
2599
2600   if (!mSegment) {
2601     // No data to be forgotten.
2602     return;
2603   }
2604
2605   TrackTime time = aCurrentTime - mStartTime;
2606   // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2607   // spend too much time pruning segments.
2608   const TrackTime minChunkSize = mSampleRate * 50 / 1000;
2609   if (time < mForgottenTime + minChunkSize) {
2610     return;
2611   }
2612
2613   mForgottenTime = std::min(GetEnd() - 1, time);
2614   mSegment->ForgetUpTo(mForgottenTime);
2615 }
2616
2617 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
2618   DisabledTrackMode mode = CombinedDisabledMode();
2619   if (aOldMode == mode) {
2620     return;
2621   }
2622
2623   for (const auto& listener : mTrackListeners) {
2624     listener->NotifyEnabledStateChanged(
2625         Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
2626   }
2627
2628   for (const auto& c : mConsumers) {
2629     if (c->GetDestination()) {
2630       c->GetDestination()->OnInputDisabledModeChanged(mode);
2631     }
2632   }
2633 }
2634
2635 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
2636                                    TrackRate aSampleRate)
2637     : MediaTrack(aSampleRate, aType,
2638                  aType == MediaSegment::AUDIO
2639                      ? static_cast<MediaSegment*>(new AudioSegment())
2640                      : static_cast<MediaSegment*>(new VideoSegment())),
2641       mMutex("mozilla::media::SourceMediaTrack") {
2642   mUpdateTrack = MakeUnique<TrackData>();
2643   mUpdateTrack->mInputRate = aSampleRate;
2644   mUpdateTrack->mResamplerChannelCount = 0;
2645   mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
2646   mUpdateTrack->mEnded = false;
2647   mUpdateTrack->mPullingEnabled = false;
2648   mUpdateTrack->mGraphThreadDone = false;
2649 }
2650
2651 void SourceMediaTrack::DestroyImpl() {
2652   GraphImpl()->AssertOnGraphThreadOrNotRunning();
2653   for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2654     // Disconnect before we come under mMutex's lock since it can call back
2655     // through RemoveDirectListenerImpl() and deadlock.
2656     mConsumers[i]->Disconnect();
2657   }
2658
2659   // Hold mMutex while mGraph is reset so that other threads holding mMutex
2660   // can null-check know that the graph will not destroyed.
2661   MutexAutoLock lock(mMutex);
2662   mUpdateTrack = nullptr;
2663   MediaTrack::DestroyImpl();
2664 }
2665
2666 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
2667   class Message : public ControlMessage {
2668    public:
2669     Message(SourceMediaTrack* aTrack, bool aEnabled)
2670         : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
2671     void Run() override {
2672       TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2673       MutexAutoLock lock(mTrack->mMutex);
2674       if (!mTrack->mUpdateTrack) {
2675         // We can't enable pulling for a track that has ended. We ignore
2676         // this if we're disabling pulling, since shutdown sequences are
2677         // complex. If there's truly an issue we'll have issues enabling anyway.
2678         MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
2679         return;
2680       }
2681       MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
2682                  "Pulling is not allowed for video");
2683       mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
2684     }
2685     SourceMediaTrack* mTrack;
2686     bool mEnabled;
2687   };
2688   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
2689 }
2690
2691 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
2692   TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2693   TrackTime t;
2694   TrackTime current;
2695   {
2696     if (mEnded) {
2697       return false;
2698     }
2699     MutexAutoLock lock(mMutex);
2700     if (mUpdateTrack->mEnded) {
2701       return false;
2702     }
2703     if (!mUpdateTrack->mPullingEnabled) {
2704       return false;
2705     }
2706     // Compute how much track time we'll need assuming we don't block
2707     // the track at all.
2708     t = GraphTimeToTrackTime(aDesiredUpToTime);
2709     current = GetEnd() + mUpdateTrack->mData->GetDuration();
2710   }
2711   if (t <= current) {
2712     return false;
2713   }
2714   LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2715                           GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
2716                           GraphImpl()->MediaTimeToSeconds(current)));
2717   for (auto& l : mTrackListeners) {
2718     l->NotifyPull(Graph(), current, t);
2719   }
2720   return true;
2721 }
2722
2723 /**
2724  * This moves chunks from aIn to aOut. For audio this is simple. For video
2725  * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2726  *
2727  * We also handle "resetters" from captured media elements. This type of source
2728  * pushes future frames into the track, and should it need to remove some, e.g.,
2729  * because of a seek or pause, it tells us by letting time go backwards. Without
2730  * this, tracks would be live for too long after a seek or pause.
2731  */
2732 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
2733                           MediaSegment* aOut, TrackTime aCurrentTime,
2734                           TrackTime aDesiredUpToTime)
2735     MOZ_REQUIRES(aTrack->GetMutex()) {
2736   MOZ_ASSERT(aIn->GetType() == aOut->GetType());
2737   MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
2738   MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
2739   if (aIn->GetType() == MediaSegment::AUDIO) {
2740     AudioSegment* in = static_cast<AudioSegment*>(aIn);
2741     AudioSegment* out = static_cast<AudioSegment*>(aOut);
2742     TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
2743     TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
2744
2745     out->AppendSlice(*in, 0, end);
2746     in->RemoveLeading(end);
2747
2748     aTrack->GetMutex().AssertCurrentThreadOwns();
2749     out->ApplyVolume(aTrack->GetVolumeLocked());
2750   } else {
2751     VideoSegment* in = static_cast<VideoSegment*>(aIn);
2752     VideoSegment* out = static_cast<VideoSegment*>(aOut);
2753     for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
2754       MOZ_ASSERT(!c->mTimeStamp.IsNull());
2755       VideoChunk* last = out->GetLastChunk();
2756       if (!last || last->mTimeStamp.IsNull()) {
2757         // This is the first frame, or the last frame pushed to `out` has been
2758         // all consumed. Just append and we deal with its duration later.
2759         out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2760                          c->mFrame.GetIntrinsicSize(),
2761                          c->mFrame.GetPrincipalHandle(),
2762                          c->mFrame.GetForceBlack(), c->mTimeStamp);
2763         if (c->GetDuration() > 0) {
2764           out->ExtendLastFrameBy(c->GetDuration());
2765         }
2766         continue;
2767       }
2768
2769       // We now know when this frame starts, aka when the last frame ends.
2770
2771       if (c->mTimeStamp < last->mTimeStamp) {
2772         // Time is going backwards. This is a resetting frame from
2773         // DecodedStream. Clear everything up to currentTime.
2774         out->Clear();
2775         out->AppendNullData(aCurrentTime);
2776       }
2777
2778       // Append the current frame (will have duration 0).
2779       out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2780                        c->mFrame.GetIntrinsicSize(),
2781                        c->mFrame.GetPrincipalHandle(),
2782                        c->mFrame.GetForceBlack(), c->mTimeStamp);
2783       if (c->GetDuration() > 0) {
2784         out->ExtendLastFrameBy(c->GetDuration());
2785       }
2786     }
2787     if (out->GetDuration() < aDesiredUpToTime) {
2788       out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
2789     }
2790     in->Clear();
2791     MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
2792   }
2793 }
2794
2795 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
2796                                            GraphTime aDesiredUpToTime) {
2797   MutexAutoLock lock(mMutex);
2798
2799   if (!mUpdateTrack) {
2800     MOZ_ASSERT(mEnded);
2801     return;
2802   }
2803
2804   TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
2805
2806   ApplyTrackDisabling(mUpdateTrack->mData.get());
2807
2808   if (!mUpdateTrack->mData->IsEmpty()) {
2809     for (const auto& l : mTrackListeners) {
2810       l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
2811     }
2812   }
2813   TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
2814   TrackTime endTime = trackDesiredUpToTime;
2815   if (mUpdateTrack->mEnded) {
2816     endTime = std::min(trackDesiredUpToTime,
2817                        GetEnd() + mUpdateTrack->mData->GetDuration());
2818   }
2819   LOG(LogLevel::Verbose,
2820       ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
2821        GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
2822   MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
2823                 trackCurrentTime, endTime);
2824   if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
2825     mEnded = true;
2826     mUpdateTrack = nullptr;
2827   }
2828 }
2829
2830 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
2831   mMutex.AssertCurrentThreadOwns();
2832   if (aSegment->GetType() != MediaSegment::AUDIO ||
2833       mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
2834     return;
2835   }
2836   AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
2837   segment->ResampleChunks(mUpdateTrack->mResampler,
2838                           &mUpdateTrack->mResamplerChannelCount,
2839                           mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
2840 }
2841
2842 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
2843     GraphTime aCurrentTime, GraphTime aBlockedTime) {
2844   MutexAutoLock lock(mMutex);
2845   MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
2846 }
2847
2848 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
2849   MutexAutoLock lock(mMutex);
2850   if (!mUpdateTrack) {
2851     return;
2852   }
2853   MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
2854   // Set the new input rate and reset the resampler.
2855   mUpdateTrack->mInputRate = aRate;
2856   mUpdateTrack->mResampler.own(nullptr);
2857   mUpdateTrack->mResamplerChannelCount = 0;
2858 }
2859
2860 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
2861                                        MediaSegment* aRawSegment) {
2862   MutexAutoLock lock(mMutex);
2863   MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
2864   TrackTime appended = 0;
2865   if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
2866     aSegment->Clear();
2867     return appended;
2868   }
2869
2870   // Data goes into mData, and on the next iteration of the MTG moves
2871   // into the track's segment after NotifyQueuedTrackChanges().  This adds
2872   // 0-10ms of delay before data gets to direct listeners.
2873   // Indirect listeners (via subsequent TrackUnion nodes) are synced to
2874   // playout time, and so can be delayed by buffering.
2875
2876   // Apply track disabling before notifying any consumers directly
2877   // or inserting into the graph
2878   mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment);
2879
2880   ResampleAudioToGraphSampleRate(aSegment);
2881
2882   // Must notify first, since AppendFrom() will empty out aSegment
2883   NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
2884   appended = aSegment->GetDuration();
2885   mUpdateTrack->mData->AppendFrom(aSegment);  // note: aSegment is now dead
2886   {
2887     auto graph = GraphImpl();
2888     MonitorAutoLock lock(graph->GetMonitor());
2889     if (graph->CurrentDriver()) {  // graph has not completed forced shutdown
2890       graph->EnsureNextIteration();
2891     }
2892   }
2893
2894   return appended;
2895 }
2896
2897 TrackTime SourceMediaTrack::ClearFutureData() {
2898   MutexAutoLock lock(mMutex);
2899   auto graph = GraphImpl();
2900   if (!mUpdateTrack || !graph) {
2901     return 0;
2902   }
2903
2904   TrackTime duration = mUpdateTrack->mData->GetDuration();
2905   mUpdateTrack->mData->Clear();
2906   return duration;
2907 }
2908
2909 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
2910   mMutex.AssertCurrentThreadOwns();
2911
2912   for (const auto& l : mDirectTrackListeners) {
2913     TrackTime offset = 0;  // FIX! need a separate TrackTime.... or the end of
2914                            // the internal buffer
2915     l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
2916                                                      *aSegment);
2917   }
2918 }
2919
2920 void SourceMediaTrack::AddDirectListenerImpl(
2921     already_AddRefed<DirectMediaTrackListener> aListener) {
2922   MOZ_ASSERT(mGraph->OnGraphThread());
2923   MutexAutoLock lock(mMutex);
2924
2925   RefPtr<DirectMediaTrackListener> listener = aListener;
2926   LOG(LogLevel::Debug,
2927       ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
2928        listener.get(), this));
2929
2930   MOZ_ASSERT(mType == MediaSegment::VIDEO);
2931   for (const auto& l : mDirectTrackListeners) {
2932     if (l == listener) {
2933       listener->NotifyDirectListenerInstalled(
2934           DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
2935       return;
2936     }
2937   }
2938
2939   mDirectTrackListeners.AppendElement(listener);
2940
2941   LOG(LogLevel::Debug,
2942       ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
2943   listener->NotifyDirectListenerInstalled(
2944       DirectMediaTrackListener::InstallationResult::SUCCESS);
2945
2946   if (mDisabledMode != DisabledTrackMode::ENABLED) {
2947     listener->IncreaseDisabled(mDisabledMode);
2948   }
2949
2950   if (mEnded) {
2951     return;
2952   }
2953
2954   // Pass buffered data to the listener
2955   VideoSegment bufferedData;
2956   size_t videoFrames = 0;
2957   VideoSegment& segment = *GetData<VideoSegment>();
2958   for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
2959        iter.Next()) {
2960     if (iter->mTimeStamp.IsNull()) {
2961       // No timestamp means this is only for the graph's internal book-keeping,
2962       // denoting a late start of the track.
2963       continue;
2964     }
2965     ++videoFrames;
2966     bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2967                              iter->mFrame.GetIntrinsicSize(),
2968                              iter->mFrame.GetPrincipalHandle(),
2969                              iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2970   }
2971
2972   VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
2973   for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
2974        iter.Next()) {
2975     ++videoFrames;
2976     MOZ_ASSERT(!iter->mTimeStamp.IsNull());
2977     bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2978                              iter->mFrame.GetIntrinsicSize(),
2979                              iter->mFrame.GetPrincipalHandle(),
2980                              iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2981   }
2982
2983   LOG(LogLevel::Info,
2984       ("%p: Notifying direct listener %p of %zu video frames and duration "
2985        "%" PRId64,
2986        GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
2987   listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
2988 }
2989
2990 void SourceMediaTrack::RemoveDirectListenerImpl(
2991     DirectMediaTrackListener* aListener) {
2992   mGraph->AssertOnGraphThreadOrNotRunning();
2993   MutexAutoLock lock(mMutex);
2994   for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
2995     const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
2996     if (l == aListener) {
2997       if (mDisabledMode != DisabledTrackMode::ENABLED) {
2998         aListener->DecreaseDisabled(mDisabledMode);
2999       }
3000       aListener->NotifyDirectListenerUninstalled();
3001       mDirectTrackListeners.RemoveElementAt(i);
3002     }
3003   }
3004 }
3005
3006 void SourceMediaTrack::End() {
3007   MutexAutoLock lock(mMutex);
3008   if (!mUpdateTrack) {
3009     // Already ended
3010     return;
3011   }
3012   mUpdateTrack->mEnded = true;
3013   if (auto graph = GraphImpl()) {
3014     MonitorAutoLock lock(graph->GetMonitor());
3015     if (graph->CurrentDriver()) {  // graph has not completed forced shutdown
3016       graph->EnsureNextIteration();
3017     }
3018   }
3019 }
3020
3021 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
3022   MOZ_ASSERT(mGraph->OnGraphThread());
3023   {
3024     MutexAutoLock lock(mMutex);
3025     const DisabledTrackMode oldMode = mDirectDisabledMode;
3026     const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
3027     const bool enabled = aMode == DisabledTrackMode::ENABLED;
3028     mDirectDisabledMode = aMode;
3029     for (const auto& l : mDirectTrackListeners) {
3030       if (!oldEnabled && enabled) {
3031         LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3032                               "direct listener enabled",
3033                               GraphImpl(), this));
3034         l->DecreaseDisabled(oldMode);
3035       } else if (oldEnabled && !enabled) {
3036         LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3037                               "direct listener disabled",
3038                               GraphImpl(), this));
3039         l->IncreaseDisabled(aMode);
3040       }
3041     }
3042   }
3043   MediaTrack::SetDisabledTrackModeImpl(aMode);
3044 }
3045
3046 uint32_t SourceMediaTrack::NumberOfChannels() const {
3047   AudioSegment* audio = GetData<AudioSegment>();
3048   MOZ_DIAGNOSTIC_ASSERT(audio);
3049   if (!audio) {
3050     return 0;
3051   }
3052   return audio->MaxChannelCount();
3053 }
3054
3055 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3056   GraphImpl()->AssertOnGraphThreadOrNotRunning();
3057   MutexAutoLock lock(mMutex);
3058
3059   for (auto& l : mDirectTrackListeners.Clone()) {
3060     l->NotifyDirectListenerUninstalled();
3061   }
3062   mDirectTrackListeners.Clear();
3063 }
3064
3065 void SourceMediaTrack::SetVolume(float aVolume) {
3066   MutexAutoLock lock(mMutex);
3067   mVolume = aVolume;
3068 }
3069
3070 float SourceMediaTrack::GetVolumeLocked() {
3071   mMutex.AssertCurrentThreadOwns();
3072   return mVolume;
3073 }
3074
3075 SourceMediaTrack::~SourceMediaTrack() = default;
3076
3077 void MediaInputPort::Init() {
3078   mGraph->AssertOnGraphThreadOrNotRunning();
3079   LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)",
3080                         mSource->GraphImpl(), this, mSource, mDest));
3081   // Only connect the port if it wasn't disconnected on allocation.
3082   if (mSource) {
3083     mSource->AddConsumer(this);
3084     mDest->AddInput(this);
3085   }
3086   // mPortCount decremented via MediaInputPort::Destroy's message
3087   ++mGraph->mPortCount;
3088 }
3089
3090 void MediaInputPort::Disconnect() {
3091   mGraph->AssertOnGraphThreadOrNotRunning();
3092   NS_ASSERTION(!mSource == !mDest,
3093                "mSource and mDest must either both be null or both non-null");
3094
3095   if (!mSource) {
3096     return;
3097   }
3098
3099   mSource->RemoveConsumer(this);
3100   mDest->RemoveInput(this);
3101   mSource = nullptr;
3102   mDest = nullptr;
3103
3104   mGraph->SetTrackOrderDirty();
3105 }
3106
3107 MediaTrack* MediaInputPort::GetSource() const {
3108   mGraph->AssertOnGraphThreadOrNotRunning();
3109   return mSource;
3110 }
3111
3112 ProcessedMediaTrack* MediaInputPort::GetDestination() const {
3113   mGraph->AssertOnGraphThreadOrNotRunning();
3114   return mDest;
3115 }
3116
3117 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
3118     MediaInputPort const* aPort, GraphTime aTime) {
3119   InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
3120   if (!aPort) {
3121     result.mStart = aTime;
3122     result.mInputIsBlocked = true;
3123     return result;
3124   }
3125   aPort->mGraph->AssertOnGraphThreadOrNotRunning();
3126   if (aTime >= aPort->mDest->mStartBlocking) {
3127     return result;
3128   }
3129   result.mStart = aTime;
3130   result.mEnd = aPort->mDest->mStartBlocking;
3131   result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
3132   if (!result.mInputIsBlocked) {
3133     result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
3134   }
3135   return result;
3136 }
3137
3138 void MediaInputPort::Suspended() {
3139   mGraph->AssertOnGraphThreadOrNotRunning();
3140   mDest->InputSuspended(this);
3141 }
3142
3143 void MediaInputPort::Resumed() {
3144   mGraph->AssertOnGraphThreadOrNotRunning();
3145   mDest->InputResumed(this);
3146 }
3147
3148 void MediaInputPort::Destroy() {
3149   class Message : public ControlMessage {
3150    public:
3151     explicit Message(MediaInputPort* aPort)
3152         : ControlMessage(nullptr), mPort(aPort) {}
3153     void Run() override {
3154       TRACE("MediaInputPort::Destroy ControlMessage");
3155       mPort->Disconnect();
3156       --mPort->GraphImpl()->mPortCount;
3157       mPort->SetGraphImpl(nullptr);
3158       NS_RELEASE(mPort);
3159     }
3160     void RunDuringShutdown() override { Run(); }
3161     MediaInputPort* mPort;
3162   };
3163   // Keep a reference to the graph, since Message might RunDuringShutdown()
3164   // synchronously and make GraphImpl() invalid.
3165   RefPtr<MediaTrackGraphImpl> graph = mGraph;
3166   graph->AppendMessage(MakeUnique<Message>(this));
3167   --graph->mMainThreadPortCount;
3168 }
3169
3170 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
3171   mGraph->AssertOnGraphThreadOrNotRunning();
3172   return mGraph;
3173 }
3174
3175 MediaTrackGraph* MediaInputPort::Graph() const {
3176   mGraph->AssertOnGraphThreadOrNotRunning();
3177   return mGraph;
3178 }
3179
3180 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
3181   MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3182   DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
3183   MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
3184   mGraph = aGraph;
3185 }
3186
3187 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
3188     MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
3189   // This method creates two references to the MediaInputPort: one for
3190   // the main thread, and one for the MediaTrackGraph.
3191   class Message : public ControlMessage {
3192    public:
3193     explicit Message(MediaInputPort* aPort)
3194         : ControlMessage(aPort->mDest), mPort(aPort) {}
3195     void Run() override {
3196       TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3197       mPort->Init();
3198       // The graph holds its reference implicitly
3199       mPort->GraphImpl()->SetTrackOrderDirty();
3200       Unused << mPort.forget();
3201     }
3202     void RunDuringShutdown() override { Run(); }
3203     RefPtr<MediaInputPort> mPort;
3204   };
3205
3206   MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
3207   RefPtr<MediaInputPort> port;
3208   if (aTrack->IsDestroyed()) {
3209     // Create a port that's disconnected, which is what it'd be after its source
3210     // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3211     // this later is fine.
3212     port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
3213                               aOutputNumber);
3214   } else {
3215     MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
3216     port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
3217                               aOutputNumber);
3218   }
3219   ++GraphImpl()->mMainThreadPortCount;
3220   GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3221   return port.forget();
3222 }
3223
3224 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
3225   class Message : public ControlMessage {
3226    public:
3227     Message(ProcessedMediaTrack* aTrack, bool aAutoend)
3228         : ControlMessage(aTrack), mAutoend(aAutoend) {}
3229     void Run() override {
3230       TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3231       static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
3232     }
3233     bool mAutoend;
3234   };
3235   if (mMainThreadDestroyed) {
3236     return;
3237   }
3238   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
3239 }
3240
3241 void ProcessedMediaTrack::DestroyImpl() {
3242   for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3243     mInputs[i]->Disconnect();
3244   }
3245
3246   for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3247     mSuspendedInputs[i]->Disconnect();
3248   }
3249
3250   MediaTrack::DestroyImpl();
3251   // The track order is only important if there are connections, in which
3252   // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3253   // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3254   // SetTrackOrderDirty(), for other reasons.
3255 }
3256
3257 MediaTrackGraphImpl::MediaTrackGraphImpl(
3258     GraphDriverType aDriverRequested, GraphRunType aRunTypeRequested,
3259     TrackRate aSampleRate, uint32_t aChannelCount,
3260     CubebUtils::AudioDeviceID aOutputDeviceID,
3261     nsISerialEventTarget* aMainThread)
3262     : MediaTrackGraph(aSampleRate),
3263       mGraphRunner(aRunTypeRequested == SINGLE_THREAD
3264                        ? GraphRunner::Create(this)
3265                        : already_AddRefed<GraphRunner>(nullptr)),
3266       mFirstCycleBreaker(0)
3267       // An offline graph is not initially processing.
3268       ,
3269       mEndTime(aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX),
3270       mPortCount(0),
3271       mOutputDeviceID(aOutputDeviceID),
3272       mMonitor("MediaTrackGraphImpl"),
3273       mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
3274       mPostedRunInStableStateEvent(false),
3275       mGraphDriverRunning(false),
3276       mPostedRunInStableState(false),
3277       mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER),
3278       mTrackOrderDirty(false),
3279       mMainThread(aMainThread),
3280       mSelfRef(this),
3281       mGlobalVolume(CubebUtils::GetVolumeScale())
3282 #ifdef DEBUG
3283       ,
3284       mCanRunMessagesSynchronously(false)
3285 #endif
3286       ,
3287       mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3288       mAudioOutputLatency(0.0),
3289       mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
3290   bool failedToGetShutdownBlocker = false;
3291   if (!IsNonRealtime()) {
3292     failedToGetShutdownBlocker = !AddShutdownBlocker();
3293   }
3294
3295   if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
3296       failedToGetShutdownBlocker) {
3297     // At least one of the following happened
3298     // - Failed to create thread.
3299     // - Failed to install a shutdown blocker when one is needed.
3300     // Because we have a fail state, jump to last phase of the lifecycle.
3301     mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
3302     RemoveShutdownBlocker();  // No-op if blocker wasn't added.
3303 #ifdef DEBUG
3304     mCanRunMessagesSynchronously = true;
3305 #endif
3306     return;
3307   }
3308   if (mRealtime) {
3309     if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3310       // Always start with zero input channels, and no particular preferences
3311       // for the input channel.
3312       mDriver = new AudioCallbackDriver(this, nullptr, mSampleRate,
3313                                         aChannelCount, 0, mOutputDeviceID,
3314                                         nullptr, AudioInputType::Unknown);
3315     } else {
3316       mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
3317     }
3318   } else {
3319     mDriver =
3320         new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
3321   }
3322
3323   mLastMainThreadUpdate = TimeStamp::Now();
3324
3325   RegisterWeakAsyncMemoryReporter(this);
3326 }
3327
3328 #ifdef DEBUG
3329 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
3330   return aDriver->OnThread() ||
3331          (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
3332 }
3333 #endif
3334
3335 void MediaTrackGraphImpl::Destroy() {
3336   // First unregister from memory reporting.
3337   UnregisterWeakMemoryReporter(this);
3338
3339   // Clear the self reference which will destroy this instance if all
3340   // associated GraphDrivers are destroyed.
3341   mSelfRef = nullptr;
3342 }
3343
3344 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3345 // GTests can create a graph without a window.
3346 /* static */
3347 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
3348     uint64_t aWindowID, bool aShouldResistFingerprinting, TrackRate aSampleRate,
3349     CubebUtils::AudioDeviceID aOutputDeviceID) {
3350   MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3351
3352   TrackRate sampleRate =
3353       aSampleRate
3354           ? aSampleRate
3355           : CubebUtils::PreferredSampleRate(aShouldResistFingerprinting);
3356   GraphKey key(aWindowID, sampleRate, aOutputDeviceID);
3357
3358   return gGraphs.Get(key);
3359 }
3360
3361 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3362 // window is a real inner Window, not a WindowProxy.
3363 /* static */
3364 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
3365     nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
3366     CubebUtils::AudioDeviceID aOutputDeviceID) {
3367   return MediaTrackGraphImpl::GetInstanceIfExists(
3368       aWindow->WindowID(),
3369       aWindow->AsGlobal()->ShouldResistFingerprinting(
3370           RFPTarget::AudioSampleRate),
3371       aSampleRate, aOutputDeviceID);
3372 }
3373
3374 /* static */
3375 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
3376     GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
3377     bool aShouldResistFingerprinting, TrackRate aSampleRate,
3378     CubebUtils::AudioDeviceID aOutputDeviceID,
3379     nsISerialEventTarget* aMainThread) {
3380   MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3381
3382   TrackRate sampleRate =
3383       aSampleRate
3384           ? aSampleRate
3385           : CubebUtils::PreferredSampleRate(aShouldResistFingerprinting);
3386   MediaTrackGraphImpl* graph = GetInstanceIfExists(
3387       aWindowID, aShouldResistFingerprinting, sampleRate, aOutputDeviceID);
3388
3389   if (!graph) {
3390     GraphRunType runType = DIRECT_DRIVER;
3391     if (aGraphDriverRequested != OFFLINE_THREAD_DRIVER &&
3392         (Preferences::GetBool("media.audiograph.single_thread.enabled",
3393                               false))) {
3394       runType = SINGLE_THREAD;
3395     }
3396
3397     // In a real time graph, the number of output channels is determined by
3398     // the underlying number of channel of the default audio output device, and
3399     // capped to 8.
3400     uint32_t channelCount =
3401         std::min<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
3402     graph = new MediaTrackGraphImpl(aGraphDriverRequested, runType, sampleRate,
3403                                     channelCount, aOutputDeviceID, aMainThread);
3404     GraphKey key(aWindowID, sampleRate, aOutputDeviceID);
3405     gGraphs.InsertOrUpdate(key, graph);
3406
3407     LOG(LogLevel::Debug,
3408         ("Starting up MediaTrackGraph %p for window 0x%" PRIx64, graph,
3409          aWindowID));
3410   }
3411
3412   return graph;
3413 }
3414
3415 /* static */
3416 MediaTrackGraph* MediaTrackGraph::GetInstance(
3417     GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
3418     TrackRate aSampleRate, CubebUtils::AudioDeviceID aOutputDeviceID) {
3419   return MediaTrackGraphImpl::GetInstance(
3420       aGraphDriverRequested, aWindow->WindowID(),
3421       aWindow->AsGlobal()->ShouldResistFingerprinting(
3422           RFPTarget::AudioSampleRate),
3423       aSampleRate, aOutputDeviceID,
3424       aWindow->EventTargetFor(TaskCategory::Other));
3425 }
3426
3427 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
3428     TrackRate aSampleRate, nsPIDOMWindowInner* aWindow) {
3429   MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3430
3431   nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
3432   // aWindow can be null when the document is being unlinked, so this works when
3433   // with a generic main thread if that's the case.
3434   if (aWindow) {
3435     mainThread =
3436         aWindow->AsGlobal()->AbstractMainThreadFor(TaskCategory::Other);
3437   }
3438
3439   // Offline graphs have 0 output channel count: they write the output to a
3440   // buffer, not an audio output track.
3441   MediaTrackGraphImpl* graph =
3442       new MediaTrackGraphImpl(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, aSampleRate,
3443                               0, DEFAULT_OUTPUT_DEVICE, mainThread);
3444
3445   LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
3446
3447   return graph;
3448 }
3449
3450 void MediaTrackGraph::ForceShutDown() {
3451   MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3452
3453   MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3454
3455   graph->ForceShutDown();
3456 }
3457
3458 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIThreadObserver,
3459                   nsITimerCallback, nsINamed)
3460
3461 NS_IMETHODIMP
3462 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3463                                     nsISupports* aData, bool aAnonymize) {
3464   MOZ_ASSERT(NS_IsMainThread());
3465   if (mMainThreadTrackCount == 0) {
3466     // No tracks to report.
3467     FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3468     return NS_OK;
3469   }
3470
3471   class Message final : public ControlMessage {
3472    public:
3473     Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
3474             nsISupports* aHandlerData)
3475         : ControlMessage(nullptr),
3476           mGraph(aGraph),
3477           mHandleReport(aHandleReport),
3478           mHandlerData(aHandlerData) {}
3479     void Run() override {
3480       TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3481       mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3482                                           mHandlerData.forget());
3483     }
3484     void RunDuringShutdown() override {
3485       // Run this message during shutdown too, so that endReports is called.
3486       Run();
3487     }
3488     MediaTrackGraphImpl* mGraph;
3489     // nsMemoryReporterManager keeps the callback and data alive only if it
3490     // does not time out.
3491     nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3492     nsCOMPtr<nsISupports> mHandlerData;
3493   };
3494
3495   AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3496
3497   return NS_OK;
3498 }
3499
3500 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3501     already_AddRefed<nsIHandleReportCallback> aHandleReport,
3502     already_AddRefed<nsISupports> aHandlerData) {
3503   class FinishCollectRunnable final : public Runnable {
3504    public:
3505     explicit FinishCollectRunnable(
3506         already_AddRefed<nsIHandleReportCallback> aHandleReport,
3507         already_AddRefed<nsISupports> aHandlerData)
3508         : mozilla::Runnable("FinishCollectRunnable"),
3509           mHandleReport(aHandleReport),
3510           mHandlerData(aHandlerData) {}
3511
3512     NS_IMETHOD Run() override {
3513       TRACE("MTG::FinishCollectReports ControlMessage");
3514       MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3515                                                 std::move(mAudioTrackSizes));
3516       return NS_OK;
3517     }
3518
3519     nsTArray<AudioNodeSizes> mAudioTrackSizes;
3520
3521    private:
3522     ~FinishCollectRunnable() = default;
3523
3524     // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3525     // constructor modifies the ref-count, which cannot be done off main
3526     // thread.
3527     RefPtr<nsIHandleReportCallback> mHandleReport;
3528     RefPtr<nsISupports> mHandlerData;
3529   };
3530
3531   RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
3532       std::move(aHandleReport), std::move(aHandlerData));
3533
3534   auto audioTrackSizes = &runnable->mAudioTrackSizes;
3535
3536   for (MediaTrack* t : AllTracks()) {
3537     AudioNodeTrack* track = t->AsAudioNodeTrack();
3538     if (track) {
3539       AudioNodeSizes* usage = audioTrackSizes->AppendElement();
3540       track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3541     }
3542   }
3543
3544   mMainThread->Dispatch(runnable.forget());
3545 }
3546
3547 void MediaTrackGraphImpl::FinishCollectReports(
3548     nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3549     const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
3550   MOZ_ASSERT(NS_IsMainThread());
3551
3552   nsCOMPtr<nsIMemoryReporterManager> manager =
3553       do_GetService("@mozilla.org/memory-reporter-manager;1");
3554
3555   if (!manager) return;
3556
3557 #define REPORT(_path, _amount, _desc)                                    \
3558   aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3559                           nsLiteralCString(_desc), aData);
3560
3561   for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
3562     const AudioNodeSizes& usage = aAudioTrackSizes[i];
3563     const char* const nodeType =
3564         usage.mNodeType ? usage.mNodeType : "<unknown>";
3565
3566     nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3567                                nodeType);
3568     REPORT(enginePath, usage.mEngine,
3569            "Memory used by AudioNode engine objects (Web Audio).");
3570
3571     nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
3572                               nodeType);
3573     REPORT(trackPath, usage.mTrack,
3574            "Memory used by AudioNode track objects (Web Audio).");
3575   }
3576
3577   size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3578   if (hrtfLoaders) {
3579     REPORT(nsLiteralCString(
3580                "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3581            hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
3582   }
3583
3584 #undef REPORT
3585
3586   manager->EndReport();
3587 }
3588
3589 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
3590   SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
3591   AddTrack(track);
3592   return track;
3593 }
3594
3595 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
3596     MediaSegment::Type aType) {
3597   ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
3598   AddTrack(track);
3599   return track;
3600 }
3601
3602 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
3603   AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
3604   AddTrack(track);
3605   return track;
3606 }
3607
3608 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
3609     CrossGraphReceiver* aReceiver) {
3610   CrossGraphTransmitter* track =
3611       new CrossGraphTransmitter(GraphRate(), aReceiver);
3612   AddTrack(track);
3613   return track;
3614 }
3615
3616 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
3617     TrackRate aTransmitterRate) {
3618   CrossGraphReceiver* track =
3619       new CrossGraphReceiver(GraphRate(), aTransmitterRate);
3620   AddTrack(track);
3621   return track;
3622 }
3623
3624 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
3625   MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3626 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3627   if (graph->mRealtime) {
3628     bool found = false;
3629     for (const auto& currentGraph : gGraphs.Values()) {
3630       if (currentGraph == graph) {
3631         found = true;
3632         break;
3633       }
3634     }
3635     MOZ_DIAGNOSTIC_ASSERT(found, "Graph must not be shutting down");
3636   }
3637 #endif
3638   NS_ADDREF(aTrack);
3639   aTrack->SetGraphImpl(graph);
3640   ++graph->mMainThreadTrackCount;
3641   graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
3642 }
3643
3644 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
3645   MOZ_ASSERT(NS_IsMainThread());
3646   MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
3647   if (--mMainThreadTrackCount == 0) {
3648     LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
3649                          "main thread. Graph will shut down.",
3650                          this, aTrack));
3651     // Find the graph in the hash table and remove it.
3652     for (auto iter = gGraphs.Iter(); !iter.Done(); iter.Next()) {
3653       if (iter.UserData() == this) {
3654         iter.Remove();
3655         break;
3656       }
3657     }
3658     // The graph thread will shut itself down soon, but won't be able to do
3659     // that if JS continues to run.
3660     InterruptJS();
3661   }
3662 }
3663
3664 auto MediaTrackGraph::NotifyWhenDeviceStarted(MediaTrack* aTrack)
3665     -> RefPtr<GraphStartedPromise> {
3666   MOZ_ASSERT(NS_IsMainThread());
3667   MozPromiseHolder<GraphStartedPromise> h;
3668   RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
3669   aTrack->GraphImpl()->NotifyWhenGraphStarted(aTrack, std::move(h));
3670   return p;
3671 }
3672
3673 void MediaTrackGraphImpl::NotifyWhenGraphStarted(
3674     RefPtr<MediaTrack> aTrack,
3675     MozPromiseHolder<GraphStartedPromise>&& aHolder) {
3676   class GraphStartedNotificationControlMessage : public ControlMessage {
3677     RefPtr<MediaTrack> mMediaTrack;
3678     MozPromiseHolder<GraphStartedPromise> mHolder;
3679
3680    public:
3681     GraphStartedNotificationControlMessage(
3682         RefPtr<MediaTrack> aTrack,
3683         MozPromiseHolder<GraphStartedPromise>&& aHolder)
3684         : ControlMessage(nullptr),
3685           mMediaTrack(std::move(aTrack)),
3686           mHolder(std::move(aHolder)) {}
3687     void Run() override {
3688       TRACE("MTG::GraphStartedNotificationControlMessage ControlMessage");
3689       // This runs on the graph thread, so when this runs, and the current
3690       // driver is an AudioCallbackDriver, we know the audio hardware is
3691       // started. If not, we are going to switch soon, keep reposting this
3692       // ControlMessage.
3693       MediaTrackGraphImpl* graphImpl = mMediaTrack->GraphImpl();
3694       if (graphImpl->CurrentDriver()->AsAudioCallbackDriver() &&
3695           graphImpl->CurrentDriver()->ThreadRunning() &&
3696           !graphImpl->CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3697         // Avoid Resolve's locking on the graph thread by doing it on main.
3698         graphImpl->Dispatch(NS_NewRunnableFunction(
3699             "MediaTrackGraphImpl::NotifyWhenGraphStarted::Resolver",
3700             [holder = std::move(mHolder)]() mutable {
3701               holder.Resolve(true, __func__);
3702             }));
3703       } else {
3704         graphImpl->DispatchToMainThreadStableState(
3705             NewRunnableMethod<
3706                 StoreCopyPassByRRef<RefPtr<MediaTrack>>,
3707                 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
3708                 "MediaTrackGraphImpl::NotifyWhenGraphStarted", graphImpl,
3709                 &MediaTrackGraphImpl::NotifyWhenGraphStarted,
3710                 std::move(mMediaTrack), std::move(mHolder)));
3711       }
3712     }
3713     void RunDuringShutdown() override {
3714       mHolder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
3715     }
3716   };
3717
3718   if (aTrack->IsDestroyed()) {
3719     aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
3720     return;
3721   }
3722
3723   MediaTrackGraphImpl* graph = aTrack->GraphImpl();
3724   graph->AppendMessage(MakeUnique<GraphStartedNotificationControlMessage>(
3725       std::move(aTrack), std::move(aHolder)));
3726 }
3727
3728 class AudioContextOperationControlMessage : public ControlMessage {
3729   using AudioContextOperationPromise =
3730       MediaTrackGraph::AudioContextOperationPromise;
3731
3732  public:
3733   AudioContextOperationControlMessage(
3734       MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3735       AudioContextOperation aOperation,
3736       MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
3737       : ControlMessage(aDestinationTrack),
3738         mTracks(std::move(aTracks)),
3739         mAudioContextOperation(aOperation),
3740         mHolder(std::move(aHolder)) {}
3741   void Run() override {
3742     TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3743                   kAudioContextOptionsStrings[static_cast<uint8_t>(
3744                       mAudioContextOperation)]);
3745     mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
3746   }
3747   void RunDuringShutdown() override {
3748     MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
3749                "We should be reviving the graph?");
3750     mHolder.Reject(false, __func__);
3751   }
3752
3753   nsTArray<RefPtr<MediaTrack>> mTracks;
3754   AudioContextOperation mAudioContextOperation;
3755   MozPromiseHolder<AudioContextOperationPromise> mHolder;
3756 };
3757
3758 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3759     AudioContextOperationControlMessage* aMessage) {
3760   MOZ_ASSERT(OnGraphThread());
3761   // Initialize state to zero. This silences a GCC warning about uninitialized
3762   // values, because although the switch below initializes state for all valid
3763   // enum values, the actual value could be any integer that fits in the enum.
3764   AudioContextState state{0};
3765   switch (aMessage->mAudioContextOperation) {
3766     // Suspend and Close operations may be performed immediately because no
3767     // specific kind of GraphDriver is required.  CheckDriver() will schedule
3768     // a change to a SystemCallbackDriver if all tracks are suspended.
3769     case AudioContextOperation::Suspend:
3770       state = AudioContextState::Suspended;
3771       break;
3772     case AudioContextOperation::Close:
3773       state = AudioContextState::Closed;
3774       break;
3775     case AudioContextOperation::Resume:
3776       // Resume operations require an AudioCallbackDriver.  CheckDriver() will
3777       // schedule an AudioCallbackDriver if necessary and process pending
3778       // operations if and when an AudioCallbackDriver is running.
3779       mPendingResumeOperations.EmplaceBack(aMessage);
3780       return;
3781   }
3782   // First resolve any pending Resume promises for the same AudioContext so as
3783   // to resolve its associated promises in the same order as they were
3784   // created.  These Resume operations are considered complete and immediately
3785   // canceled by the Suspend or Close.
3786   MediaTrack* destinationTrack = aMessage->GetTrack();
3787   bool shrinking = false;
3788   auto moveDest = mPendingResumeOperations.begin();
3789   for (PendingResumeOperation& op : mPendingResumeOperations) {
3790     if (op.DestinationTrack() == destinationTrack) {
3791       op.Apply(this);
3792       shrinking = true;
3793       continue;
3794     }
3795     if (shrinking) {  // Fill-in gaps in the array.
3796       *moveDest = std::move(op);
3797     }
3798     ++moveDest;
3799   }
3800   mPendingResumeOperations.TruncateLength(moveDest -
3801                                           mPendingResumeOperations.begin());
3802
3803   for (MediaTrack* track : aMessage->mTracks) {
3804     track->IncrementSuspendCount();
3805   }
3806   // Resolve after main thread state is up to date with completed processing.
3807   DispatchToMainThreadStableState(NS_NewRunnableFunction(
3808       "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
3809       [holder = std::move(aMessage->mHolder), state]() mutable {
3810         holder.Resolve(state, __func__);
3811       }));
3812 }
3813
3814 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
3815     AudioContextOperationControlMessage* aMessage)
3816     : mDestinationTrack(aMessage->GetTrack()),
3817       mTracks(std::move(aMessage->mTracks)),
3818       mHolder(std::move(aMessage->mHolder)) {
3819   MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
3820 }
3821
3822 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
3823     MediaTrackGraphImpl* aGraph) {
3824   MOZ_ASSERT(aGraph->OnGraphThread());
3825   for (MediaTrack* track : mTracks) {
3826     track->DecrementSuspendCount();
3827   }
3828   // The graph is provided through the parameter so that it is available even
3829   // when the track is destroyed.
3830   aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
3831       "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
3832         holder.Resolve(AudioContextState::Running, __func__);
3833       }));
3834 }
3835
3836 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
3837   // The graph is shutting down before the operation completed.
3838   MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
3839              mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
3840                  MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
3841   mHolder.Reject(false, __func__);
3842 }
3843
3844 auto MediaTrackGraph::ApplyAudioContextOperation(
3845     MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3846     AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
3847   MozPromiseHolder<AudioContextOperationPromise> holder;
3848   RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
3849   MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3850   graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
3851       aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
3852   return p;
3853 }
3854
3855 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount() const {
3856   MOZ_ASSERT(OnGraphThread());
3857   // The audio output channel count for a graph is the maximum of the output
3858   // channel count of all the tracks that are in mAudioOutputs, or the max audio
3859   // output channel count the machine can do, whichever is smaller.
3860   uint32_t channelCount = 0;
3861   for (auto& tkv : mAudioOutputs) {
3862     channelCount = std::max(channelCount, tkv.mTrack->NumberOfChannels());
3863   }
3864   channelCount = std::min(channelCount, mMaxOutputChannelCount);
3865   if (channelCount) {
3866     return channelCount;
3867   } else {
3868     if (CurrentDriver()->AsAudioCallbackDriver()) {
3869       return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
3870     }
3871     return 2;
3872   }
3873 }
3874
3875 double MediaTrackGraph::AudioOutputLatency() {
3876   return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
3877 }
3878
3879 double MediaTrackGraphImpl::AudioOutputLatency() {
3880   MOZ_ASSERT(NS_IsMainThread());
3881   if (mAudioOutputLatency != 0.0) {
3882     return mAudioOutputLatency;
3883   }
3884   MonitorAutoLock lock(mMonitor);
3885   if (CurrentDriver()->AsAudioCallbackDriver()) {
3886     mAudioOutputLatency = CurrentDriver()
3887                               ->AsAudioCallbackDriver()
3888                               ->AudioOutputLatency()
3889                               .ToSeconds();
3890   } else {
3891     // Failure mode: return 0.0 if running on a normal thread.
3892     mAudioOutputLatency = 0.0;
3893   }
3894
3895   return mAudioOutputLatency;
3896 }
3897
3898 bool MediaTrackGraph::IsNonRealtime() const {
3899   return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
3900 }
3901
3902 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
3903   MOZ_ASSERT(NS_IsMainThread(), "main thread only");
3904
3905   MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3906   NS_ASSERTION(!graph->mRealtime, "non-realtime only");
3907
3908   class Message : public ControlMessage {
3909    public:
3910     explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
3911         : ControlMessage(nullptr),
3912           mGraph(aGraph),
3913           mTicksToProcess(aTicksToProcess) {}
3914     void Run() override {
3915       TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
3916       MOZ_ASSERT(mGraph->mEndTime == 0,
3917                  "StartNonRealtimeProcessing should be called only once");
3918       mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
3919           mGraph->mStateComputedTime + mTicksToProcess);
3920     }
3921     // The graph owns this message.
3922     MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
3923     uint32_t mTicksToProcess;
3924   };
3925
3926   graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
3927 }
3928
3929 void MediaTrackGraphImpl::InterruptJS() {
3930   MonitorAutoLock lock(mMonitor);
3931   mInterruptJSCalled = true;
3932   if (mJSContext) {
3933     JS_RequestInterruptCallback(mJSContext);
3934   }
3935 }
3936
3937 static bool InterruptCallback(JSContext* aCx) {
3938   // Interrupt future calls also.
3939   JS_RequestInterruptCallback(aCx);
3940   // Stop execution.
3941   return false;
3942 }
3943
3944 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
3945   MOZ_ASSERT(OnGraphThread());
3946   MOZ_ASSERT(aCx);
3947
3948   auto* impl = static_cast<MediaTrackGraphImpl*>(this);
3949   MonitorAutoLock lock(impl->mMonitor);
3950   if (impl->mJSContext) {
3951     MOZ_ASSERT(impl->mJSContext == aCx);
3952     return;
3953   }
3954   JS_AddInterruptCallback(aCx, InterruptCallback);
3955   impl->mJSContext = aCx;
3956   if (impl->mInterruptJSCalled) {
3957     JS_RequestInterruptCallback(aCx);
3958   }
3959 }
3960
3961 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
3962   MediaTrack* t = aPort->GetSource();
3963   if (!t->IsSuspended()) {
3964     mInputs.AppendElement(aPort);
3965   } else {
3966     mSuspendedInputs.AppendElement(aPort);
3967   }
3968   GraphImpl()->SetTrackOrderDirty();
3969 }
3970
3971 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
3972   GraphImpl()->AssertOnGraphThreadOrNotRunning();
3973   mInputs.RemoveElement(aPort);
3974   mSuspendedInputs.AppendElement(aPort);
3975   GraphImpl()->SetTrackOrderDirty();
3976 }
3977
3978 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
3979   GraphImpl()->AssertOnGraphThreadOrNotRunning();
3980   mSuspendedInputs.RemoveElement(aPort);
3981   mInputs.AppendElement(aPort);
3982   GraphImpl()->SetTrackOrderDirty();
3983 }
3984
3985 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
3986   MOZ_ASSERT(OnGraphThread());
3987   LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
3988   if (GraphDriver* nextDriver = NextDriver()) {
3989     if (nextDriver != CurrentDriver()) {
3990       LOG(LogLevel::Debug,
3991           ("%p: Discarding previous next driver: %p", this, nextDriver));
3992     }
3993   }
3994   mNextDriver = aNextDriver;
3995 }
3996
3997 void MediaTrackGraph::RegisterCaptureTrackForWindow(
3998     uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
3999   MOZ_ASSERT(NS_IsMainThread());
4000   MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4001   graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
4002 }
4003
4004 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
4005     uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
4006   MOZ_ASSERT(NS_IsMainThread());
4007   WindowAndTrack winAndTrack;
4008   winAndTrack.mWindowId = aWindowId;
4009   winAndTrack.mCaptureTrackSink = aCaptureTrack;
4010   mWindowCaptureTracks.AppendElement(winAndTrack);
4011 }
4012
4013 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4014   MOZ_ASSERT(NS_IsMainThread());
4015   MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4016   graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
4017 }
4018
4019 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4020   MOZ_ASSERT(NS_IsMainThread());
4021   mWindowCaptureTracks.RemoveElementsBy(
4022       [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
4023 }
4024
4025 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
4026     uint64_t aWindowId, MediaTrack* aMediaTrack) {
4027   return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
4028                                                          aMediaTrack);
4029 }
4030
4031 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
4032     uint64_t aWindowId, MediaTrack* aMediaTrack) {
4033   MOZ_ASSERT(NS_IsMainThread());
4034   for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
4035     if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
4036       ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
4037       return sink->AllocateInputPort(aMediaTrack);
4038     }
4039   }
4040   return nullptr;
4041 }
4042
4043 void MediaTrackGraph::DispatchToMainThreadStableState(
4044     already_AddRefed<nsIRunnable> aRunnable) {
4045   AssertOnGraphThreadOrNotRunning();
4046   static_cast<MediaTrackGraphImpl*>(this)
4047       ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
4048 }
4049
4050 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
4051   MOZ_ASSERT(NS_IsMainThread());
4052   return mMainThreadGraphTime;
4053 }
4054
4055 GraphTime MediaTrackGraph::ProcessedTime() const {
4056   AssertOnGraphThreadOrNotRunning();
4057   return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
4058 }
4059
4060 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4061     CubebUtils::AudioDeviceID aID) {
4062   MOZ_ASSERT(OnGraphThreadOrNotRunning());
4063   DeviceInputTrack* t =
4064       mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4065   return t ? t->MaxRequestedInputChannels() : 0;
4066 }
4067
4068 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
4069     CubebUtils::AudioDeviceID aID) {
4070   MOZ_ASSERT(OnGraphThreadOrNotRunning());
4071   DeviceInputTrack* t =
4072       mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4073   return t && t->HasVoiceInput() ? AudioInputType::Voice
4074                                  : AudioInputType::Unknown;
4075 }
4076
4077 void MediaTrackGraphImpl::SetNewNativeInput() {
4078   MOZ_ASSERT(NS_IsMainThread());
4079   MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4080
4081   LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
4082
4083   NonNativeInputTrack* track =
4084       mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
4085   if (!track) {
4086     LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
4087     return;
4088   }
4089
4090   const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
4091   const PrincipalHandle principal = track->mPrincipalHandle;
4092
4093   LOG(LogLevel::Debug,
4094       ("%p Select device %p as the new native input device", this, deviceId));
4095
4096   struct TrackListener {
4097     DeviceInputConsumerTrack* track;
4098     // Keep its reference so it won't be dropped when after
4099     // DisconnectDeviceInput().
4100     RefPtr<AudioDataListener> listener;
4101   };
4102   nsTArray<TrackListener> pairs;
4103
4104   for (const auto& t : track->GetConsumerTracks()) {
4105     pairs.AppendElement(
4106         TrackListener{t.get(), t->GetAudioDataListener().get()});
4107   }
4108
4109   for (TrackListener& pair : pairs) {
4110     pair.track->DisconnectDeviceInput();
4111   }
4112
4113   for (TrackListener& pair : pairs) {
4114     pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
4115     LOG(LogLevel::Debug,
4116         ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4117          pair.track, deviceId));
4118   }
4119
4120   LOG(LogLevel::Debug,
4121       ("%p Native input device is set to device %p now", this, deviceId));
4122
4123   MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4124 }
4125
4126 // nsIThreadObserver methods
4127
4128 NS_IMETHODIMP
4129 MediaTrackGraphImpl::OnDispatchedEvent() {
4130   MonitorAutoLock lock(mMonitor);
4131   EnsureNextIteration();
4132   return NS_OK;
4133 }
4134
4135 NS_IMETHODIMP
4136 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
4137   return NS_OK;
4138 }
4139
4140 NS_IMETHODIMP
4141 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
4142   return NS_OK;
4143 }
4144 }  // namespace mozilla