Bug 1869043 assert that graph set access is main thread only r=padenot
[gecko.git] / dom / media / MediaTrackGraph.cpp
blob3eef864e30f4c12a609196b7ab5effc121b5fe0f
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaTrackGraphImpl.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "CrossGraphPort.h"
12 #include "VideoSegment.h"
13 #include "nsContentUtils.h"
14 #include "nsGlobalWindowInner.h"
15 #include "nsPrintfCString.h"
16 #include "nsServiceManagerUtils.h"
17 #include "prerror.h"
18 #include "mozilla/Logging.h"
19 #include "mozilla/Attributes.h"
20 #include "ForwardedInputTrack.h"
21 #include "ImageContainer.h"
22 #include "AudioCaptureTrack.h"
23 #include "AudioNodeTrack.h"
24 #include "AudioNodeExternalInputTrack.h"
25 #if defined(MOZ_WEBRTC)
26 # include "MediaEngineWebRTCAudio.h"
27 #endif // MOZ_WEBRTC
28 #include "MediaTrackListener.h"
29 #include "mozilla/dom/BaseAudioContextBinding.h"
30 #include "mozilla/dom/Document.h"
31 #include "mozilla/dom/WorkletThread.h"
32 #include "mozilla/media/MediaUtils.h"
33 #include <algorithm>
34 #include "GeckoProfiler.h"
35 #include "VideoFrameContainer.h"
36 #include "mozilla/AbstractThread.h"
37 #include "mozilla/StaticPrefs_dom.h"
38 #include "mozilla/StaticPrefs_media.h"
39 #include "transport/runnable_utils.h"
40 #include "VideoUtils.h"
41 #include "GraphRunner.h"
42 #include "Tracing.h"
43 #include "UnderrunHandler.h"
44 #include "mozilla/CycleCollectedJSRuntime.h"
45 #include "mozilla/Preferences.h"
47 #include "webaudio/blink/DenormalDisabler.h"
48 #include "webaudio/blink/HRTFDatabaseLoader.h"
50 using namespace mozilla::layers;
51 using namespace mozilla::dom;
52 using namespace mozilla::gfx;
53 using namespace mozilla::media;
55 namespace mozilla {
57 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
58 #ifdef LOG
59 # undef LOG
60 #endif // LOG
61 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
63 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
64 return mNativeInputTrack.get();
67 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
68 CubebUtils::AudioDeviceID aID) {
69 if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
70 return mNativeInputTrack.get();
72 for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
73 if (t->mDeviceId == aID) {
74 return t.get();
77 return nullptr;
80 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
81 if (mNonNativeInputTracks.IsEmpty()) {
82 return nullptr;
84 return mNonNativeInputTracks[0].get();
87 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
88 if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
89 MOZ_ASSERT(!mNativeInputTrack);
90 mNativeInputTrack = native;
91 } else {
92 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
93 MOZ_ASSERT(nonNative);
94 struct DeviceTrackComparator {
95 public:
96 bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
97 CubebUtils::AudioDeviceID aDeviceId) const {
98 return aTrack->mDeviceId == aDeviceId;
101 MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
102 DeviceTrackComparator()));
103 mNonNativeInputTracks.AppendElement(nonNative);
107 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
108 if (aTrack->AsNativeInputTrack()) {
109 MOZ_ASSERT(mNativeInputTrack);
110 MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
111 mNativeInputTrack = nullptr;
112 } else {
113 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
114 MOZ_ASSERT(nonNative);
115 DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
116 MOZ_ASSERT(removed);
121 * A hash table containing the graph instances, one per Window ID,
122 * sample rate, and device ID combination.
125 struct MediaTrackGraphImpl::Lookup final {
126 HashNumber Hash() const {
127 return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
129 const uint64_t mWindowID;
130 const TrackRate mSampleRate;
131 const CubebUtils::AudioDeviceID mOutputDeviceID;
134 // Implicit to support GraphHashSet.lookup(*graph).
135 MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const {
136 return {mWindowID, mSampleRate, mOutputDeviceID};
139 namespace {
140 struct GraphHasher { // for HashSet
141 using Lookup = const MediaTrackGraphImpl::Lookup;
143 static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); }
145 static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) {
146 return aGraph->mWindowID == aLookup.mWindowID &&
147 aGraph->GraphRate() == aLookup.mSampleRate &&
148 aGraph->mOutputDeviceID == aLookup.mOutputDeviceID;
152 // The weak reference to the graph is removed when its last track is removed.
153 using GraphHashSet =
154 HashSet<MediaTrackGraphImpl*, GraphHasher, InfallibleAllocPolicy>;
155 GraphHashSet* Graphs() {
156 MOZ_ASSERT(NS_IsMainThread());
157 static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity
158 return &sGraphs;
160 } // anonymous namespace
162 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode,
163 MediaSegment* aSegment,
164 MediaSegment* aRawSegment) {
165 if (aDisabledMode == DisabledTrackMode::ENABLED) {
166 return;
168 if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
169 aSegment->ReplaceWithDisabled();
170 if (aRawSegment) {
171 aRawSegment->ReplaceWithDisabled();
173 } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
174 aSegment->ReplaceWithNull();
175 if (aRawSegment) {
176 aRawSegment->ReplaceWithNull();
178 } else {
179 MOZ_CRASH("Unsupported mode");
183 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
184 MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
185 "All tracks should have been destroyed by messages from the main "
186 "thread");
187 LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
188 LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
191 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
192 MOZ_ASSERT(OnGraphThreadOrNotRunning());
193 aTrack->mStartTime = mProcessedTime;
195 if (aTrack->IsSuspended()) {
196 mSuspendedTracks.AppendElement(aTrack);
197 LOG(LogLevel::Debug,
198 ("%p: Adding media track %p, in the suspended track array", this,
199 aTrack));
200 } else {
201 mTracks.AppendElement(aTrack);
202 LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack,
203 mTracks.Length()));
206 SetTrackOrderDirty();
209 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
210 MOZ_ASSERT(OnGraphThreadOrNotRunning());
211 // Remove references in mTrackUpdates before we allow aTrack to die.
212 // Pending updates are not needed (since the main thread has already given
213 // up the track) so we will just drop them.
215 MonitorAutoLock lock(mMonitor);
216 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
217 if (mTrackUpdates[i].mTrack == aTrack) {
218 mTrackUpdates[i].mTrack = nullptr;
223 // Ensure that mFirstCycleBreaker is updated when necessary.
224 SetTrackOrderDirty();
226 UnregisterAllAudioOutputs(aTrack);
228 if (aTrack->IsSuspended()) {
229 mSuspendedTracks.RemoveElement(aTrack);
230 } else {
231 mTracks.RemoveElement(aTrack);
234 LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
235 mTracks.Length()));
237 NS_RELEASE(aTrack); // probably destroying it
240 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
241 const MediaTrack* aTrack, GraphTime aTime) const {
242 MOZ_ASSERT(
243 aTime <= mStateComputedTime,
244 "Don't ask about times where we haven't made blocking decisions yet");
245 return std::max<TrackTime>(
246 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
249 GraphTime MediaTrackGraphImpl::IterationEnd() const {
250 MOZ_ASSERT(OnGraphThread());
251 return mIterationEndTime;
254 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
255 GraphTime aPrevCurrentTime) {
256 MOZ_ASSERT(OnGraphThread());
257 for (MediaTrack* track : AllTracks()) {
258 // Shouldn't have already notified of ended *and* have output!
259 MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
260 !track->mNotifiedEnded);
262 // Calculate blocked time and fire Blocked/Unblocked events
263 GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
264 NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
265 track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
266 blockedTime);
267 LOG(LogLevel::Verbose,
268 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
269 MediaTimeToSeconds(track->mStartTime),
270 MediaTimeToSeconds(blockedTime)));
271 track->mStartBlocking = mStateComputedTime;
273 TrackTime trackCurrentTime =
274 track->GraphTimeToTrackTime(mStateComputedTime);
275 if (track->mEnded) {
276 MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
277 if (!track->mNotifiedEnded) {
278 // Playout of this track ended and listeners have not been notified.
279 track->mNotifiedEnded = true;
280 SetTrackOrderDirty();
281 for (const auto& listener : track->mTrackListeners) {
282 listener->NotifyOutput(this, track->GetEnd());
283 listener->NotifyEnded(this);
286 } else {
287 for (const auto& listener : track->mTrackListeners) {
288 listener->NotifyOutput(this, trackCurrentTime);
294 template <typename C, typename Chunk>
295 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
296 C& aSegment,
297 TrackTime aStart,
298 TrackTime aEnd) {
299 MOZ_ASSERT(OnGraphThreadOrNotRunning());
300 MOZ_ASSERT(aTrack);
302 TrackTime offset = 0;
303 for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
304 chunk.Next()) {
305 if (offset >= aEnd) {
306 break;
308 offset += chunk->GetDuration();
309 if (chunk->IsNull() || offset < aStart) {
310 continue;
312 const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
313 if (principalHandle != aSegment.GetLastPrincipalHandle()) {
314 aSegment.SetLastPrincipalHandle(principalHandle);
315 LOG(LogLevel::Debug,
316 ("%p: MediaTrack %p, principalHandle "
317 "changed in %sChunk with duration %lld",
318 this, aTrack,
319 aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
320 (long long)chunk->GetDuration()));
321 for (const auto& listener : aTrack->mTrackListeners) {
322 listener->NotifyPrincipalHandleChanged(this, principalHandle);
328 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
329 MOZ_ASSERT(OnGraphThreadOrNotRunning());
330 for (MediaTrack* track : AllTracks()) {
331 TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
332 TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
333 if (!track->mSegment) {
334 continue;
336 if (track->mType == MediaSegment::AUDIO) {
337 ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
338 track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
339 } else if (track->mType == MediaSegment::VIDEO) {
340 ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
341 track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
342 } else {
343 MOZ_CRASH("Unknown track type");
348 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
349 GraphTime aEndBlockingDecisions) {
350 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
351 // underrun currently, since we'll always be able to produce data for them
352 // unless they block on some other track.
353 if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
354 return aEndBlockingDecisions;
356 // This track isn't ended or suspended. We don't need to call
357 // TrackTimeToGraphTime since an underrun is the only thing that can block
358 // it.
359 GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
360 #ifdef DEBUG
361 if (bufferEnd < mProcessedTime) {
362 LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
363 "bufferEnd %f < mProcessedTime %f (%" PRId64
364 " < %" PRId64 "), TrackTime %" PRId64,
365 this, aTrack, MediaTimeToSeconds(bufferEnd),
366 MediaTimeToSeconds(mProcessedTime), bufferEnd,
367 mProcessedTime, aTrack->GetEnd()));
368 NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
370 #endif
371 return std::min(bufferEnd, aEndBlockingDecisions);
374 namespace {
375 // Value of mCycleMarker for unvisited tracks in cycle detection.
376 const uint32_t NOT_VISITED = UINT32_MAX;
377 // Value of mCycleMarker for ordered tracks in muted cycles.
378 const uint32_t IN_MUTED_CYCLE = 1;
379 } // namespace
381 bool MediaTrackGraphImpl::AudioTrackPresent() {
382 MOZ_ASSERT(OnGraphThreadOrNotRunning());
384 bool audioTrackPresent = false;
385 for (MediaTrack* track : mTracks) {
386 if (track->AsAudioNodeTrack()) {
387 audioTrackPresent = true;
388 break;
391 if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
392 audioTrackPresent = true;
393 break;
397 // We may not have audio input device when we only have AudioNodeTracks. But
398 // if audioTrackPresent is false, we must have no input device.
399 MOZ_DIAGNOSTIC_ASSERT_IF(
400 !audioTrackPresent,
401 !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
403 return audioTrackPresent;
406 void MediaTrackGraphImpl::CheckDriver() {
407 MOZ_ASSERT(OnGraphThread());
408 // An offline graph has only one driver.
409 // Otherwise, if a switch is already pending, let that happen.
410 if (!mRealtime || Switching()) {
411 return;
414 AudioCallbackDriver* audioCallbackDriver =
415 CurrentDriver()->AsAudioCallbackDriver();
416 if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
417 for (PendingResumeOperation& op : mPendingResumeOperations) {
418 op.Apply(this);
420 mPendingResumeOperations.Clear();
423 // Note that this looks for any audio tracks, input or output, and switches
424 // to a SystemClockDriver if there are none active or no resume operations
425 // to make any active.
426 bool needAudioCallbackDriver =
427 !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
428 if (!needAudioCallbackDriver) {
429 if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
430 SwitchAtNextIteration(
431 new SystemClockDriver(this, CurrentDriver(), mSampleRate));
433 return;
436 NativeInputTrack* native =
437 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
438 CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
439 uint32_t inputChannelCount =
440 native ? AudioInputChannelCount(native->mDeviceId) : 0;
441 AudioInputType inputPreference =
442 native ? AudioInputDevicePreference(native->mDeviceId)
443 : AudioInputType::Unknown;
445 uint32_t graphOutputChannelCount = AudioOutputChannelCount();
446 if (!audioCallbackDriver) {
447 if (graphOutputChannelCount > 0) {
448 AudioCallbackDriver* driver = new AudioCallbackDriver(
449 this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
450 inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
451 SwitchAtNextIteration(driver);
453 return;
456 // Check if this graph should switch to a different number of output channels.
457 // Generally, a driver switch is explicitly made by an event (e.g., setting
458 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
459 // directly playing back via another HTMLMediaElement, the number of channels
460 // of the media determines how many channels to output, and it can change
461 // dynamically.
462 if (graphOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
463 AudioCallbackDriver* driver = new AudioCallbackDriver(
464 this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
465 inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
466 SwitchAtNextIteration(driver);
470 void MediaTrackGraphImpl::UpdateTrackOrder() {
471 if (!mTrackOrderDirty) {
472 return;
475 mTrackOrderDirty = false;
477 // The algorithm for finding cycles is based on Tim Leslie's iterative
478 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
479 // connected components (SCC) algorithm. There are variations (a) to
480 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
481 // re-run the algorithm over SCCs with breaks at DelayNodes.
483 // [1] http://www.timl.id.au/?p=327
484 // [2]
485 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
486 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
488 // There are two stacks. One for the depth-first search (DFS),
489 mozilla::LinkedList<MediaTrack> dfsStack;
490 // and another for tracks popped from the DFS stack, but still being
491 // considered as part of SCCs involving tracks on the stack.
492 mozilla::LinkedList<MediaTrack> sccStack;
494 // An index into mTracks for the next track found with no unsatisfied
495 // upstream dependencies.
496 uint32_t orderedTrackCount = 0;
498 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
499 MediaTrack* t = mTracks[i];
500 ProcessedMediaTrack* pt = t->AsProcessedTrack();
501 if (pt) {
502 // The dfsStack initially contains a list of all processed tracks in
503 // unchanged order.
504 dfsStack.insertBack(t);
505 pt->mCycleMarker = NOT_VISITED;
506 } else {
507 // SourceMediaTracks have no inputs and so can be ordered now.
508 mTracks[orderedTrackCount] = t;
509 ++orderedTrackCount;
513 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
514 // counter to label mCycleMarker on the next visited track in the DFS
515 // uniquely in the set of visited tracks that are still being considered.
517 // In this implementation, the counter descends so that the values are
518 // strictly greater than the values that mCycleMarker takes when the track
519 // has been ordered (0 or IN_MUTED_CYCLE).
521 // Each new track labelled, as the DFS searches upstream, receives a value
522 // less than those used for all other tracks being considered.
523 uint32_t nextStackMarker = NOT_VISITED - 1;
524 // Reset list of DelayNodes in cycles stored at the tail of mTracks.
525 mFirstCycleBreaker = mTracks.Length();
527 // Rearrange dfsStack order as required to DFS upstream and pop tracks
528 // in processing order to place in mTracks.
529 while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
530 const auto& inputs = pt->mInputs;
531 MOZ_ASSERT(pt->AsProcessedTrack());
532 if (pt->mCycleMarker == NOT_VISITED) {
533 // Record the position on the visited stack, so that any searches
534 // finding this track again know how much of the stack is in the cycle.
535 pt->mCycleMarker = nextStackMarker;
536 --nextStackMarker;
537 // Not-visited input tracks should be processed first.
538 // SourceMediaTracks have already been ordered.
539 for (uint32_t i = inputs.Length(); i--;) {
540 if (inputs[i]->GetSource()->IsSuspended()) {
541 continue;
543 auto input = inputs[i]->GetSource()->AsProcessedTrack();
544 if (input && input->mCycleMarker == NOT_VISITED) {
545 // It can be that this track has an input which is from a suspended
546 // AudioContext.
547 if (input->isInList()) {
548 input->remove();
549 dfsStack.insertFront(input);
553 continue;
556 // Returning from DFS. Pop from dfsStack.
557 pt->remove();
559 // cycleStackMarker keeps track of the highest marker value on any
560 // upstream track, if any, found receiving input, directly or indirectly,
561 // from the visited stack (and so from |ps|, making a cycle). In a
562 // variation from Tarjan's SCC algorithm, this does not include |ps|
563 // unless it is part of the cycle.
564 uint32_t cycleStackMarker = 0;
565 for (uint32_t i = inputs.Length(); i--;) {
566 if (inputs[i]->GetSource()->IsSuspended()) {
567 continue;
569 auto input = inputs[i]->GetSource()->AsProcessedTrack();
570 if (input) {
571 cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
575 if (cycleStackMarker <= IN_MUTED_CYCLE) {
576 // All inputs have been ordered and their stack markers have been removed.
577 // This track is not part of a cycle. It can be processed next.
578 pt->mCycleMarker = 0;
579 mTracks[orderedTrackCount] = pt;
580 ++orderedTrackCount;
581 continue;
584 // A cycle has been found. Record this track for ordering when all
585 // tracks in this SCC have been popped from the DFS stack.
586 sccStack.insertFront(pt);
588 if (cycleStackMarker > pt->mCycleMarker) {
589 // Cycles have been found that involve tracks that remain on the stack.
590 // Leave mCycleMarker indicating the most downstream (last) track on
591 // the stack known to be part of this SCC. In this way, any searches on
592 // other paths that find |ps| will know (without having to traverse from
593 // this track again) that they are part of this SCC (i.e. part of an
594 // intersecting cycle).
595 pt->mCycleMarker = cycleStackMarker;
596 continue;
599 // |pit| is the root of an SCC involving no other tracks on dfsStack, the
600 // complete SCC has been recorded, and tracks in this SCC are part of at
601 // least one cycle.
602 MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
603 // If there are DelayNodes in this SCC, then they may break the cycles.
604 bool haveDelayNode = false;
605 auto next = sccStack.getFirst();
606 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
607 // (There may be other tracks later in sccStack from other incompletely
608 // searched SCCs, involving tracks still on dfsStack.)
610 // DelayNodes in cycles must behave differently from those not in cycles,
611 // so all DelayNodes in the SCC must be identified.
612 while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
613 cycleStackMarker) {
614 auto nt = next->AsAudioNodeTrack();
615 // Get next before perhaps removing from list below.
616 next = next->getNext();
617 if (nt && nt->Engine()->AsDelayNodeEngine()) {
618 haveDelayNode = true;
619 // DelayNodes break cycles by producing their output in a
620 // preprocessing phase; they do not need to be ordered before their
621 // consumers. Order them at the tail of mTracks so that they can be
622 // handled specially. Do so now, so that DFS ignores them.
623 nt->remove();
624 nt->mCycleMarker = 0;
625 --mFirstCycleBreaker;
626 mTracks[mFirstCycleBreaker] = nt;
629 auto after_scc = next;
630 while ((next = sccStack.getFirst()) != after_scc) {
631 next->remove();
632 auto removed = static_cast<ProcessedMediaTrack*>(next);
633 if (haveDelayNode) {
634 // Return tracks to the DFS stack again (to order and detect cycles
635 // without delayNodes). Any of these tracks that are still inputs
636 // for tracks on the visited stack must be returned to the front of
637 // the stack to be ordered before their dependents. We know that none
638 // of these tracks need input from tracks on the visited stack, so
639 // they can all be searched and ordered before the current stack head
640 // is popped.
641 removed->mCycleMarker = NOT_VISITED;
642 dfsStack.insertFront(removed);
643 } else {
644 // Tracks in cycles without any DelayNodes must be muted, and so do
645 // not need input and can be ordered now. They must be ordered before
646 // their consumers so that their muted output is available.
647 removed->mCycleMarker = IN_MUTED_CYCLE;
648 mTracks[orderedTrackCount] = removed;
649 ++orderedTrackCount;
654 MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
657 TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput,
658 GraphTime aPlayedTime,
659 uint32_t aOutputChannelCount) {
660 MOZ_ASSERT(OnGraphThread());
661 MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
663 TrackTime ticksWritten = 0;
665 ticksWritten = 0;
666 MediaTrack* track = aOutput.mTrack;
667 AudioSegment* audio = track->GetData<AudioSegment>();
668 AudioSegment output;
670 TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
672 // We don't update Track->mTracksStartTime here to account for time spent
673 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
674 // the blocked period has completed. But we do need to make sure we play
675 // from the right offsets in the track buffer, even if we've already
676 // written silence for some amount of blocked time after the current time.
677 GraphTime t = aPlayedTime;
678 while (t < mStateComputedTime) {
679 bool blocked = t >= track->mStartBlocking;
680 GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
681 NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
683 // Check how many ticks of sound we can provide if we are blocked some
684 // time in the middle of this cycle.
685 TrackTime toWrite = end - t;
687 if (blocked) {
688 output.InsertNullDataAtStart(toWrite);
689 ticksWritten += toWrite;
690 LOG(LogLevel::Verbose,
691 ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
692 "%f to %f (%" PRId64 " to %" PRId64 ")",
693 this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
694 offset, offset + toWrite));
695 } else {
696 TrackTime endTicksNeeded = offset + toWrite;
697 TrackTime endTicksAvailable = audio->GetDuration();
699 if (endTicksNeeded <= endTicksAvailable) {
700 LOG(LogLevel::Verbose,
701 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
702 "(samples %" PRId64 " to %" PRId64 ")",
703 this, track, toWrite, MediaTimeToSeconds(t),
704 MediaTimeToSeconds(end), offset, endTicksNeeded));
705 output.AppendSlice(*audio, offset, endTicksNeeded);
706 ticksWritten += toWrite;
707 offset = endTicksNeeded;
708 } else {
709 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
710 // ended."); If we are at the end of the track, maybe write the
711 // remaining samples, and pad with/output silence.
712 if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
713 output.AppendSlice(*audio, offset, endTicksAvailable);
715 LOG(LogLevel::Verbose,
716 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
717 "(samples %" PRId64 " to %" PRId64 ")",
718 this, track, toWrite, MediaTimeToSeconds(t),
719 MediaTimeToSeconds(end), offset, endTicksNeeded));
720 uint32_t available = endTicksAvailable - offset;
721 ticksWritten += available;
722 toWrite -= available;
723 offset = endTicksAvailable;
725 output.AppendNullData(toWrite);
726 LOG(LogLevel::Verbose,
727 ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
728 "%f (samples %" PRId64 " to %" PRId64 ")",
729 this, track, toWrite, MediaTimeToSeconds(t),
730 MediaTimeToSeconds(end), offset, endTicksNeeded));
731 ticksWritten += toWrite;
733 output.ApplyVolume(mGlobalVolume * aOutput.mVolume);
735 t = end;
737 output.Mix(mMixer, aOutputChannelCount, mSampleRate);
739 return ticksWritten;
742 DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread(
743 CubebUtils::AudioDeviceID aID) {
744 MOZ_ASSERT(NS_IsMainThread());
745 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
746 return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
749 NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() {
750 MOZ_ASSERT(NS_IsMainThread());
751 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
752 return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
755 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
756 MOZ_ASSERT(OnGraphThread());
757 LOG(LogLevel::Debug,
758 ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
760 mDeviceInputTrackManagerGraphThread.Add(aTrack);
762 if (aTrack->AsNativeInputTrack()) {
763 // Switch Drivers since we're adding input (to input-only or full-duplex)
764 AudioCallbackDriver* driver = new AudioCallbackDriver(
765 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
766 AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID,
767 aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId));
768 LOG(LogLevel::Debug,
769 ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
770 this, driver));
771 SwitchAtNextIteration(driver);
772 } else {
773 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
774 MOZ_ASSERT(nonNative);
775 // Start non-native input right away.
776 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
777 MakeRefPtr<AudioInputSourceListener>(nonNative),
778 nonNative->GenerateSourceId(), nonNative->mDeviceId,
779 AudioInputChannelCount(nonNative->mDeviceId),
780 AudioInputDevicePreference(nonNative->mDeviceId) ==
781 AudioInputType::Voice,
782 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
786 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
787 MOZ_ASSERT(NS_IsMainThread());
788 MOZ_ASSERT(aTrack);
790 LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
791 aTrack, aTrack->mDeviceId));
793 class Message : public ControlMessage {
794 public:
795 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
796 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
797 void Run() override {
798 TRACE("MTG::OpenAudioInputImpl ControlMessage");
799 mGraph->OpenAudioInputImpl(mInputTrack);
801 MediaTrackGraphImpl* mGraph;
802 DeviceInputTrack* mInputTrack;
805 mDeviceInputTrackManagerMainThread.Add(aTrack);
807 this->AppendMessage(MakeUnique<Message>(this, aTrack));
810 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
811 MOZ_ASSERT(OnGraphThread());
813 LOG(LogLevel::Debug,
814 ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
816 if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
817 nonNative->StopAudio();
818 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
819 return;
822 MOZ_ASSERT(aTrack->AsNativeInputTrack());
824 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
826 // Switch Drivers since we're adding or removing an input (to nothing/system
827 // or output only)
828 bool audioTrackPresent = AudioTrackPresent();
830 GraphDriver* driver;
831 if (audioTrackPresent) {
832 // We still have audio output
833 LOG(LogLevel::Debug,
834 ("%p: CloseInput: output present (AudioCallback)", this));
836 driver = new AudioCallbackDriver(
837 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
838 AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID, nullptr,
839 AudioInputDevicePreference(aTrack->mDeviceId));
840 SwitchAtNextIteration(driver);
841 } else if (CurrentDriver()->AsAudioCallbackDriver()) {
842 LOG(LogLevel::Debug,
843 ("%p: CloseInput: no output present (SystemClockCallback)", this));
845 driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
846 SwitchAtNextIteration(driver);
847 } // else SystemClockDriver->SystemClockDriver, no switch
850 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
851 MOZ_ASSERT(OnGraphThreadOrNotRunning());
853 mAudioOutputs.RemoveElementsBy([aTrack](const TrackAndVolume& aOutput) {
854 return aOutput.mTrack == aTrack;
858 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
859 MOZ_ASSERT(NS_IsMainThread());
860 MOZ_ASSERT(aTrack);
862 LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
863 this, aTrack, aTrack->mDeviceId));
865 class Message : public ControlMessage {
866 public:
867 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
868 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
869 void Run() override {
870 TRACE("MTG::CloseAudioInputImpl ControlMessage");
871 mGraph->CloseAudioInputImpl(mInputTrack);
873 MediaTrackGraphImpl* mGraph;
874 DeviceInputTrack* mInputTrack;
877 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
878 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
879 // for more details.
880 mDeviceInputTrackManagerMainThread.Remove(aTrack);
882 this->AppendMessage(MakeUnique<Message>(this, aTrack));
884 if (aTrack->AsNativeInputTrack()) {
885 LOG(LogLevel::Debug,
886 ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
887 SetNewNativeInput();
891 // All AudioInput listeners get the same speaker data (at least for now).
892 void MediaTrackGraphImpl::NotifyOutputData(AudioDataValue* aBuffer,
893 size_t aFrames, TrackRate aRate,
894 uint32_t aChannels) {
895 if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
896 return;
899 #if defined(MOZ_WEBRTC)
900 for (const auto& track : mTracks) {
901 if (const auto& t = track->AsAudioProcessingTrack()) {
902 t->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
905 #endif
908 void MediaTrackGraphImpl::NotifyInputStopped() {
909 NativeInputTrack* native =
910 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
911 if (!native) {
912 return;
914 native->NotifyInputStopped(this);
917 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
918 size_t aFrames, TrackRate aRate,
919 uint32_t aChannels,
920 uint32_t aAlreadyBuffered) {
921 // Either we have an audio input device, or we just removed the audio input
922 // this iteration, and we're switching back to an output-only driver next
923 // iteration.
924 NativeInputTrack* native =
925 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
926 MOZ_ASSERT(native || Switching());
927 if (!native) {
928 return;
930 native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
931 aAlreadyBuffered);
934 void MediaTrackGraphImpl::DeviceChangedImpl() {
935 MOZ_ASSERT(OnGraphThread());
936 NativeInputTrack* native =
937 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
938 if (!native) {
939 return;
941 native->DeviceChanged(this);
944 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
945 MOZ_ASSERT(OnGraphThread());
946 mMaxOutputChannelCount = aMaxChannelCount;
949 void MediaTrackGraphImpl::DeviceChanged() {
950 // This is safe to be called from any thread: this message comes from an
951 // underlying platform API, and we don't have much guarantees. If it is not
952 // called from the main thread (and it probably will rarely be), it will post
953 // itself to the main thread, and the actual device change message will be ran
954 // and acted upon on the graph thread.
955 if (!NS_IsMainThread()) {
956 RefPtr<nsIRunnable> runnable = WrapRunnable(
957 RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
958 mMainThread->Dispatch(runnable.forget());
959 return;
962 class Message : public ControlMessage {
963 public:
964 explicit Message(MediaTrackGraph* aGraph)
965 : ControlMessage(nullptr),
966 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
967 void Run() override {
968 TRACE("MTG::DeviceChangeImpl ControlMessage");
969 mGraphImpl->DeviceChangedImpl();
971 // We know that this is valid, because the graph can't shutdown if it has
972 // messages.
973 MediaTrackGraphImpl* mGraphImpl;
976 if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
977 // This is a special case where the origin of this event cannot control the
978 // lifetime of the graph, because the graph is controling the lifetime of
979 // the AudioCallbackDriver where the event originated.
980 // We know the graph is soon going away, so there's no need to notify about
981 // this device change.
982 return;
985 // Reset the latency, it will get fetched again next time it's queried.
986 MOZ_ASSERT(NS_IsMainThread());
987 mAudioOutputLatency = 0.0;
989 // Dispatch to the bg thread to do the (potentially expensive) query of the
990 // maximum channel count, and then dispatch back to the main thread, then to
991 // the graph, with the new info.
992 RefPtr<MediaTrackGraphImpl> self = this;
993 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
994 "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
995 uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
996 self->Dispatch(NS_NewRunnableFunction(
997 "MaxChannelCountUpdateToMainThread",
998 [self{self}, maxChannelCount]() {
999 class MessageToGraph : public ControlMessage {
1000 public:
1001 explicit MessageToGraph(MediaTrackGraph* aGraph,
1002 uint32_t aMaxChannelCount)
1003 : ControlMessage(nullptr),
1004 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
1005 mMaxChannelCount(aMaxChannelCount) {}
1006 void Run() override {
1007 TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1008 mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
1010 MediaTrackGraphImpl* mGraphImpl;
1011 uint32_t mMaxChannelCount;
1013 self->AppendMessage(
1014 MakeUnique<MessageToGraph>(self, maxChannelCount));
1015 }));
1016 }));
1018 AppendMessage(MakeUnique<Message>(this));
1021 static const char* GetAudioInputTypeString(const AudioInputType& aType) {
1022 return aType == AudioInputType::Voice ? "Voice" : "Unknown";
1025 void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1026 MOZ_ASSERT(OnGraphThread());
1027 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
1028 impl->ReevaluateInputDevice(aID);
1031 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1032 MOZ_ASSERT(OnGraphThread());
1033 LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
1035 DeviceInputTrack* track =
1036 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
1037 if (!track) {
1038 LOG(LogLevel::Debug,
1039 ("%p: No DeviceInputTrack for this device. Ignore", this));
1040 return;
1043 bool needToSwitch = false;
1045 if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
1046 if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
1047 LOG(LogLevel::Debug,
1048 ("%p: %u-channel non-native input device %p (track %p) is "
1049 "re-configured to %d-channel",
1050 this, nonNative->NumberOfChannels(), aID, track,
1051 AudioInputChannelCount(aID)));
1052 needToSwitch = true;
1054 if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
1055 LOG(LogLevel::Debug,
1056 ("%p: %s-type non-native input device %p (track %p) is re-configured "
1057 "to %s-type",
1058 this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
1059 track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1060 needToSwitch = true;
1063 if (needToSwitch) {
1064 nonNative->StopAudio();
1065 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
1066 MakeRefPtr<AudioInputSourceListener>(nonNative),
1067 nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
1068 AudioInputDevicePreference(aID) == AudioInputType::Voice,
1069 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
1072 return;
1075 MOZ_ASSERT(track->AsNativeInputTrack());
1077 if (AudioCallbackDriver* audioCallbackDriver =
1078 CurrentDriver()->AsAudioCallbackDriver()) {
1079 if (audioCallbackDriver->InputChannelCount() !=
1080 AudioInputChannelCount(aID)) {
1081 LOG(LogLevel::Debug,
1082 ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1083 "re-configured to %d-channel",
1084 this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
1085 AudioInputChannelCount(aID)));
1086 needToSwitch = true;
1088 if (audioCallbackDriver->InputDevicePreference() !=
1089 AudioInputDevicePreference(aID)) {
1090 LOG(LogLevel::Debug,
1091 ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1092 "re-configured to %s-type",
1093 this,
1094 GetAudioInputTypeString(
1095 audioCallbackDriver->InputDevicePreference()),
1096 audioCallbackDriver,
1097 GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1098 needToSwitch = true;
1100 } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1101 // We're already in the process of switching to a audio callback driver,
1102 // which will happen at the next iteration.
1103 // However, maybe it's not the correct number of channels. Re-query the
1104 // correct channel amount at this time.
1105 needToSwitch = true;
1108 if (needToSwitch) {
1109 AudioCallbackDriver* newDriver = new AudioCallbackDriver(
1110 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
1111 AudioInputChannelCount(aID), mOutputDeviceID, aID,
1112 AudioInputDevicePreference(aID));
1113 SwitchAtNextIteration(newDriver);
1117 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1118 // either we're on the right thread (and calling CurrentDriver() is safe),
1119 // or we're going to fail the assert anyway, so don't cross-check
1120 // via CurrentDriver().
1121 return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
1124 bool MediaTrackGraphImpl::OnGraphThread() const {
1125 // we're on the right thread (and calling mDriver is safe),
1126 MOZ_ASSERT(mDriver);
1127 if (mGraphRunner && mGraphRunner->OnThread()) {
1128 return true;
1130 return mDriver->OnThread();
1133 bool MediaTrackGraphImpl::Destroyed() const {
1134 MOZ_ASSERT(NS_IsMainThread());
1135 return !mSelfRef;
1138 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1139 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1140 if (mRealtime) {
1141 return true;
1144 TimeStamp now = TimeStamp::Now();
1145 // For offline graphs, update now if it has been long enough since the last
1146 // update, or if it has reached the end.
1147 if ((now - mLastMainThreadUpdate).ToMilliseconds() >
1148 CurrentDriver()->IterationDuration() ||
1149 mStateComputedTime >= mEndTime) {
1150 mLastMainThreadUpdate = now;
1151 return true;
1153 return false;
1156 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
1157 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1158 mMonitor.AssertCurrentThreadOwns();
1160 // We don't want to frequently update the main thread about timing update
1161 // when we are not running in realtime.
1162 if (aFinalUpdate || ShouldUpdateMainThread()) {
1163 // Strip updates that will be obsoleted below, so as to keep the length of
1164 // mTrackUpdates sane.
1165 size_t keptUpdateCount = 0;
1166 for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
1167 MediaTrack* track = mTrackUpdates[i].mTrack;
1168 // RemoveTrackGraphThread() clears mTrack in updates for
1169 // tracks that are removed from the graph.
1170 MOZ_ASSERT(!track || track->GraphImpl() == this);
1171 if (!track || track->MainThreadNeedsUpdates()) {
1172 // Discard this update as it has either been cleared when the track
1173 // was destroyed or there will be a newer update below.
1174 continue;
1176 if (keptUpdateCount != i) {
1177 mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
1178 MOZ_ASSERT(!mTrackUpdates[i].mTrack);
1180 ++keptUpdateCount;
1182 mTrackUpdates.TruncateLength(keptUpdateCount);
1184 mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
1185 mSuspendedTracks.Length());
1186 for (MediaTrack* track : AllTracks()) {
1187 if (!track->MainThreadNeedsUpdates()) {
1188 continue;
1190 TrackUpdate* update = mTrackUpdates.AppendElement();
1191 update->mTrack = track;
1192 // No blocking to worry about here, since we've passed
1193 // UpdateCurrentTimeForTracks.
1194 update->mNextMainThreadCurrentTime =
1195 track->GraphTimeToTrackTime(mProcessedTime);
1196 update->mNextMainThreadEnded = track->mNotifiedEnded;
1198 mNextMainThreadGraphTime = mProcessedTime;
1199 if (!mPendingUpdateRunnables.IsEmpty()) {
1200 mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
1204 // If this is the final update, then a stable state event will soon be
1205 // posted just before this thread finishes, and so there is no need to also
1206 // post here.
1207 if (!aFinalUpdate &&
1208 // Don't send the message to the main thread if it's not going to have
1209 // any work to do.
1210 !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
1211 EnsureStableStateEventPosted();
1215 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
1216 if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
1217 return aTime;
1219 return RoundUpToNextAudioBlock(aTime);
1222 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
1223 uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
1224 uint64_t nextBlock = block + 1;
1225 GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1226 return nextTime;
1229 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1230 uint32_t aTrackIndex, TrackRate aSampleRate) {
1231 MOZ_ASSERT(OnGraphThread());
1232 MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
1233 "Cycle breaker is not AudioNodeTrack?");
1235 while (mProcessedTime < mStateComputedTime) {
1236 // Microtask checkpoints are in between render quanta.
1237 nsAutoMicroTask mt;
1239 GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
1240 for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
1241 auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
1242 MOZ_ASSERT(nt->AsAudioNodeTrack());
1243 nt->ProduceOutputBeforeInput(mProcessedTime);
1245 for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
1246 ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
1247 if (pt) {
1248 pt->ProcessInput(
1249 mProcessedTime, next,
1250 (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
1253 mProcessedTime = next;
1255 NS_ASSERTION(mProcessedTime == mStateComputedTime,
1256 "Something went wrong with rounding to block boundaries");
1259 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1260 UniquePtr<ControlMessageInterface> aMessage) {
1261 MOZ_ASSERT(OnGraphThread());
1263 if (mFrontMessageQueue.IsEmpty()) {
1264 mFrontMessageQueue.AppendElement();
1267 // Only one block is used for messages from the graph thread.
1268 MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1269 mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
1272 void MediaTrackGraphImpl::RunMessagesInQueue() {
1273 TRACE("MTG::RunMessagesInQueue");
1274 MOZ_ASSERT(OnGraphThread());
1275 // Calculate independent action times for each batch of messages (each
1276 // batch corresponding to an event loop task). This isolates the performance
1277 // of different scripts to some extent.
1278 for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1279 nsTArray<UniquePtr<ControlMessageInterface>>& messages =
1280 mFrontMessageQueue[i].mMessages;
1282 for (uint32_t j = 0; j < messages.Length(); ++j) {
1283 TRACE("ControlMessage::Run");
1284 messages[j]->Run();
1287 mFrontMessageQueue.Clear();
1290 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
1291 TRACE("MTG::UpdateGraph");
1292 MOZ_ASSERT(OnGraphThread());
1293 MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1294 // The next state computed time can be the same as the previous: it
1295 // means the driver would have been blocking indefinitly, but the graph has
1296 // been woken up right after having been to sleep.
1297 MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1299 CheckDriver();
1300 UpdateTrackOrder();
1302 // Always do another iteration if there are tracks waiting to resume.
1303 bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
1305 for (MediaTrack* track : mTracks) {
1306 if (SourceMediaTrack* is = track->AsSourceTrack()) {
1307 ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
1308 is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
1310 if (track->mEnded) {
1311 // The track's not suspended, and since it's ended, underruns won't
1312 // stop it playing out. So there's no blocking other than what we impose
1313 // here.
1314 GraphTime endTime = track->GetEnd() + track->mStartTime;
1315 if (endTime <= mStateComputedTime) {
1316 LOG(LogLevel::Verbose,
1317 ("%p: MediaTrack %p is blocked due to being ended", this, track));
1318 track->mStartBlocking = mStateComputedTime;
1319 } else {
1320 LOG(LogLevel::Verbose,
1321 ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1322 "time %f, end at %f)",
1323 this, track, MediaTimeToSeconds(mStateComputedTime),
1324 MediaTimeToSeconds(endTime)));
1325 // Data can't be added to a ended track, so underruns are irrelevant.
1326 MOZ_ASSERT(endTime <= aEndBlockingDecisions);
1327 track->mStartBlocking = endTime;
1329 } else {
1330 track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
1332 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1333 if (SourceMediaTrack* s = track->AsSourceTrack()) {
1334 if (s->Ended()) {
1335 continue;
1338 MutexAutoLock lock(s->mMutex);
1339 if (!s->mUpdateTrack->mPullingEnabled) {
1340 // The invariant that data must be provided is only enforced when
1341 // pulling.
1342 continue;
1345 if (track->GetEnd() <
1346 track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
1347 LOG(LogLevel::Error,
1348 ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1349 "but wasn't fed "
1350 "enough data. TrackListeners=%zu. Track-end=%f, "
1351 "Iteration-end=%f",
1352 this, track,
1353 (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
1354 track->mTrackListeners.Length(),
1355 MediaTimeToSeconds(track->GetEnd()),
1356 MediaTimeToSeconds(
1357 track->GraphTimeToTrackTime(aEndBlockingDecisions))));
1358 MOZ_DIAGNOSTIC_ASSERT(false,
1359 "A non-ended SourceMediaTrack wasn't fed "
1360 "enough data by NotifyPull");
1363 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1367 for (MediaTrack* track : mSuspendedTracks) {
1368 track->mStartBlocking = mStateComputedTime;
1371 // If the loop is woken up so soon that IterationEnd() barely advances or
1372 // if an offline graph is not currently rendering, we end up having
1373 // aEndBlockingDecisions == mStateComputedTime.
1374 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1375 // empty, Process() will not find any unblocked track and so will not
1376 // ensure another iteration. If the graph should be rendering, then ensure
1377 // another iteration to render.
1378 if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
1379 mStateComputedTime < mEndTime)) {
1380 EnsureNextIteration();
1384 void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) {
1385 TRACE("MTG::Process");
1386 MOZ_ASSERT(OnGraphThread());
1387 // Play track contents.
1388 bool allBlockedForever = true;
1389 // True when we've done ProcessInput for all processed tracks.
1390 bool doneAllProducing = false;
1391 const GraphTime oldProcessedTime = mProcessedTime;
1393 // Figure out what each track wants to do
1394 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
1395 MediaTrack* track = mTracks[i];
1396 if (!doneAllProducing) {
1397 ProcessedMediaTrack* pt = track->AsProcessedTrack();
1398 if (pt) {
1399 AudioNodeTrack* n = track->AsAudioNodeTrack();
1400 if (n) {
1401 #ifdef DEBUG
1402 // Verify that the sampling rate for all of the following tracks is
1403 // the same
1404 for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
1405 AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
1406 if (nextTrack) {
1407 MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
1408 "All AudioNodeTracks in the graph must have the same "
1409 "sampling rate");
1412 #endif
1413 // Since an AudioNodeTrack is present, go ahead and
1414 // produce audio block by block for all the rest of the tracks.
1415 ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
1416 doneAllProducing = true;
1417 } else {
1418 pt->ProcessInput(mProcessedTime, mStateComputedTime,
1419 ProcessedMediaTrack::ALLOW_END);
1420 // Assert that a live track produced enough data
1421 MOZ_ASSERT_IF(!track->mEnded,
1422 track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1423 track, mStateComputedTime));
1427 if (track->mStartBlocking > oldProcessedTime) {
1428 allBlockedForever = false;
1431 mProcessedTime = mStateComputedTime;
1433 if (aMixerReceiver) {
1434 MOZ_ASSERT(mRealtime, "If there's a mixer, this graph must be realtime");
1435 MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(),
1436 "Driver must be AudioCallbackDriver if aMixerReceiver");
1437 // Use the number of channel the driver expects: this is the number of
1438 // channel that can be output by the underlying system level audio stream.
1439 uint32_t outputChannelCount =
1440 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
1441 mMixer.StartMixing();
1442 // This is the number of frames that are written to the output buffer, for
1443 // this iteration.
1444 TrackTime ticksPlayed = 0;
1445 for (auto& t : mAudioOutputs) {
1446 TrackTime ticksPlayedForThisTrack =
1447 PlayAudio(t, oldProcessedTime, outputChannelCount);
1448 if (ticksPlayed == 0) {
1449 ticksPlayed = ticksPlayedForThisTrack;
1450 } else {
1451 MOZ_ASSERT(
1452 !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
1453 "Each track should have the same number of frames.");
1457 if (ticksPlayed == 0) {
1458 // Nothing was played, so the mixer doesn't know how many frames were
1459 // processed. We still tell it so AudioCallbackDriver knows how much has
1460 // been processed. (bug 1406027)
1461 mMixer.Mix(nullptr, outputChannelCount,
1462 mStateComputedTime - oldProcessedTime, mSampleRate);
1464 aMixerReceiver->MixerCallback(mMixer.MixedChunk(), mSampleRate);
1467 if (!allBlockedForever) {
1468 EnsureNextIteration();
1472 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1473 MOZ_ASSERT(OnGraphThread());
1474 if (mForceShutDownReceived) {
1475 for (MediaTrack* track : AllTracks()) {
1476 track->OnGraphThreadDone();
1480 MonitorAutoLock lock(mMonitor);
1481 bool finalUpdate =
1482 mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
1483 PrepareUpdatesToMainThreadState(finalUpdate);
1484 if (!finalUpdate) {
1485 SwapMessageQueues();
1486 return true;
1488 // The JSContext will not be used again.
1489 // Clear main thread access while under monitor.
1490 mJSContext = nullptr;
1492 dom::WorkletThread::DeleteCycleCollectedJSContext();
1493 // Enter shutdown mode when this iteration is completed.
1494 // No need to Destroy tracks here. The main-thread owner of each
1495 // track is responsible for calling Destroy on them.
1496 return false;
1499 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime,
1500 GraphTime aIterationEnd,
1501 MixerCallbackReceiver* aMixerReceiver)
1502 -> IterationResult {
1503 if (mGraphRunner) {
1504 return mGraphRunner->OneIteration(aStateTime, aIterationEnd,
1505 aMixerReceiver);
1508 return OneIterationImpl(aStateTime, aIterationEnd, aMixerReceiver);
1511 auto MediaTrackGraphImpl::OneIterationImpl(
1512 GraphTime aStateTime, GraphTime aIterationEnd,
1513 MixerCallbackReceiver* aMixerReceiver) -> IterationResult {
1514 TRACE("MTG::OneIterationImpl");
1516 mIterationEndTime = aIterationEnd;
1518 if (SoftRealTimeLimitReached()) {
1519 TRACE("MTG::Demoting real-time thread!");
1520 DemoteThreadFromRealTime();
1523 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1524 // thread, and so the monitor need not be held to check mLifecycleState.
1525 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1526 // graphs that have not started.
1528 // While changes occur on mainthread, this assert confirms that
1529 // this code shouldn't run if mainthread might be changing the state (to
1530 // > LIFECYCLE_RUNNING)
1532 // Ignore mutex warning: static during execution of the graph
1533 MOZ_PUSH_IGNORE_THREAD_SAFETY
1534 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
1535 MOZ_POP_THREAD_SAFETY
1537 MOZ_ASSERT(OnGraphThread());
1539 WebCore::DenormalDisabler disabler;
1541 // Process graph message from the main thread for this iteration.
1542 RunMessagesInQueue();
1544 // Process MessagePort events.
1545 // These require a single thread, which has an nsThread with an event queue.
1546 if (mGraphRunner || !mRealtime) {
1547 TRACE("MTG::MessagePort events");
1548 NS_ProcessPendingEvents(nullptr);
1551 GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
1552 UpdateGraph(stateTime);
1554 mStateComputedTime = stateTime;
1556 GraphTime oldProcessedTime = mProcessedTime;
1557 Process(aMixerReceiver);
1558 MOZ_ASSERT(mProcessedTime == stateTime);
1560 UpdateCurrentTimeForTracks(oldProcessedTime);
1562 ProcessChunkMetadata(oldProcessedTime);
1564 // Process graph messages queued from RunMessageAfterProcessing() on this
1565 // thread during the iteration.
1566 RunMessagesInQueue();
1568 if (!UpdateMainThreadState()) {
1569 if (Switching()) {
1570 // We'll never get to do this switch. Clear mNextDriver to break the
1571 // ref-cycle graph->nextDriver->currentDriver->graph.
1572 SwitchAtNextIteration(nullptr);
1574 return IterationResult::CreateStop(
1575 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1576 &MediaTrackGraphImpl::SignalMainThreadCleanup));
1579 if (Switching()) {
1580 RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
1581 return IterationResult::CreateSwitchDriver(
1582 nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
1583 "MediaTrackGraphImpl::SetCurrentDriver", this,
1584 &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
1587 return IterationResult::CreateStillProcessing();
1590 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
1591 MOZ_ASSERT(NS_IsMainThread());
1592 mMonitor.AssertCurrentThreadOwns();
1594 MediaTrack* track = aUpdate->mTrack;
1595 if (!track) return;
1596 track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1597 track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
1599 if (track->ShouldNotifyTrackEnded()) {
1600 track->NotifyMainThreadListeners();
1604 void MediaTrackGraphImpl::ForceShutDown() {
1605 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1606 LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
1608 if (mShutdownBlocker) {
1609 // Avoid waiting forever for a graph to shut down
1610 // synchronously. Reports are that some 3rd-party audio drivers
1611 // occasionally hang in shutdown (both for us and Chrome).
1612 NS_NewTimerWithCallback(
1613 getter_AddRefs(mShutdownTimer), this,
1614 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1615 nsITimer::TYPE_ONE_SHOT);
1618 class Message final : public ControlMessage {
1619 public:
1620 explicit Message(MediaTrackGraphImpl* aGraph)
1621 : ControlMessage(nullptr), mGraph(aGraph) {}
1622 void Run() override {
1623 TRACE("MTG::ForceShutdown ControlMessage");
1624 mGraph->mForceShutDownReceived = true;
1626 // The graph owns this message.
1627 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
1630 if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
1631 // If both the track and port counts are zero, the regular shutdown
1632 // sequence will progress shortly to shutdown threads and destroy the graph.
1633 AppendMessage(MakeUnique<Message>(this));
1634 InterruptJS();
1638 NS_IMETHODIMP
1639 MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
1640 MOZ_ASSERT(NS_IsMainThread());
1641 NS_ASSERTION(!mShutdownBlocker,
1642 "MediaTrackGraph took too long to shut down!");
1643 // Sigh, graph took too long to shut down. Stop blocking system
1644 // shutdown and hope all is well.
1645 RemoveShutdownBlocker();
1646 return NS_OK;
1649 static nsCString GetDocumentTitle(uint64_t aWindowID) {
1650 MOZ_ASSERT(NS_IsMainThread());
1651 nsCString title;
1652 auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID);
1653 if (!win) {
1654 return title;
1656 Document* doc = win->GetExtantDoc();
1657 if (!doc) {
1658 return title;
1660 nsAutoString titleUTF16;
1661 doc->GetTitle(titleUTF16);
1662 CopyUTF16toUTF8(titleUTF16, title);
1663 return title;
1666 NS_IMETHODIMP
1667 MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic,
1668 const char16_t* aData) {
1669 MOZ_ASSERT(NS_IsMainThread());
1670 MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0);
1671 nsCString streamName = GetDocumentTitle(mWindowID);
1672 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
1673 if (streamName.IsEmpty()) {
1674 return NS_OK;
1676 QueueControlMessageWithNoShutdown(
1677 [self = RefPtr{this}, this, streamName = std::move(streamName)] {
1678 CurrentDriver()->SetStreamName(streamName);
1680 return NS_OK;
1683 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1684 MOZ_ASSERT(NS_IsMainThread());
1685 MOZ_ASSERT(!mShutdownBlocker);
1687 class Blocker : public media::ShutdownBlocker {
1688 const RefPtr<MediaTrackGraphImpl> mGraph;
1690 public:
1691 Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
1692 : media::ShutdownBlocker(aName), mGraph(aGraph) {}
1694 NS_IMETHOD
1695 BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
1696 mGraph->ForceShutDown();
1697 return NS_OK;
1701 nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
1702 if (!barrier) {
1703 // We're already shutting down, we won't be able to add a blocker, bail.
1704 LOG(LogLevel::Error,
1705 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1706 this));
1707 return false;
1710 // Blocker names must be distinct.
1711 nsString blockerName;
1712 blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
1713 mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
1714 nsresult rv = barrier->AddBlocker(mShutdownBlocker,
1715 NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
1716 __LINE__, u"MediaTrackGraph shutdown"_ns);
1717 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
1718 return true;
1721 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1722 if (!mShutdownBlocker) {
1723 return;
1725 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
1726 mShutdownBlocker = nullptr;
1729 NS_IMETHODIMP
1730 MediaTrackGraphImpl::GetName(nsACString& aName) {
1731 aName.AssignLiteral("MediaTrackGraphImpl");
1732 return NS_OK;
1735 namespace {
1737 class MediaTrackGraphShutDownRunnable : public Runnable {
1738 public:
1739 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
1740 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
1741 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1742 // See bug 1535398.
1743 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
1744 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1745 MOZ_ASSERT(NS_IsMainThread());
1746 MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
1747 "We should know the graph thread control loop isn't running!");
1749 LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
1751 // We've asserted the graph isn't running. Use mDriver instead of
1752 // CurrentDriver to avoid thread-safety checks
1753 #if 0 // AudioCallbackDrivers are released asynchronously anyways
1754 // XXX a better test would be have setting mGraphDriverRunning make sure
1755 // any current callback has finished and block future ones -- or just
1756 // handle it all in Shutdown()!
1757 if (mGraph->mDriver->AsAudioCallbackDriver()) {
1758 MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1760 #endif
1762 for (MediaTrackGraphImpl::PendingResumeOperation& op :
1763 mGraph->mPendingResumeOperations) {
1764 op.Abort();
1767 if (mGraph->mGraphRunner) {
1768 RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
1771 RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
1773 // Release the driver now so that an AudioCallbackDriver will release its
1774 // SharedThreadPool reference. Each SharedThreadPool reference must be
1775 // released before SharedThreadPool::SpinUntilEmpty() runs on
1776 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
1777 // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1778 // which won't otherwise release its reference on the graph until
1779 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1780 mGraph->SetCurrentDriver(nullptr);
1782 // Safe to access these without the monitor since the graph isn't running.
1783 // We may be one of several graphs. Drop ticket to eventually unblock
1784 // shutdown.
1785 if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
1786 MOZ_ASSERT(
1787 false,
1788 "AudioCallbackDriver took too long to shut down and we let shutdown"
1789 " continue - freezing and leaking");
1791 // The timer fired, so we may be deeper in shutdown now. Block any
1792 // further teardown and just leak, for safety.
1793 return NS_OK;
1796 // mGraph's thread is not running so it's OK to do whatever here
1797 for (MediaTrack* track : mGraph->AllTracks()) {
1798 // Clean up all MediaSegments since we cannot release Images too
1799 // late during shutdown. Also notify listeners that they were removed
1800 // so they can clean up any gfx resources.
1801 track->RemoveAllResourcesAndListenersImpl();
1804 #ifdef DEBUG
1806 MonitorAutoLock lock(mGraph->mMonitor);
1807 MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
1809 #endif
1810 mGraph->mPendingUpdateRunnables.Clear();
1812 mGraph->RemoveShutdownBlocker();
1814 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1815 // stage, since completion of that stage requires all tracks to be freed,
1816 // which requires shutdown to proceed.
1818 if (mGraph->IsEmpty()) {
1819 // mGraph is no longer needed, so delete it.
1820 mGraph->Destroy();
1821 } else {
1822 // The graph is not empty. We must be in a forced shutdown.
1823 // Some later AppendMessage will detect that the graph has
1824 // been emptied, and delete it.
1825 NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
1826 mGraph->LifecycleStateRef() =
1827 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
1829 return NS_OK;
1832 private:
1833 RefPtr<MediaTrackGraphImpl> mGraph;
1836 class MediaTrackGraphStableStateRunnable : public Runnable {
1837 public:
1838 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
1839 bool aSourceIsMTG)
1840 : Runnable("MediaTrackGraphStableStateRunnable"),
1841 mGraph(aGraph),
1842 mSourceIsMTG(aSourceIsMTG) {}
1843 NS_IMETHOD Run() override {
1844 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1845 if (mGraph) {
1846 mGraph->RunInStableState(mSourceIsMTG);
1848 return NS_OK;
1851 private:
1852 RefPtr<MediaTrackGraphImpl> mGraph;
1853 bool mSourceIsMTG;
1857 * Control messages forwarded from main thread to graph manager thread
1859 class CreateMessage : public ControlMessage {
1860 public:
1861 explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
1862 void Run() override {
1863 TRACE("MTG::AddTrackGraphThread ControlMessage");
1864 mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
1866 void RunDuringShutdown() override {
1867 // Make sure to run this message during shutdown too, to make sure
1868 // that we balance the number of tracks registered with the graph
1869 // as they're destroyed during shutdown.
1870 Run();
1874 } // namespace
1876 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
1877 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1879 nsTArray<nsCOMPtr<nsIRunnable>> runnables;
1880 // When we're doing a forced shutdown, pending control messages may be
1881 // run on the main thread via RunDuringShutdown. Those messages must
1882 // run without the graph monitor being held. So, we collect them here.
1883 nsTArray<UniquePtr<ControlMessageInterface>>
1884 controlMessagesToRunDuringShutdown;
1887 MonitorAutoLock lock(mMonitor);
1888 if (aSourceIsMTG) {
1889 MOZ_ASSERT(mPostedRunInStableStateEvent);
1890 mPostedRunInStableStateEvent = false;
1893 // This should be kept in sync with the LifecycleState enum in
1894 // MediaTrackGraphImpl.h
1895 const char* LifecycleState_str[] = {
1896 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
1897 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1898 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1899 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
1901 if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
1902 LOG(LogLevel::Debug,
1903 ("%p: Running stable state callback. Current state: %s", this,
1904 LifecycleState_str[LifecycleStateRef()]));
1907 runnables = std::move(mUpdateRunnables);
1908 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
1909 TrackUpdate* update = &mTrackUpdates[i];
1910 if (update->mTrack) {
1911 ApplyTrackUpdate(update);
1914 mTrackUpdates.Clear();
1916 mMainThreadGraphTime = mNextMainThreadGraphTime;
1918 if (mCurrentTaskMessageQueue.IsEmpty()) {
1919 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1920 IsEmpty()) {
1921 // Complete shutdown. First, ensure that this graph is no longer used.
1922 // A new graph graph will be created if one is needed.
1923 // Asynchronously clean up old graph. We don't want to do this
1924 // synchronously because it spins the event loop waiting for threads
1925 // to shut down, and we don't want to do that in a stable state handler.
1926 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1927 LOG(LogLevel::Debug,
1928 ("%p: Sending MediaTrackGraphShutDownRunnable", this));
1929 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1930 mMainThread->Dispatch(event.forget());
1932 } else {
1933 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1934 MessageBlock* block = mBackMessageQueue.AppendElement();
1935 block->mMessages = std::move(mCurrentTaskMessageQueue);
1936 EnsureNextIteration();
1939 // If this MediaTrackGraph has entered regular (non-forced) shutdown it
1940 // is not able to process any more messages. Those messages being added to
1941 // the graph in the first place is an error.
1942 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
1943 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
1944 mForceShutDownReceived);
1947 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
1948 // Start the driver now. We couldn't start it earlier because the graph
1949 // might exit immediately on finding it has no tracks. The first message
1950 // for a new graph must create a track. Ensure that his message runs on
1951 // the first iteration.
1952 MOZ_ASSERT(MessagesQueued());
1953 SwapMessageQueues();
1955 LOG(LogLevel::Debug,
1956 ("%p: Starting a graph with a %s", this,
1957 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
1958 : "SystemClockDriver"));
1959 LifecycleStateRef() = LIFECYCLE_RUNNING;
1960 mGraphDriverRunning = true;
1961 RefPtr<GraphDriver> driver = CurrentDriver();
1962 driver->Start();
1963 // It's not safe to Shutdown() a thread from StableState, and
1964 // releasing this may shutdown a SystemClockDriver thread.
1965 // Proxy the release to outside of StableState.
1966 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
1967 driver.forget(),
1968 true); // always proxy
1971 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1972 mForceShutDownReceived) {
1973 // Defer calls to RunDuringShutdown() to happen while mMonitor is not
1974 // held.
1975 for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
1976 MessageBlock& mb = mBackMessageQueue[i];
1977 controlMessagesToRunDuringShutdown.AppendElements(
1978 std::move(mb.mMessages));
1980 mBackMessageQueue.Clear();
1981 MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
1982 // Stop MediaTrackGraph threads.
1983 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1984 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1985 mMainThread->Dispatch(event.forget());
1988 mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
1991 // Make sure we get a new current time in the next event loop task
1992 if (!aSourceIsMTG) {
1993 MOZ_ASSERT(mPostedRunInStableState);
1994 mPostedRunInStableState = false;
1997 for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
1998 controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
2001 #ifdef DEBUG
2002 mCanRunMessagesSynchronously =
2003 !mGraphDriverRunning &&
2004 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2005 #endif
2007 for (uint32_t i = 0; i < runnables.Length(); ++i) {
2008 runnables[i]->Run();
2012 void MediaTrackGraphImpl::EnsureRunInStableState() {
2013 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2015 if (mPostedRunInStableState) return;
2016 mPostedRunInStableState = true;
2017 nsCOMPtr<nsIRunnable> event =
2018 new MediaTrackGraphStableStateRunnable(this, false);
2019 nsContentUtils::RunInStableState(event.forget());
2022 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
2023 MOZ_ASSERT(OnGraphThread());
2024 mMonitor.AssertCurrentThreadOwns();
2026 if (mPostedRunInStableStateEvent) return;
2027 mPostedRunInStableStateEvent = true;
2028 nsCOMPtr<nsIRunnable> event =
2029 new MediaTrackGraphStableStateRunnable(this, true);
2030 mMainThread->Dispatch(event.forget());
2033 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2034 MOZ_ASSERT(mDriver->OnThread());
2036 MonitorAutoLock lock(mMonitor);
2037 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2038 // graphs that have not started.
2039 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
2040 LOG(LogLevel::Debug,
2041 ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2042 LifecycleStateRef() =
2043 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
2044 EnsureStableStateEventPosted();
2047 void MediaTrackGraphImpl::AppendMessage(
2048 UniquePtr<ControlMessageInterface> aMessage) {
2049 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2050 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
2052 if (!mGraphDriverRunning &&
2053 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2054 // The graph control loop is not running and main thread cleanup has
2055 // happened. From now on we can't append messages to
2056 // mCurrentTaskMessageQueue, because that will never be processed again, so
2057 // just RunDuringShutdown this message. This should only happen during
2058 // forced shutdown, or after a non-realtime graph has finished processing.
2059 #ifdef DEBUG
2060 MOZ_ASSERT(mCanRunMessagesSynchronously);
2061 mCanRunMessagesSynchronously = false;
2062 #endif
2063 aMessage->RunDuringShutdown();
2064 #ifdef DEBUG
2065 mCanRunMessagesSynchronously = true;
2066 #endif
2067 if (IsEmpty() &&
2068 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
2069 Destroy();
2071 return;
2074 mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
2075 EnsureRunInStableState();
2078 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
2079 mMainThread->Dispatch(std::move(aRunnable));
2082 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
2083 MediaSegment* aSegment)
2084 : mSampleRate(aSampleRate),
2085 mType(aType),
2086 mSegment(aSegment),
2087 mStartTime(0),
2088 mForgottenTime(0),
2089 mEnded(false),
2090 mNotifiedEnded(false),
2091 mDisabledMode(DisabledTrackMode::ENABLED),
2092 mStartBlocking(GRAPH_TIME_MAX),
2093 mSuspendedCount(0),
2094 mMainThreadCurrentTime(0),
2095 mMainThreadEnded(false),
2096 mEndedNotificationSent(false),
2097 mMainThreadDestroyed(false),
2098 mGraph(nullptr) {
2099 MOZ_COUNT_CTOR(MediaTrack);
2100 MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
2103 MediaTrack::~MediaTrack() {
2104 MOZ_COUNT_DTOR(MediaTrack);
2105 NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
2106 NS_ASSERTION(mMainThreadListeners.IsEmpty(),
2107 "All main thread listeners should have been removed");
2110 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
2111 size_t amount = 0;
2113 // Not owned:
2114 // - mGraph - Not reported here
2115 // - mConsumers - elements
2116 // Future:
2117 // - mLastPlayedVideoFrame
2118 // - mTrackListeners - elements
2120 amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2121 amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2122 amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
2124 return amount;
2127 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
2128 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
2131 void MediaTrack::IncrementSuspendCount() {
2132 ++mSuspendedCount;
2133 if (mSuspendedCount != 1 || !mGraph) {
2134 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2135 return;
2137 AssertOnGraphThreadOrNotRunning();
2138 auto* graph = GraphImpl();
2139 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2140 mConsumers[i]->Suspended();
2142 MOZ_ASSERT(graph->mTracks.Contains(this));
2143 graph->mTracks.RemoveElement(this);
2144 graph->mSuspendedTracks.AppendElement(this);
2145 graph->SetTrackOrderDirty();
2148 void MediaTrack::DecrementSuspendCount() {
2149 MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
2150 --mSuspendedCount;
2151 if (mSuspendedCount != 0 || !mGraph) {
2152 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2153 return;
2155 AssertOnGraphThreadOrNotRunning();
2156 auto* graph = GraphImpl();
2157 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2158 mConsumers[i]->Resumed();
2160 MOZ_ASSERT(graph->mSuspendedTracks.Contains(this));
2161 graph->mSuspendedTracks.RemoveElement(this);
2162 graph->mTracks.AppendElement(this);
2163 graph->SetTrackOrderDirty();
2166 void ProcessedMediaTrack::DecrementSuspendCount() {
2167 mCycleMarker = NOT_VISITED;
2168 MediaTrack::DecrementSuspendCount();
2171 MediaTrackGraphImpl* MediaTrack::GraphImpl() {
2172 return static_cast<MediaTrackGraphImpl*>(mGraph);
2175 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const {
2176 return static_cast<MediaTrackGraphImpl*>(mGraph);
2179 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
2180 MOZ_ASSERT(!mGraph, "Should only be called once");
2181 MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
2182 mGraph = aGraph;
2185 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
2186 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
2187 SetGraphImpl(graph);
2190 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
2191 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2192 aTime <= mStartBlocking,
2193 "Incorrectly ignoring blocking!");
2194 return aTime - mStartTime;
2197 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
2198 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2199 aTime + mStartTime <= mStartBlocking,
2200 "Incorrectly ignoring blocking!");
2201 return aTime + mStartTime;
2204 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
2205 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
2208 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2209 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2211 for (auto& l : mTrackListeners.Clone()) {
2212 l->NotifyRemoved(Graph());
2214 mTrackListeners.Clear();
2216 RemoveAllDirectListenersImpl();
2218 if (mSegment) {
2219 mSegment->Clear();
2223 void MediaTrack::DestroyImpl() {
2224 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2225 mConsumers[i]->Disconnect();
2227 if (mSegment) {
2228 mSegment->Clear();
2230 mGraph = nullptr;
2233 void MediaTrack::Destroy() {
2234 // Keep this track alive until we leave this method
2235 RefPtr<MediaTrack> kungFuDeathGrip = this;
2236 // Keep a reference to the graph, since Message might RunDuringShutdown()
2237 // synchronously and make GraphImpl() invalid.
2238 RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
2240 QueueControlOrShutdownMessage(
2241 [self = RefPtr{this}, this](IsInShutdown aInShutdown) {
2242 if (aInShutdown == IsInShutdown::No) {
2243 OnGraphThreadDone();
2245 TRACE("MediaTrack::Destroy ControlMessage");
2246 RemoveAllResourcesAndListenersImpl();
2247 auto* graph = GraphImpl();
2248 DestroyImpl();
2249 graph->RemoveTrackGraphThread(this);
2251 graph->RemoveTrack(this);
2252 // Message::RunDuringShutdown may have removed this track from the graph,
2253 // but our kungFuDeathGrip above will have kept this track alive if
2254 // necessary.
2255 mMainThreadDestroyed = true;
2258 TrackTime MediaTrack::GetEnd() const {
2259 return mSegment ? mSegment->GetDuration() : 0;
2262 void MediaTrack::AddAudioOutput(void* aKey) {
2263 MOZ_ASSERT(NS_IsMainThread());
2264 if (mMainThreadDestroyed) {
2265 return;
2267 LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
2268 GraphImpl()->RegisterAudioOutput(this, aKey);
2271 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
2272 float aVolume) {
2273 MOZ_ASSERT(NS_IsMainThread());
2274 for (auto& params : mAudioOutputParams) {
2275 if (params.mKey == aKey && aTrack == params.mTrack) {
2276 params.mVolume = aVolume;
2277 UpdateAudioOutput(aTrack);
2278 return;
2281 MOZ_CRASH("Audio output key not found when setting the volume.");
2284 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
2285 if (mMainThreadDestroyed) {
2286 return;
2288 GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
2291 void MediaTrack::RemoveAudioOutput(void* aKey) {
2292 MOZ_ASSERT(NS_IsMainThread());
2293 if (mMainThreadDestroyed) {
2294 return;
2296 LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
2297 GraphImpl()->UnregisterAudioOutput(this, aKey);
2300 void MediaTrackGraphImpl::RegisterAudioOutput(MediaTrack* aTrack, void* aKey) {
2301 MOZ_ASSERT(NS_IsMainThread());
2302 MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey}));
2304 mAudioOutputParams.EmplaceBack(TrackKeyAndVolume{aTrack, aKey, 1.f});
2306 UpdateAudioOutput(aTrack);
2309 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
2310 void* aKey) {
2311 MOZ_ASSERT(NS_IsMainThread());
2313 size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey});
2314 MOZ_ASSERT(index != mAudioOutputParams.NoIndex);
2315 mAudioOutputParams.UnorderedRemoveElementAt(index);
2317 UpdateAudioOutput(aTrack);
2320 void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack) {
2321 MOZ_ASSERT(NS_IsMainThread());
2322 MOZ_ASSERT(!aTrack->IsDestroyed());
2324 float volume = 0.f;
2325 bool found = false;
2326 for (const auto& params : mAudioOutputParams) {
2327 if (params.mTrack == aTrack) {
2328 volume += params.mVolume;
2329 found = true;
2333 QueueControlMessageWithNoShutdown([track = RefPtr{aTrack}, volume, found] {
2334 TRACE("MediaTrack::UpdateAudioOutput ControlMessage");
2335 MediaTrackGraphImpl* graph = track->GraphImpl();
2336 auto& audioOutputsRef = graph->mAudioOutputs;
2337 if (found) {
2338 for (auto& outputRef : audioOutputsRef) {
2339 if (outputRef.mTrack == track) {
2340 outputRef.mVolume = volume;
2341 return;
2344 audioOutputsRef.EmplaceBack(TrackAndVolume{track, volume});
2345 } else {
2346 DebugOnly<bool> removed = audioOutputsRef.RemoveElement(track);
2347 MOZ_ASSERT(removed);
2352 void MediaTrack::Suspend() {
2353 // This can happen if this method has been called asynchronously, and the
2354 // track has been destroyed since then.
2355 if (mMainThreadDestroyed) {
2356 return;
2358 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2359 TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2360 IncrementSuspendCount();
2364 void MediaTrack::Resume() {
2365 // This can happen if this method has been called asynchronously, and the
2366 // track has been destroyed since then.
2367 if (mMainThreadDestroyed) {
2368 return;
2370 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2371 TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2372 DecrementSuspendCount();
2376 void MediaTrack::AddListenerImpl(
2377 already_AddRefed<MediaTrackListener> aListener) {
2378 RefPtr<MediaTrackListener> l(aListener);
2379 mTrackListeners.AppendElement(std::move(l));
2381 PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
2382 mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
2383 Graph(), lastPrincipalHandle);
2384 if (mNotifiedEnded) {
2385 mTrackListeners.LastElement()->NotifyEnded(Graph());
2387 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
2388 mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2392 void MediaTrack::AddListener(MediaTrackListener* aListener) {
2393 MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
2394 if (mMainThreadDestroyed) {
2395 return;
2397 QueueControlMessageWithNoShutdown(
2398 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2399 TRACE("MediaTrack::AddListenerImpl ControlMessage");
2400 AddListenerImpl(listener.forget());
2404 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
2405 for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2406 if (mTrackListeners[i] == aListener) {
2407 mTrackListeners[i]->NotifyRemoved(Graph());
2408 mTrackListeners.RemoveElementAt(i);
2409 return;
2414 RefPtr<GenericPromise> MediaTrack::RemoveListener(
2415 MediaTrackListener* aListener) {
2416 MozPromiseHolder<GenericPromise> promiseHolder;
2417 RefPtr<GenericPromise> p = promiseHolder.Ensure(__func__);
2418 if (mMainThreadDestroyed) {
2419 promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
2420 return p;
2422 QueueControlOrShutdownMessage(
2423 [self = RefPtr{this}, this, listener = RefPtr{aListener},
2424 promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable {
2425 TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2426 // During shutdown we still want the listener's NotifyRemoved to be
2427 // called, since not doing that might block shutdown of other modules.
2428 RemoveListenerImpl(listener);
2429 promiseHolder.Resolve(true, __func__);
2431 return p;
2434 void MediaTrack::AddDirectListenerImpl(
2435 already_AddRefed<DirectMediaTrackListener> aListener) {
2436 AssertOnGraphThread();
2437 // Base implementation, for tracks that don't support direct track listeners.
2438 RefPtr<DirectMediaTrackListener> listener = aListener;
2439 listener->NotifyDirectListenerInstalled(
2440 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
2443 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
2444 if (mMainThreadDestroyed) {
2445 return;
2447 QueueControlMessageWithNoShutdown(
2448 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2449 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2450 AddDirectListenerImpl(listener.forget());
2454 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
2455 // Base implementation, the listener was never added so nothing to do.
2458 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
2459 if (mMainThreadDestroyed) {
2460 return;
2462 QueueControlOrShutdownMessage(
2463 [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) {
2464 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2465 // During shutdown we still want the listener's
2466 // NotifyDirectListenerUninstalled to be called, since not doing that
2467 // might block shutdown of other modules.
2468 RemoveDirectListenerImpl(listener);
2472 void MediaTrack::RunAfterPendingUpdates(
2473 already_AddRefed<nsIRunnable> aRunnable) {
2474 MOZ_ASSERT(NS_IsMainThread());
2475 if (mMainThreadDestroyed) {
2476 return;
2478 QueueControlOrShutdownMessage(
2479 [self = RefPtr{this}, this,
2480 runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable {
2481 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2482 if (aInShutdown == IsInShutdown::No) {
2483 Graph()->DispatchToMainThreadStableState(runnable.forget());
2484 } else {
2485 // Don't run mRunnable now as it may call AppendMessage() which would
2486 // assume that there are no remaining
2487 // controlMessagesToRunDuringShutdown.
2488 MOZ_ASSERT(NS_IsMainThread());
2489 GraphImpl()->Dispatch(runnable.forget());
2494 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2495 AssertOnGraphThread();
2496 MOZ_DIAGNOSTIC_ASSERT(
2497 aMode == DisabledTrackMode::ENABLED ||
2498 mDisabledMode == DisabledTrackMode::ENABLED,
2499 "Changing disabled track mode for a track is not allowed");
2500 DisabledTrackMode oldMode = CombinedDisabledMode();
2501 mDisabledMode = aMode;
2502 NotifyIfDisabledModeChangedFrom(oldMode);
2505 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
2506 if (mMainThreadDestroyed) {
2507 return;
2509 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() {
2510 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2511 SetDisabledTrackModeImpl(aMode);
2515 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
2516 MediaSegment* aRawSegment) {
2517 AssertOnGraphThread();
2518 mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment);
2521 void MediaTrack::AddMainThreadListener(
2522 MainThreadMediaTrackListener* aListener) {
2523 MOZ_ASSERT(NS_IsMainThread());
2524 MOZ_ASSERT(aListener);
2525 MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2527 mMainThreadListeners.AppendElement(aListener);
2529 // If it is not yet time to send the notification, then exit here.
2530 if (!mEndedNotificationSent) {
2531 return;
2534 class NotifyRunnable final : public Runnable {
2535 public:
2536 explicit NotifyRunnable(MediaTrack* aTrack)
2537 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
2539 NS_IMETHOD Run() override {
2540 TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2541 MOZ_ASSERT(NS_IsMainThread());
2542 mTrack->NotifyMainThreadListeners();
2543 return NS_OK;
2546 private:
2547 ~NotifyRunnable() = default;
2549 RefPtr<MediaTrack> mTrack;
2552 nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2553 GraphImpl()->Dispatch(runnable.forget());
2556 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2557 GraphTime aBlockedTime) {
2558 mStartTime += aBlockedTime;
2560 if (!mSegment) {
2561 // No data to be forgotten.
2562 return;
2565 TrackTime time = aCurrentTime - mStartTime;
2566 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2567 // spend too much time pruning segments.
2568 const TrackTime minChunkSize = mSampleRate * 50 / 1000;
2569 if (time < mForgottenTime + minChunkSize) {
2570 return;
2573 mForgottenTime = std::min(GetEnd() - 1, time);
2574 mSegment->ForgetUpTo(mForgottenTime);
2577 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
2578 DisabledTrackMode mode = CombinedDisabledMode();
2579 if (aOldMode == mode) {
2580 return;
2583 for (const auto& listener : mTrackListeners) {
2584 listener->NotifyEnabledStateChanged(
2585 Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
2588 for (const auto& c : mConsumers) {
2589 if (c->GetDestination()) {
2590 c->GetDestination()->OnInputDisabledModeChanged(mode);
2595 void MediaTrack::QueueMessage(UniquePtr<ControlMessageInterface> aMessage) {
2596 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
2597 MOZ_RELEASE_ASSERT(!IsDestroyed());
2598 GraphImpl()->AppendMessage(std::move(aMessage));
2601 void MediaTrack::RunMessageAfterProcessing(
2602 UniquePtr<ControlMessageInterface> aMessage) {
2603 AssertOnGraphThread();
2604 GraphImpl()->RunMessageAfterProcessing(std::move(aMessage));
2607 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
2608 TrackRate aSampleRate)
2609 : MediaTrack(aSampleRate, aType,
2610 aType == MediaSegment::AUDIO
2611 ? static_cast<MediaSegment*>(new AudioSegment())
2612 : static_cast<MediaSegment*>(new VideoSegment())),
2613 mMutex("mozilla::media::SourceMediaTrack") {
2614 mUpdateTrack = MakeUnique<TrackData>();
2615 mUpdateTrack->mInputRate = aSampleRate;
2616 mUpdateTrack->mResamplerChannelCount = 0;
2617 mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
2618 mUpdateTrack->mEnded = false;
2619 mUpdateTrack->mPullingEnabled = false;
2620 mUpdateTrack->mGraphThreadDone = false;
2623 void SourceMediaTrack::DestroyImpl() {
2624 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2625 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2626 // Disconnect before we come under mMutex's lock since it can call back
2627 // through RemoveDirectListenerImpl() and deadlock.
2628 mConsumers[i]->Disconnect();
2631 // Hold mMutex while mGraph is reset so that other threads holding mMutex
2632 // can null-check know that the graph will not destroyed.
2633 MutexAutoLock lock(mMutex);
2634 mUpdateTrack = nullptr;
2635 MediaTrack::DestroyImpl();
2638 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
2639 class Message : public ControlMessage {
2640 public:
2641 Message(SourceMediaTrack* aTrack, bool aEnabled)
2642 : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
2643 void Run() override {
2644 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2645 MutexAutoLock lock(mTrack->mMutex);
2646 if (!mTrack->mUpdateTrack) {
2647 // We can't enable pulling for a track that has ended. We ignore
2648 // this if we're disabling pulling, since shutdown sequences are
2649 // complex. If there's truly an issue we'll have issues enabling anyway.
2650 MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
2651 return;
2653 MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
2654 "Pulling is not allowed for video");
2655 mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
2657 SourceMediaTrack* mTrack;
2658 bool mEnabled;
2660 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
2663 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
2664 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2665 TrackTime t;
2666 TrackTime current;
2668 if (mEnded) {
2669 return false;
2671 MutexAutoLock lock(mMutex);
2672 if (mUpdateTrack->mEnded) {
2673 return false;
2675 if (!mUpdateTrack->mPullingEnabled) {
2676 return false;
2678 // Compute how much track time we'll need assuming we don't block
2679 // the track at all.
2680 t = GraphTimeToTrackTime(aDesiredUpToTime);
2681 current = GetEnd() + mUpdateTrack->mData->GetDuration();
2683 if (t <= current) {
2684 return false;
2686 LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2687 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
2688 GraphImpl()->MediaTimeToSeconds(current)));
2689 for (auto& l : mTrackListeners) {
2690 l->NotifyPull(Graph(), current, t);
2692 return true;
2696 * This moves chunks from aIn to aOut. For audio this is simple. For video
2697 * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2699 * We also handle "resetters" from captured media elements. This type of source
2700 * pushes future frames into the track, and should it need to remove some, e.g.,
2701 * because of a seek or pause, it tells us by letting time go backwards. Without
2702 * this, tracks would be live for too long after a seek or pause.
2704 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
2705 MediaSegment* aOut, TrackTime aCurrentTime,
2706 TrackTime aDesiredUpToTime)
2707 MOZ_REQUIRES(aTrack->GetMutex()) {
2708 MOZ_ASSERT(aIn->GetType() == aOut->GetType());
2709 MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
2710 MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
2711 if (aIn->GetType() == MediaSegment::AUDIO) {
2712 AudioSegment* in = static_cast<AudioSegment*>(aIn);
2713 AudioSegment* out = static_cast<AudioSegment*>(aOut);
2714 TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
2715 TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
2717 out->AppendSlice(*in, 0, end);
2718 in->RemoveLeading(end);
2720 aTrack->GetMutex().AssertCurrentThreadOwns();
2721 out->ApplyVolume(aTrack->GetVolumeLocked());
2722 } else {
2723 VideoSegment* in = static_cast<VideoSegment*>(aIn);
2724 VideoSegment* out = static_cast<VideoSegment*>(aOut);
2725 for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
2726 MOZ_ASSERT(!c->mTimeStamp.IsNull());
2727 VideoChunk* last = out->GetLastChunk();
2728 if (!last || last->mTimeStamp.IsNull()) {
2729 // This is the first frame, or the last frame pushed to `out` has been
2730 // all consumed. Just append and we deal with its duration later.
2731 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2732 c->mFrame.GetIntrinsicSize(),
2733 c->mFrame.GetPrincipalHandle(),
2734 c->mFrame.GetForceBlack(), c->mTimeStamp);
2735 if (c->GetDuration() > 0) {
2736 out->ExtendLastFrameBy(c->GetDuration());
2738 continue;
2741 // We now know when this frame starts, aka when the last frame ends.
2743 if (c->mTimeStamp < last->mTimeStamp) {
2744 // Time is going backwards. This is a resetting frame from
2745 // DecodedStream. Clear everything up to currentTime.
2746 out->Clear();
2747 out->AppendNullData(aCurrentTime);
2750 // Append the current frame (will have duration 0).
2751 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2752 c->mFrame.GetIntrinsicSize(),
2753 c->mFrame.GetPrincipalHandle(),
2754 c->mFrame.GetForceBlack(), c->mTimeStamp);
2755 if (c->GetDuration() > 0) {
2756 out->ExtendLastFrameBy(c->GetDuration());
2759 if (out->GetDuration() < aDesiredUpToTime) {
2760 out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
2762 in->Clear();
2763 MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
2767 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
2768 GraphTime aDesiredUpToTime) {
2769 MutexAutoLock lock(mMutex);
2771 if (!mUpdateTrack) {
2772 MOZ_ASSERT(mEnded);
2773 return;
2776 TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
2778 ApplyTrackDisabling(mUpdateTrack->mData.get());
2780 if (!mUpdateTrack->mData->IsEmpty()) {
2781 for (const auto& l : mTrackListeners) {
2782 l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
2785 TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
2786 TrackTime endTime = trackDesiredUpToTime;
2787 if (mUpdateTrack->mEnded) {
2788 endTime = std::min(trackDesiredUpToTime,
2789 GetEnd() + mUpdateTrack->mData->GetDuration());
2791 LOG(LogLevel::Verbose,
2792 ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
2793 GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
2794 MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
2795 trackCurrentTime, endTime);
2796 if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
2797 mEnded = true;
2798 mUpdateTrack = nullptr;
2802 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
2803 mMutex.AssertCurrentThreadOwns();
2804 if (aSegment->GetType() != MediaSegment::AUDIO ||
2805 mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
2806 return;
2808 AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
2809 segment->ResampleChunks(mUpdateTrack->mResampler,
2810 &mUpdateTrack->mResamplerChannelCount,
2811 mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
2814 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
2815 GraphTime aCurrentTime, GraphTime aBlockedTime) {
2816 MutexAutoLock lock(mMutex);
2817 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
2820 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
2821 MutexAutoLock lock(mMutex);
2822 if (!mUpdateTrack) {
2823 return;
2825 MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
2826 // Set the new input rate and reset the resampler.
2827 mUpdateTrack->mInputRate = aRate;
2828 mUpdateTrack->mResampler.own(nullptr);
2829 mUpdateTrack->mResamplerChannelCount = 0;
2832 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
2833 MediaSegment* aRawSegment) {
2834 MutexAutoLock lock(mMutex);
2835 MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
2836 TrackTime appended = 0;
2837 if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
2838 aSegment->Clear();
2839 return appended;
2842 // Data goes into mData, and on the next iteration of the MTG moves
2843 // into the track's segment after NotifyQueuedTrackChanges(). This adds
2844 // 0-10ms of delay before data gets to direct listeners.
2845 // Indirect listeners (via subsequent TrackUnion nodes) are synced to
2846 // playout time, and so can be delayed by buffering.
2848 // Apply track disabling before notifying any consumers directly
2849 // or inserting into the graph
2850 mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment);
2852 ResampleAudioToGraphSampleRate(aSegment);
2854 // Must notify first, since AppendFrom() will empty out aSegment
2855 NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
2856 appended = aSegment->GetDuration();
2857 mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead
2859 auto graph = GraphImpl();
2860 MonitorAutoLock lock(graph->GetMonitor());
2861 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
2862 graph->EnsureNextIteration();
2866 return appended;
2869 TrackTime SourceMediaTrack::ClearFutureData() {
2870 MutexAutoLock lock(mMutex);
2871 auto graph = GraphImpl();
2872 if (!mUpdateTrack || !graph) {
2873 return 0;
2876 TrackTime duration = mUpdateTrack->mData->GetDuration();
2877 mUpdateTrack->mData->Clear();
2878 return duration;
2881 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
2882 mMutex.AssertCurrentThreadOwns();
2884 for (const auto& l : mDirectTrackListeners) {
2885 TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of
2886 // the internal buffer
2887 l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
2888 *aSegment);
2892 void SourceMediaTrack::AddDirectListenerImpl(
2893 already_AddRefed<DirectMediaTrackListener> aListener) {
2894 AssertOnGraphThread();
2895 MutexAutoLock lock(mMutex);
2897 RefPtr<DirectMediaTrackListener> listener = aListener;
2898 LOG(LogLevel::Debug,
2899 ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
2900 listener.get(), this));
2902 MOZ_ASSERT(mType == MediaSegment::VIDEO);
2903 for (const auto& l : mDirectTrackListeners) {
2904 if (l == listener) {
2905 listener->NotifyDirectListenerInstalled(
2906 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
2907 return;
2911 mDirectTrackListeners.AppendElement(listener);
2913 LOG(LogLevel::Debug,
2914 ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
2915 listener->NotifyDirectListenerInstalled(
2916 DirectMediaTrackListener::InstallationResult::SUCCESS);
2918 if (mDisabledMode != DisabledTrackMode::ENABLED) {
2919 listener->IncreaseDisabled(mDisabledMode);
2922 if (mEnded) {
2923 return;
2926 // Pass buffered data to the listener
2927 VideoSegment bufferedData;
2928 size_t videoFrames = 0;
2929 VideoSegment& segment = *GetData<VideoSegment>();
2930 for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
2931 iter.Next()) {
2932 if (iter->mTimeStamp.IsNull()) {
2933 // No timestamp means this is only for the graph's internal book-keeping,
2934 // denoting a late start of the track.
2935 continue;
2937 ++videoFrames;
2938 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2939 iter->mFrame.GetIntrinsicSize(),
2940 iter->mFrame.GetPrincipalHandle(),
2941 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2944 VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
2945 for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
2946 iter.Next()) {
2947 ++videoFrames;
2948 MOZ_ASSERT(!iter->mTimeStamp.IsNull());
2949 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2950 iter->mFrame.GetIntrinsicSize(),
2951 iter->mFrame.GetPrincipalHandle(),
2952 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2955 LOG(LogLevel::Info,
2956 ("%p: Notifying direct listener %p of %zu video frames and duration "
2957 "%" PRId64,
2958 GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
2959 listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
2962 void SourceMediaTrack::RemoveDirectListenerImpl(
2963 DirectMediaTrackListener* aListener) {
2964 mGraph->AssertOnGraphThreadOrNotRunning();
2965 MutexAutoLock lock(mMutex);
2966 for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
2967 const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
2968 if (l == aListener) {
2969 if (mDisabledMode != DisabledTrackMode::ENABLED) {
2970 aListener->DecreaseDisabled(mDisabledMode);
2972 aListener->NotifyDirectListenerUninstalled();
2973 mDirectTrackListeners.RemoveElementAt(i);
2978 void SourceMediaTrack::End() {
2979 MutexAutoLock lock(mMutex);
2980 if (!mUpdateTrack) {
2981 // Already ended
2982 return;
2984 mUpdateTrack->mEnded = true;
2985 if (auto graph = GraphImpl()) {
2986 MonitorAutoLock lock(graph->GetMonitor());
2987 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
2988 graph->EnsureNextIteration();
2993 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2994 AssertOnGraphThread();
2996 MutexAutoLock lock(mMutex);
2997 const DisabledTrackMode oldMode = mDirectDisabledMode;
2998 const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
2999 const bool enabled = aMode == DisabledTrackMode::ENABLED;
3000 mDirectDisabledMode = aMode;
3001 for (const auto& l : mDirectTrackListeners) {
3002 if (!oldEnabled && enabled) {
3003 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3004 "direct listener enabled",
3005 GraphImpl(), this));
3006 l->DecreaseDisabled(oldMode);
3007 } else if (oldEnabled && !enabled) {
3008 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3009 "direct listener disabled",
3010 GraphImpl(), this));
3011 l->IncreaseDisabled(aMode);
3015 MediaTrack::SetDisabledTrackModeImpl(aMode);
3018 uint32_t SourceMediaTrack::NumberOfChannels() const {
3019 AudioSegment* audio = GetData<AudioSegment>();
3020 MOZ_DIAGNOSTIC_ASSERT(audio);
3021 if (!audio) {
3022 return 0;
3024 return audio->MaxChannelCount();
3027 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3028 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3029 MutexAutoLock lock(mMutex);
3031 for (auto& l : mDirectTrackListeners.Clone()) {
3032 l->NotifyDirectListenerUninstalled();
3034 mDirectTrackListeners.Clear();
3037 void SourceMediaTrack::SetVolume(float aVolume) {
3038 MutexAutoLock lock(mMutex);
3039 mVolume = aVolume;
3042 float SourceMediaTrack::GetVolumeLocked() {
3043 mMutex.AssertCurrentThreadOwns();
3044 return mVolume;
3047 SourceMediaTrack::~SourceMediaTrack() = default;
3049 void MediaInputPort::Init() {
3050 mGraph->AssertOnGraphThreadOrNotRunning();
3051 LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph,
3052 this, mSource, mDest));
3053 // Only connect the port if it wasn't disconnected on allocation.
3054 if (mSource) {
3055 mSource->AddConsumer(this);
3056 mDest->AddInput(this);
3058 // mPortCount decremented via MediaInputPort::Destroy's message
3059 ++mGraph->mPortCount;
3062 void MediaInputPort::Disconnect() {
3063 mGraph->AssertOnGraphThreadOrNotRunning();
3064 NS_ASSERTION(!mSource == !mDest,
3065 "mSource and mDest must either both be null or both non-null");
3067 if (!mSource) {
3068 return;
3071 mSource->RemoveConsumer(this);
3072 mDest->RemoveInput(this);
3073 mSource = nullptr;
3074 mDest = nullptr;
3076 mGraph->SetTrackOrderDirty();
3079 MediaTrack* MediaInputPort::GetSource() const {
3080 mGraph->AssertOnGraphThreadOrNotRunning();
3081 return mSource;
3084 ProcessedMediaTrack* MediaInputPort::GetDestination() const {
3085 mGraph->AssertOnGraphThreadOrNotRunning();
3086 return mDest;
3089 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
3090 MediaInputPort const* aPort, GraphTime aTime) {
3091 InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
3092 if (!aPort) {
3093 result.mStart = aTime;
3094 result.mInputIsBlocked = true;
3095 return result;
3097 aPort->mGraph->AssertOnGraphThreadOrNotRunning();
3098 if (aTime >= aPort->mDest->mStartBlocking) {
3099 return result;
3101 result.mStart = aTime;
3102 result.mEnd = aPort->mDest->mStartBlocking;
3103 result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
3104 if (!result.mInputIsBlocked) {
3105 result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
3107 return result;
3110 void MediaInputPort::Suspended() {
3111 mGraph->AssertOnGraphThreadOrNotRunning();
3112 mDest->InputSuspended(this);
3115 void MediaInputPort::Resumed() {
3116 mGraph->AssertOnGraphThreadOrNotRunning();
3117 mDest->InputResumed(this);
3120 void MediaInputPort::Destroy() {
3121 class Message : public ControlMessage {
3122 public:
3123 explicit Message(MediaInputPort* aPort)
3124 : ControlMessage(nullptr), mPort(aPort) {}
3125 void Run() override {
3126 TRACE("MediaInputPort::Destroy ControlMessage");
3127 mPort->Disconnect();
3128 --mPort->GraphImpl()->mPortCount;
3129 mPort->SetGraphImpl(nullptr);
3130 NS_RELEASE(mPort);
3132 void RunDuringShutdown() override { Run(); }
3133 MediaInputPort* mPort;
3135 // Keep a reference to the graph, since Message might RunDuringShutdown()
3136 // synchronously and make GraphImpl() invalid.
3137 RefPtr<MediaTrackGraphImpl> graph = mGraph;
3138 graph->AppendMessage(MakeUnique<Message>(this));
3139 --graph->mMainThreadPortCount;
3142 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
3143 mGraph->AssertOnGraphThreadOrNotRunning();
3144 return mGraph;
3147 MediaTrackGraph* MediaInputPort::Graph() const {
3148 mGraph->AssertOnGraphThreadOrNotRunning();
3149 return mGraph;
3152 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
3153 MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3154 DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
3155 MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
3156 mGraph = aGraph;
3159 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
3160 MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
3161 // This method creates two references to the MediaInputPort: one for
3162 // the main thread, and one for the MediaTrackGraph.
3163 class Message : public ControlMessage {
3164 public:
3165 explicit Message(MediaInputPort* aPort)
3166 : ControlMessage(aPort->mDest), mPort(aPort) {}
3167 void Run() override {
3168 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3169 mPort->Init();
3170 // The graph holds its reference implicitly
3171 mPort->GraphImpl()->SetTrackOrderDirty();
3172 Unused << mPort.forget();
3174 void RunDuringShutdown() override { Run(); }
3175 RefPtr<MediaInputPort> mPort;
3178 MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
3179 RefPtr<MediaInputPort> port;
3180 if (aTrack->IsDestroyed()) {
3181 // Create a port that's disconnected, which is what it'd be after its source
3182 // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3183 // this later is fine.
3184 port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
3185 aOutputNumber);
3186 } else {
3187 MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
3188 port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
3189 aOutputNumber);
3191 ++GraphImpl()->mMainThreadPortCount;
3192 GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3193 return port.forget();
3196 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
3197 class Message : public ControlMessage {
3198 public:
3199 Message(ProcessedMediaTrack* aTrack, bool aAutoend)
3200 : ControlMessage(aTrack), mAutoend(aAutoend) {}
3201 void Run() override {
3202 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3203 static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
3205 bool mAutoend;
3207 if (mMainThreadDestroyed) {
3208 return;
3210 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
3213 void ProcessedMediaTrack::DestroyImpl() {
3214 for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3215 mInputs[i]->Disconnect();
3218 for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3219 mSuspendedInputs[i]->Disconnect();
3222 MediaTrack::DestroyImpl();
3223 // The track order is only important if there are connections, in which
3224 // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3225 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3226 // SetTrackOrderDirty(), for other reasons.
3229 MediaTrackGraphImpl::MediaTrackGraphImpl(
3230 GraphDriverType aDriverRequested, GraphRunType aRunTypeRequested,
3231 uint64_t aWindowID, TrackRate aSampleRate, uint32_t aChannelCount,
3232 CubebUtils::AudioDeviceID aOutputDeviceID,
3233 nsISerialEventTarget* aMainThread)
3234 : MediaTrackGraph(aSampleRate),
3235 mGraphRunner(aRunTypeRequested == SINGLE_THREAD
3236 ? GraphRunner::Create(this)
3237 : already_AddRefed<GraphRunner>(nullptr)),
3238 mFirstCycleBreaker(0)
3239 // An offline graph is not initially processing.
3241 mEndTime(aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX),
3242 mPortCount(0),
3243 mWindowID(aWindowID),
3244 mOutputDeviceID(aOutputDeviceID),
3245 mMonitor("MediaTrackGraphImpl"),
3246 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
3247 mPostedRunInStableStateEvent(false),
3248 mGraphDriverRunning(false),
3249 mPostedRunInStableState(false),
3250 mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER),
3251 mTrackOrderDirty(false),
3252 mMainThread(aMainThread),
3253 mSelfRef(this),
3254 mGlobalVolume(CubebUtils::GetVolumeScale())
3255 #ifdef DEBUG
3257 mCanRunMessagesSynchronously(false)
3258 #endif
3260 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3261 mAudioOutputLatency(0.0),
3262 mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
3263 bool failedToGetShutdownBlocker = false;
3264 if (!IsNonRealtime()) {
3265 failedToGetShutdownBlocker = !AddShutdownBlocker();
3268 if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
3269 failedToGetShutdownBlocker) {
3270 // At least one of the following happened
3271 // - Failed to create thread.
3272 // - Failed to install a shutdown blocker when one is needed.
3273 // Because we have a fail state, jump to last phase of the lifecycle.
3274 mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
3275 RemoveShutdownBlocker(); // No-op if blocker wasn't added.
3276 #ifdef DEBUG
3277 mCanRunMessagesSynchronously = true;
3278 #endif
3279 return;
3281 if (mRealtime) {
3282 if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3283 // Always start with zero input channels, and no particular preferences
3284 // for the input channel.
3285 mDriver = new AudioCallbackDriver(this, nullptr, mSampleRate,
3286 aChannelCount, 0, mOutputDeviceID,
3287 nullptr, AudioInputType::Unknown);
3288 } else {
3289 mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
3291 nsCString streamName = GetDocumentTitle(aWindowID);
3292 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
3293 mDriver->SetStreamName(streamName);
3294 } else {
3295 mDriver =
3296 new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
3299 mLastMainThreadUpdate = TimeStamp::Now();
3301 RegisterWeakAsyncMemoryReporter(this);
3304 #ifdef DEBUG
3305 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
3306 return aDriver->OnThread() ||
3307 (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
3309 #endif
3311 void MediaTrackGraphImpl::Destroy() {
3312 // First unregister from memory reporting.
3313 UnregisterWeakMemoryReporter(this);
3315 // Clear the self reference which will destroy this instance if all
3316 // associated GraphDrivers are destroyed.
3317 mSelfRef = nullptr;
3320 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3321 // GTests can create a graph without a window.
3322 /* static */
3323 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
3324 uint64_t aWindowID, TrackRate aSampleRate,
3325 CubebUtils::AudioDeviceID aOutputDeviceID) {
3326 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3327 MOZ_ASSERT(aSampleRate > 0);
3329 GraphHashSet::Ptr p =
3330 Graphs()->lookup({aWindowID, aSampleRate, aOutputDeviceID});
3331 return p ? *p : nullptr;
3334 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3335 // window is a real inner Window, not a WindowProxy.
3336 /* static */
3337 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
3338 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
3339 CubebUtils::AudioDeviceID aOutputDeviceID) {
3340 TrackRate sampleRate =
3341 aSampleRate ? aSampleRate
3342 : CubebUtils::PreferredSampleRate(
3343 aWindow->AsGlobal()->ShouldResistFingerprinting(
3344 RFPTarget::AudioSampleRate));
3345 return MediaTrackGraphImpl::GetInstanceIfExists(aWindow->WindowID(),
3346 sampleRate, aOutputDeviceID);
3349 /* static */
3350 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
3351 GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
3352 TrackRate aSampleRate, CubebUtils::AudioDeviceID aOutputDeviceID,
3353 nsISerialEventTarget* aMainThread) {
3354 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3355 MOZ_ASSERT(aSampleRate > 0);
3356 MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER,
3357 "Use CreateNonRealtimeInstance() for offline graphs");
3359 GraphHashSet* graphs = Graphs();
3360 GraphHashSet::AddPtr addPtr =
3361 graphs->lookupForAdd({aWindowID, aSampleRate, aOutputDeviceID});
3362 if (addPtr) { // graph already exists
3363 return *addPtr;
3366 GraphRunType runType = DIRECT_DRIVER;
3367 if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) {
3368 runType = SINGLE_THREAD;
3371 // In a real time graph, the number of output channels is determined by
3372 // the underlying number of channel of the default audio output device, and
3373 // capped to 8.
3374 uint32_t channelCount =
3375 std::min<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
3376 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3377 aGraphDriverRequested, runType, aWindowID, aSampleRate, channelCount,
3378 aOutputDeviceID, aMainThread);
3379 MOZ_ALWAYS_TRUE(graphs->add(addPtr, graph));
3381 nsCOMPtr<nsIObserverService> observerService =
3382 mozilla::services::GetObserverService();
3383 if (observerService) {
3384 observerService->AddObserver(graph, "document-title-changed", false);
3387 LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64,
3388 graph, aWindowID));
3390 return graph;
3393 /* static */
3394 MediaTrackGraph* MediaTrackGraph::GetInstance(
3395 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
3396 TrackRate aSampleRate, CubebUtils::AudioDeviceID aOutputDeviceID) {
3397 TrackRate sampleRate =
3398 aSampleRate ? aSampleRate
3399 : CubebUtils::PreferredSampleRate(
3400 aWindow->AsGlobal()->ShouldResistFingerprinting(
3401 RFPTarget::AudioSampleRate));
3402 return MediaTrackGraphImpl::GetInstance(
3403 aGraphDriverRequested, aWindow->WindowID(), sampleRate, aOutputDeviceID,
3404 GetMainThreadSerialEventTarget());
3407 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
3408 TrackRate aSampleRate) {
3409 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3411 nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
3412 // Offline graphs have 0 output channel count: they write the output to a
3413 // buffer, not an audio output track.
3414 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3415 OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0, aSampleRate, 0,
3416 DEFAULT_OUTPUT_DEVICE, mainThread);
3418 LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
3420 return graph;
3423 void MediaTrackGraph::ForceShutDown() {
3424 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3426 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3428 graph->ForceShutDown();
3431 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver,
3432 nsIThreadObserver, nsITimerCallback, nsINamed)
3434 NS_IMETHODIMP
3435 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3436 nsISupports* aData, bool aAnonymize) {
3437 MOZ_ASSERT(NS_IsMainThread());
3438 if (mMainThreadTrackCount == 0) {
3439 // No tracks to report.
3440 FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3441 return NS_OK;
3444 class Message final : public ControlMessage {
3445 public:
3446 Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
3447 nsISupports* aHandlerData)
3448 : ControlMessage(nullptr),
3449 mGraph(aGraph),
3450 mHandleReport(aHandleReport),
3451 mHandlerData(aHandlerData) {}
3452 void Run() override {
3453 TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3454 mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3455 mHandlerData.forget());
3457 void RunDuringShutdown() override {
3458 // Run this message during shutdown too, so that endReports is called.
3459 Run();
3461 MediaTrackGraphImpl* mGraph;
3462 // nsMemoryReporterManager keeps the callback and data alive only if it
3463 // does not time out.
3464 nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3465 nsCOMPtr<nsISupports> mHandlerData;
3468 AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3470 return NS_OK;
3473 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3474 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3475 already_AddRefed<nsISupports> aHandlerData) {
3476 class FinishCollectRunnable final : public Runnable {
3477 public:
3478 explicit FinishCollectRunnable(
3479 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3480 already_AddRefed<nsISupports> aHandlerData)
3481 : mozilla::Runnable("FinishCollectRunnable"),
3482 mHandleReport(aHandleReport),
3483 mHandlerData(aHandlerData) {}
3485 NS_IMETHOD Run() override {
3486 TRACE("MTG::FinishCollectReports ControlMessage");
3487 MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3488 std::move(mAudioTrackSizes));
3489 return NS_OK;
3492 nsTArray<AudioNodeSizes> mAudioTrackSizes;
3494 private:
3495 ~FinishCollectRunnable() = default;
3497 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3498 // constructor modifies the ref-count, which cannot be done off main
3499 // thread.
3500 RefPtr<nsIHandleReportCallback> mHandleReport;
3501 RefPtr<nsISupports> mHandlerData;
3504 RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
3505 std::move(aHandleReport), std::move(aHandlerData));
3507 auto audioTrackSizes = &runnable->mAudioTrackSizes;
3509 for (MediaTrack* t : AllTracks()) {
3510 AudioNodeTrack* track = t->AsAudioNodeTrack();
3511 if (track) {
3512 AudioNodeSizes* usage = audioTrackSizes->AppendElement();
3513 track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3517 mMainThread->Dispatch(runnable.forget());
3520 void MediaTrackGraphImpl::FinishCollectReports(
3521 nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3522 const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
3523 MOZ_ASSERT(NS_IsMainThread());
3525 nsCOMPtr<nsIMemoryReporterManager> manager =
3526 do_GetService("@mozilla.org/memory-reporter-manager;1");
3528 if (!manager) return;
3530 #define REPORT(_path, _amount, _desc) \
3531 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3532 nsLiteralCString(_desc), aData);
3534 for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
3535 const AudioNodeSizes& usage = aAudioTrackSizes[i];
3536 const char* const nodeType =
3537 usage.mNodeType ? usage.mNodeType : "<unknown>";
3539 nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3540 nodeType);
3541 REPORT(enginePath, usage.mEngine,
3542 "Memory used by AudioNode engine objects (Web Audio).");
3544 nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
3545 nodeType);
3546 REPORT(trackPath, usage.mTrack,
3547 "Memory used by AudioNode track objects (Web Audio).");
3550 size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3551 if (hrtfLoaders) {
3552 REPORT(nsLiteralCString(
3553 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3554 hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
3557 #undef REPORT
3559 manager->EndReport();
3562 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
3563 SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
3564 AddTrack(track);
3565 return track;
3568 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
3569 MediaSegment::Type aType) {
3570 ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
3571 AddTrack(track);
3572 return track;
3575 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
3576 AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
3577 AddTrack(track);
3578 return track;
3581 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
3582 CrossGraphReceiver* aReceiver) {
3583 CrossGraphTransmitter* track =
3584 new CrossGraphTransmitter(GraphRate(), aReceiver);
3585 AddTrack(track);
3586 return track;
3589 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
3590 TrackRate aTransmitterRate) {
3591 CrossGraphReceiver* track =
3592 new CrossGraphReceiver(GraphRate(), aTransmitterRate);
3593 AddTrack(track);
3594 return track;
3597 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
3598 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3599 MOZ_ASSERT(NS_IsMainThread());
3600 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3601 if (graph->mRealtime) {
3602 GraphHashSet::Ptr p = Graphs()->lookup(*graph);
3603 MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down");
3605 #endif
3606 NS_ADDREF(aTrack);
3607 aTrack->SetGraphImpl(graph);
3608 ++graph->mMainThreadTrackCount;
3609 graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
3612 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
3613 MOZ_ASSERT(NS_IsMainThread());
3614 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
3616 mAudioOutputParams.RemoveElementsBy([&](const TrackKeyAndVolume& aElement) {
3617 if (aElement.mTrack != aTrack) {
3618 return false;
3620 return true;
3623 if (--mMainThreadTrackCount == 0) {
3624 LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
3625 "main thread. Graph will shut down.",
3626 this, aTrack));
3627 if (mRealtime) {
3628 // Find the graph in the hash table and remove it.
3629 GraphHashSet* graphs = Graphs();
3630 GraphHashSet::Ptr p = graphs->lookup(*this);
3631 MOZ_ASSERT(*p == this);
3632 graphs->remove(p);
3634 nsCOMPtr<nsIObserverService> observerService =
3635 mozilla::services::GetObserverService();
3636 if (observerService) {
3637 observerService->RemoveObserver(this, "document-title-changed");
3640 // The graph thread will shut itself down soon, but won't be able to do
3641 // that if JS continues to run.
3642 InterruptJS();
3646 auto MediaTrackGraph::NotifyWhenDeviceStarted(MediaTrack* aTrack)
3647 -> RefPtr<GraphStartedPromise> {
3648 MOZ_ASSERT(NS_IsMainThread());
3649 MozPromiseHolder<GraphStartedPromise> h;
3650 RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
3651 aTrack->GraphImpl()->NotifyWhenGraphStarted(aTrack, std::move(h));
3652 return p;
3655 void MediaTrackGraphImpl::NotifyWhenGraphStarted(
3656 RefPtr<MediaTrack> aTrack,
3657 MozPromiseHolder<GraphStartedPromise>&& aHolder) {
3658 class GraphStartedNotificationControlMessage : public ControlMessage {
3659 RefPtr<MediaTrack> mMediaTrack;
3660 MozPromiseHolder<GraphStartedPromise> mHolder;
3662 public:
3663 GraphStartedNotificationControlMessage(
3664 RefPtr<MediaTrack> aTrack,
3665 MozPromiseHolder<GraphStartedPromise>&& aHolder)
3666 : ControlMessage(nullptr),
3667 mMediaTrack(std::move(aTrack)),
3668 mHolder(std::move(aHolder)) {}
3669 void Run() override {
3670 TRACE("MTG::GraphStartedNotificationControlMessage ControlMessage");
3671 // This runs on the graph thread, so when this runs, and the current
3672 // driver is an AudioCallbackDriver, we know the audio hardware is
3673 // started. If not, we are going to switch soon, keep reposting this
3674 // ControlMessage.
3675 MediaTrackGraphImpl* graphImpl = mMediaTrack->GraphImpl();
3676 if (graphImpl->CurrentDriver()->AsAudioCallbackDriver() &&
3677 graphImpl->CurrentDriver()->ThreadRunning() &&
3678 !graphImpl->CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3679 // Avoid Resolve's locking on the graph thread by doing it on main.
3680 graphImpl->Dispatch(NS_NewRunnableFunction(
3681 "MediaTrackGraphImpl::NotifyWhenGraphStarted::Resolver",
3682 [holder = std::move(mHolder)]() mutable {
3683 holder.Resolve(true, __func__);
3684 }));
3685 } else {
3686 graphImpl->DispatchToMainThreadStableState(
3687 NewRunnableMethod<
3688 StoreCopyPassByRRef<RefPtr<MediaTrack>>,
3689 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
3690 "MediaTrackGraphImpl::NotifyWhenGraphStarted", graphImpl,
3691 &MediaTrackGraphImpl::NotifyWhenGraphStarted,
3692 std::move(mMediaTrack), std::move(mHolder)));
3695 void RunDuringShutdown() override {
3696 mHolder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
3700 if (aTrack->IsDestroyed()) {
3701 aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
3702 return;
3705 MediaTrackGraphImpl* graph = aTrack->GraphImpl();
3706 graph->AppendMessage(MakeUnique<GraphStartedNotificationControlMessage>(
3707 std::move(aTrack), std::move(aHolder)));
3710 class AudioContextOperationControlMessage : public ControlMessage {
3711 using AudioContextOperationPromise =
3712 MediaTrackGraph::AudioContextOperationPromise;
3714 public:
3715 AudioContextOperationControlMessage(
3716 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3717 AudioContextOperation aOperation,
3718 MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
3719 : ControlMessage(aDestinationTrack),
3720 mTracks(std::move(aTracks)),
3721 mAudioContextOperation(aOperation),
3722 mHolder(std::move(aHolder)) {}
3723 void Run() override {
3724 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3725 kAudioContextOptionsStrings[static_cast<uint8_t>(
3726 mAudioContextOperation)]);
3727 mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
3729 void RunDuringShutdown() override {
3730 MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
3731 "We should be reviving the graph?");
3732 mHolder.Reject(false, __func__);
3735 nsTArray<RefPtr<MediaTrack>> mTracks;
3736 AudioContextOperation mAudioContextOperation;
3737 MozPromiseHolder<AudioContextOperationPromise> mHolder;
3740 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3741 AudioContextOperationControlMessage* aMessage) {
3742 MOZ_ASSERT(OnGraphThread());
3743 // Initialize state to zero. This silences a GCC warning about uninitialized
3744 // values, because although the switch below initializes state for all valid
3745 // enum values, the actual value could be any integer that fits in the enum.
3746 AudioContextState state{0};
3747 switch (aMessage->mAudioContextOperation) {
3748 // Suspend and Close operations may be performed immediately because no
3749 // specific kind of GraphDriver is required. CheckDriver() will schedule
3750 // a change to a SystemCallbackDriver if all tracks are suspended.
3751 case AudioContextOperation::Suspend:
3752 state = AudioContextState::Suspended;
3753 break;
3754 case AudioContextOperation::Close:
3755 state = AudioContextState::Closed;
3756 break;
3757 case AudioContextOperation::Resume:
3758 // Resume operations require an AudioCallbackDriver. CheckDriver() will
3759 // schedule an AudioCallbackDriver if necessary and process pending
3760 // operations if and when an AudioCallbackDriver is running.
3761 mPendingResumeOperations.EmplaceBack(aMessage);
3762 return;
3764 // First resolve any pending Resume promises for the same AudioContext so as
3765 // to resolve its associated promises in the same order as they were
3766 // created. These Resume operations are considered complete and immediately
3767 // canceled by the Suspend or Close.
3768 MediaTrack* destinationTrack = aMessage->GetTrack();
3769 bool shrinking = false;
3770 auto moveDest = mPendingResumeOperations.begin();
3771 for (PendingResumeOperation& op : mPendingResumeOperations) {
3772 if (op.DestinationTrack() == destinationTrack) {
3773 op.Apply(this);
3774 shrinking = true;
3775 continue;
3777 if (shrinking) { // Fill-in gaps in the array.
3778 *moveDest = std::move(op);
3780 ++moveDest;
3782 mPendingResumeOperations.TruncateLength(moveDest -
3783 mPendingResumeOperations.begin());
3785 for (MediaTrack* track : aMessage->mTracks) {
3786 track->IncrementSuspendCount();
3788 // Resolve after main thread state is up to date with completed processing.
3789 DispatchToMainThreadStableState(NS_NewRunnableFunction(
3790 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
3791 [holder = std::move(aMessage->mHolder), state]() mutable {
3792 holder.Resolve(state, __func__);
3793 }));
3796 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
3797 AudioContextOperationControlMessage* aMessage)
3798 : mDestinationTrack(aMessage->GetTrack()),
3799 mTracks(std::move(aMessage->mTracks)),
3800 mHolder(std::move(aMessage->mHolder)) {
3801 MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
3804 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
3805 MediaTrackGraphImpl* aGraph) {
3806 MOZ_ASSERT(aGraph->OnGraphThread());
3807 for (MediaTrack* track : mTracks) {
3808 track->DecrementSuspendCount();
3810 // The graph is provided through the parameter so that it is available even
3811 // when the track is destroyed.
3812 aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
3813 "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
3814 holder.Resolve(AudioContextState::Running, __func__);
3815 }));
3818 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
3819 // The graph is shutting down before the operation completed.
3820 MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
3821 mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
3822 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
3823 mHolder.Reject(false, __func__);
3826 auto MediaTrackGraph::ApplyAudioContextOperation(
3827 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3828 AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
3829 MozPromiseHolder<AudioContextOperationPromise> holder;
3830 RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
3831 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3832 graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
3833 aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
3834 return p;
3837 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount() const {
3838 MOZ_ASSERT(OnGraphThread());
3839 // The audio output channel count for a graph is the maximum of the output
3840 // channel count of all the tracks that are in mAudioOutputs, or the max audio
3841 // output channel count the machine can do, whichever is smaller.
3842 uint32_t channelCount = 0;
3843 for (const auto& output : mAudioOutputs) {
3844 channelCount = std::max(channelCount, output.mTrack->NumberOfChannels());
3846 channelCount = std::min(channelCount, mMaxOutputChannelCount);
3847 if (channelCount) {
3848 return channelCount;
3849 } else {
3850 if (CurrentDriver()->AsAudioCallbackDriver()) {
3851 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
3853 return 2;
3857 double MediaTrackGraph::AudioOutputLatency() {
3858 return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
3861 double MediaTrackGraphImpl::AudioOutputLatency() {
3862 MOZ_ASSERT(NS_IsMainThread());
3863 if (mAudioOutputLatency != 0.0) {
3864 return mAudioOutputLatency;
3866 MonitorAutoLock lock(mMonitor);
3867 if (CurrentDriver()->AsAudioCallbackDriver()) {
3868 mAudioOutputLatency = CurrentDriver()
3869 ->AsAudioCallbackDriver()
3870 ->AudioOutputLatency()
3871 .ToSeconds();
3872 } else {
3873 // Failure mode: return 0.0 if running on a normal thread.
3874 mAudioOutputLatency = 0.0;
3877 return mAudioOutputLatency;
3880 bool MediaTrackGraph::IsNonRealtime() const {
3881 return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
3884 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
3885 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
3887 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3888 NS_ASSERTION(!graph->mRealtime, "non-realtime only");
3890 class Message : public ControlMessage {
3891 public:
3892 explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
3893 : ControlMessage(nullptr),
3894 mGraph(aGraph),
3895 mTicksToProcess(aTicksToProcess) {}
3896 void Run() override {
3897 TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
3898 MOZ_ASSERT(mGraph->mEndTime == 0,
3899 "StartNonRealtimeProcessing should be called only once");
3900 mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
3901 mGraph->mStateComputedTime + mTicksToProcess);
3903 // The graph owns this message.
3904 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
3905 uint32_t mTicksToProcess;
3908 graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
3911 void MediaTrackGraphImpl::InterruptJS() {
3912 MonitorAutoLock lock(mMonitor);
3913 mInterruptJSCalled = true;
3914 if (mJSContext) {
3915 JS_RequestInterruptCallback(mJSContext);
3919 static bool InterruptCallback(JSContext* aCx) {
3920 // Interrupt future calls also.
3921 JS_RequestInterruptCallback(aCx);
3922 // Stop execution.
3923 return false;
3926 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
3927 MOZ_ASSERT(OnGraphThread());
3928 MOZ_ASSERT(aCx);
3930 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
3931 MonitorAutoLock lock(impl->mMonitor);
3932 if (impl->mJSContext) {
3933 MOZ_ASSERT(impl->mJSContext == aCx);
3934 return;
3936 JS_AddInterruptCallback(aCx, InterruptCallback);
3937 impl->mJSContext = aCx;
3938 if (impl->mInterruptJSCalled) {
3939 JS_RequestInterruptCallback(aCx);
3943 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
3944 MediaTrack* t = aPort->GetSource();
3945 if (!t->IsSuspended()) {
3946 mInputs.AppendElement(aPort);
3947 } else {
3948 mSuspendedInputs.AppendElement(aPort);
3950 GraphImpl()->SetTrackOrderDirty();
3953 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
3954 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3955 mInputs.RemoveElement(aPort);
3956 mSuspendedInputs.AppendElement(aPort);
3957 GraphImpl()->SetTrackOrderDirty();
3960 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
3961 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3962 mSuspendedInputs.RemoveElement(aPort);
3963 mInputs.AppendElement(aPort);
3964 GraphImpl()->SetTrackOrderDirty();
3967 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
3968 MOZ_ASSERT(OnGraphThread());
3969 LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
3970 if (GraphDriver* nextDriver = NextDriver()) {
3971 if (nextDriver != CurrentDriver()) {
3972 LOG(LogLevel::Debug,
3973 ("%p: Discarding previous next driver: %p", this, nextDriver));
3976 mNextDriver = aNextDriver;
3979 void MediaTrackGraph::RegisterCaptureTrackForWindow(
3980 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
3981 MOZ_ASSERT(NS_IsMainThread());
3982 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3983 graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
3986 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
3987 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
3988 MOZ_ASSERT(NS_IsMainThread());
3989 WindowAndTrack winAndTrack;
3990 winAndTrack.mWindowId = aWindowId;
3991 winAndTrack.mCaptureTrackSink = aCaptureTrack;
3992 mWindowCaptureTracks.AppendElement(winAndTrack);
3995 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
3996 MOZ_ASSERT(NS_IsMainThread());
3997 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3998 graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
4001 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4002 MOZ_ASSERT(NS_IsMainThread());
4003 mWindowCaptureTracks.RemoveElementsBy(
4004 [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
4007 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
4008 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4009 return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
4010 aMediaTrack);
4013 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
4014 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4015 MOZ_ASSERT(NS_IsMainThread());
4016 for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
4017 if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
4018 ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
4019 return sink->AllocateInputPort(aMediaTrack);
4022 return nullptr;
4025 void MediaTrackGraph::DispatchToMainThreadStableState(
4026 already_AddRefed<nsIRunnable> aRunnable) {
4027 AssertOnGraphThreadOrNotRunning();
4028 static_cast<MediaTrackGraphImpl*>(this)
4029 ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
4032 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
4033 MOZ_ASSERT(NS_IsMainThread());
4034 return mMainThreadGraphTime;
4037 GraphTime MediaTrackGraph::ProcessedTime() const {
4038 AssertOnGraphThreadOrNotRunning();
4039 return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
4042 void* MediaTrackGraph::CurrentDriver() const {
4043 AssertOnGraphThreadOrNotRunning();
4044 return static_cast<const MediaTrackGraphImpl*>(this)->mDriver;
4047 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4048 CubebUtils::AudioDeviceID aID) {
4049 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4050 DeviceInputTrack* t =
4051 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4052 return t ? t->MaxRequestedInputChannels() : 0;
4055 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
4056 CubebUtils::AudioDeviceID aID) {
4057 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4058 DeviceInputTrack* t =
4059 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4060 return t && t->HasVoiceInput() ? AudioInputType::Voice
4061 : AudioInputType::Unknown;
4064 void MediaTrackGraphImpl::SetNewNativeInput() {
4065 MOZ_ASSERT(NS_IsMainThread());
4066 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4068 LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
4070 NonNativeInputTrack* track =
4071 mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
4072 if (!track) {
4073 LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
4074 return;
4077 const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
4078 const PrincipalHandle principal = track->mPrincipalHandle;
4080 LOG(LogLevel::Debug,
4081 ("%p Select device %p as the new native input device", this, deviceId));
4083 struct TrackListener {
4084 DeviceInputConsumerTrack* track;
4085 // Keep its reference so it won't be dropped when after
4086 // DisconnectDeviceInput().
4087 RefPtr<AudioDataListener> listener;
4089 nsTArray<TrackListener> pairs;
4091 for (const auto& t : track->GetConsumerTracks()) {
4092 pairs.AppendElement(
4093 TrackListener{t.get(), t->GetAudioDataListener().get()});
4096 for (TrackListener& pair : pairs) {
4097 pair.track->DisconnectDeviceInput();
4100 for (TrackListener& pair : pairs) {
4101 pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
4102 LOG(LogLevel::Debug,
4103 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4104 pair.track, deviceId));
4107 LOG(LogLevel::Debug,
4108 ("%p Native input device is set to device %p now", this, deviceId));
4110 MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4113 // nsIThreadObserver methods
4115 NS_IMETHODIMP
4116 MediaTrackGraphImpl::OnDispatchedEvent() {
4117 MonitorAutoLock lock(mMonitor);
4118 EnsureNextIteration();
4119 return NS_OK;
4122 NS_IMETHODIMP
4123 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
4124 return NS_OK;
4127 NS_IMETHODIMP
4128 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
4129 return NS_OK;
4131 } // namespace mozilla