no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD CLOSED TREE
[gecko.git] / dom / media / MediaTrackGraph.cpp
blob157ad403d269a084171f5460c9d6d31f4a902448
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaTrackGraphImpl.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "CrossGraphPort.h"
12 #include "VideoSegment.h"
13 #include "nsContentUtils.h"
14 #include "nsGlobalWindowInner.h"
15 #include "nsPrintfCString.h"
16 #include "nsServiceManagerUtils.h"
17 #include "prerror.h"
18 #include "mozilla/Logging.h"
19 #include "mozilla/Attributes.h"
20 #include "ForwardedInputTrack.h"
21 #include "ImageContainer.h"
22 #include "AudioCaptureTrack.h"
23 #include "AudioDeviceInfo.h"
24 #include "AudioNodeTrack.h"
25 #include "AudioNodeExternalInputTrack.h"
26 #if defined(MOZ_WEBRTC)
27 # include "MediaEngineWebRTCAudio.h"
28 #endif // MOZ_WEBRTC
29 #include "MediaTrackListener.h"
30 #include "mozilla/dom/BaseAudioContextBinding.h"
31 #include "mozilla/dom/Document.h"
32 #include "mozilla/dom/WorkletThread.h"
33 #include "mozilla/media/MediaUtils.h"
34 #include <algorithm>
35 #include "GeckoProfiler.h"
36 #include "VideoFrameContainer.h"
37 #include "mozilla/AbstractThread.h"
38 #include "mozilla/StaticPrefs_dom.h"
39 #include "mozilla/StaticPrefs_media.h"
40 #include "transport/runnable_utils.h"
41 #include "VideoUtils.h"
42 #include "GraphRunner.h"
43 #include "Tracing.h"
44 #include "UnderrunHandler.h"
45 #include "mozilla/CycleCollectedJSRuntime.h"
46 #include "mozilla/Preferences.h"
48 #include "webaudio/blink/DenormalDisabler.h"
49 #include "webaudio/blink/HRTFDatabaseLoader.h"
51 using namespace mozilla::layers;
52 using namespace mozilla::dom;
53 using namespace mozilla::gfx;
54 using namespace mozilla::media;
56 namespace mozilla {
58 using AudioDeviceID = CubebUtils::AudioDeviceID;
59 using IsInShutdown = MediaTrack::IsInShutdown;
61 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
62 #ifdef LOG
63 # undef LOG
64 #endif // LOG
65 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
67 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
68 return mNativeInputTrack.get();
71 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
72 CubebUtils::AudioDeviceID aID) {
73 if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
74 return mNativeInputTrack.get();
76 for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
77 if (t->mDeviceId == aID) {
78 return t.get();
81 return nullptr;
84 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
85 if (mNonNativeInputTracks.IsEmpty()) {
86 return nullptr;
88 return mNonNativeInputTracks[0].get();
91 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
92 if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
93 MOZ_ASSERT(!mNativeInputTrack);
94 mNativeInputTrack = native;
95 } else {
96 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
97 MOZ_ASSERT(nonNative);
98 struct DeviceTrackComparator {
99 public:
100 bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
101 CubebUtils::AudioDeviceID aDeviceId) const {
102 return aTrack->mDeviceId == aDeviceId;
105 MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
106 DeviceTrackComparator()));
107 mNonNativeInputTracks.AppendElement(nonNative);
111 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
112 if (aTrack->AsNativeInputTrack()) {
113 MOZ_ASSERT(mNativeInputTrack);
114 MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
115 mNativeInputTrack = nullptr;
116 } else {
117 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
118 MOZ_ASSERT(nonNative);
119 DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
120 MOZ_ASSERT(removed);
125 * A hash table containing the graph instances, one per Window ID,
126 * sample rate, and device ID combination.
129 struct MediaTrackGraphImpl::Lookup final {
130 HashNumber Hash() const {
131 return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
133 const uint64_t mWindowID;
134 const TrackRate mSampleRate;
135 const CubebUtils::AudioDeviceID mOutputDeviceID;
138 // Implicit to support GraphHashSet.lookup(*graph).
139 MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const {
140 return {mWindowID, mSampleRate, PrimaryOutputDeviceID()};
143 namespace {
144 struct GraphHasher { // for HashSet
145 using Lookup = const MediaTrackGraphImpl::Lookup;
147 static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); }
149 static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) {
150 return aGraph->mWindowID == aLookup.mWindowID &&
151 aGraph->GraphRate() == aLookup.mSampleRate &&
152 aGraph->PrimaryOutputDeviceID() == aLookup.mOutputDeviceID;
156 // The weak reference to the graph is removed when its last track is removed.
157 using GraphHashSet =
158 HashSet<MediaTrackGraphImpl*, GraphHasher, InfallibleAllocPolicy>;
159 GraphHashSet* Graphs() {
160 MOZ_ASSERT(NS_IsMainThread());
161 static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity
162 return &sGraphs;
164 } // anonymous namespace
166 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode,
167 MediaSegment* aSegment,
168 MediaSegment* aRawSegment) {
169 if (aDisabledMode == DisabledTrackMode::ENABLED) {
170 return;
172 if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
173 aSegment->ReplaceWithDisabled();
174 if (aRawSegment) {
175 aRawSegment->ReplaceWithDisabled();
177 } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
178 aSegment->ReplaceWithNull();
179 if (aRawSegment) {
180 aRawSegment->ReplaceWithNull();
182 } else {
183 MOZ_CRASH("Unsupported mode");
187 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
188 MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
189 "All tracks should have been destroyed by messages from the main "
190 "thread");
191 LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
192 LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
195 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
196 MOZ_ASSERT(OnGraphThreadOrNotRunning());
197 aTrack->mStartTime = mProcessedTime;
199 if (aTrack->IsSuspended()) {
200 mSuspendedTracks.AppendElement(aTrack);
201 LOG(LogLevel::Debug,
202 ("%p: Adding media track %p, in the suspended track array", this,
203 aTrack));
204 } else {
205 mTracks.AppendElement(aTrack);
206 LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack,
207 mTracks.Length()));
210 SetTrackOrderDirty();
213 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
214 MOZ_ASSERT(OnGraphThreadOrNotRunning());
215 // Remove references in mTrackUpdates before we allow aTrack to die.
216 // Pending updates are not needed (since the main thread has already given
217 // up the track) so we will just drop them.
219 MonitorAutoLock lock(mMonitor);
220 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
221 if (mTrackUpdates[i].mTrack == aTrack) {
222 mTrackUpdates[i].mTrack = nullptr;
227 // Ensure that mFirstCycleBreaker is updated when necessary.
228 SetTrackOrderDirty();
230 UnregisterAllAudioOutputs(aTrack);
232 if (aTrack->IsSuspended()) {
233 mSuspendedTracks.RemoveElement(aTrack);
234 } else {
235 mTracks.RemoveElement(aTrack);
238 LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
239 mTracks.Length()));
241 NS_RELEASE(aTrack); // probably destroying it
244 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
245 const MediaTrack* aTrack, GraphTime aTime) const {
246 MOZ_ASSERT(
247 aTime <= mStateComputedTime,
248 "Don't ask about times where we haven't made blocking decisions yet");
249 return std::max<TrackTime>(
250 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
253 GraphTime MediaTrackGraphImpl::IterationEnd() const {
254 MOZ_ASSERT(OnGraphThread());
255 return mIterationEndTime;
258 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
259 GraphTime aPrevCurrentTime) {
260 MOZ_ASSERT(OnGraphThread());
261 for (MediaTrack* track : AllTracks()) {
262 // Shouldn't have already notified of ended *and* have output!
263 MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
264 !track->mNotifiedEnded);
266 // Calculate blocked time and fire Blocked/Unblocked events
267 GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
268 NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
269 track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
270 blockedTime);
271 LOG(LogLevel::Verbose,
272 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
273 MediaTimeToSeconds(track->mStartTime),
274 MediaTimeToSeconds(blockedTime)));
275 track->mStartBlocking = mStateComputedTime;
277 TrackTime trackCurrentTime =
278 track->GraphTimeToTrackTime(mStateComputedTime);
279 if (track->mEnded) {
280 MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
281 if (!track->mNotifiedEnded) {
282 // Playout of this track ended and listeners have not been notified.
283 track->mNotifiedEnded = true;
284 SetTrackOrderDirty();
285 for (const auto& listener : track->mTrackListeners) {
286 listener->NotifyOutput(this, track->GetEnd());
287 listener->NotifyEnded(this);
290 } else {
291 for (const auto& listener : track->mTrackListeners) {
292 listener->NotifyOutput(this, trackCurrentTime);
298 template <typename C, typename Chunk>
299 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
300 C& aSegment,
301 TrackTime aStart,
302 TrackTime aEnd) {
303 MOZ_ASSERT(OnGraphThreadOrNotRunning());
304 MOZ_ASSERT(aTrack);
306 TrackTime offset = 0;
307 for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
308 chunk.Next()) {
309 if (offset >= aEnd) {
310 break;
312 offset += chunk->GetDuration();
313 if (chunk->IsNull() || offset < aStart) {
314 continue;
316 const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
317 if (principalHandle != aSegment.GetLastPrincipalHandle()) {
318 aSegment.SetLastPrincipalHandle(principalHandle);
319 LOG(LogLevel::Debug,
320 ("%p: MediaTrack %p, principalHandle "
321 "changed in %sChunk with duration %lld",
322 this, aTrack,
323 aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
324 (long long)chunk->GetDuration()));
325 for (const auto& listener : aTrack->mTrackListeners) {
326 listener->NotifyPrincipalHandleChanged(this, principalHandle);
332 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
333 MOZ_ASSERT(OnGraphThreadOrNotRunning());
334 for (MediaTrack* track : AllTracks()) {
335 TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
336 TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
337 if (!track->mSegment) {
338 continue;
340 if (track->mType == MediaSegment::AUDIO) {
341 ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
342 track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
343 } else if (track->mType == MediaSegment::VIDEO) {
344 ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
345 track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
346 } else {
347 MOZ_CRASH("Unknown track type");
352 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
353 GraphTime aEndBlockingDecisions) {
354 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
355 // underrun currently, since we'll always be able to produce data for them
356 // unless they block on some other track.
357 if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
358 return aEndBlockingDecisions;
360 // This track isn't ended or suspended. We don't need to call
361 // TrackTimeToGraphTime since an underrun is the only thing that can block
362 // it.
363 GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
364 #ifdef DEBUG
365 if (bufferEnd < mProcessedTime) {
366 LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
367 "bufferEnd %f < mProcessedTime %f (%" PRId64
368 " < %" PRId64 "), TrackTime %" PRId64,
369 this, aTrack, MediaTimeToSeconds(bufferEnd),
370 MediaTimeToSeconds(mProcessedTime), bufferEnd,
371 mProcessedTime, aTrack->GetEnd()));
372 NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
374 #endif
375 return std::min(bufferEnd, aEndBlockingDecisions);
378 namespace {
379 // Value of mCycleMarker for unvisited tracks in cycle detection.
380 const uint32_t NOT_VISITED = UINT32_MAX;
381 // Value of mCycleMarker for ordered tracks in muted cycles.
382 const uint32_t IN_MUTED_CYCLE = 1;
383 } // namespace
385 bool MediaTrackGraphImpl::AudioTrackPresent() {
386 MOZ_ASSERT(OnGraphThreadOrNotRunning());
388 bool audioTrackPresent = false;
389 for (MediaTrack* track : mTracks) {
390 if (track->AsAudioNodeTrack()) {
391 audioTrackPresent = true;
392 break;
395 if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
396 audioTrackPresent = true;
397 break;
401 // We may not have audio input device when we only have AudioNodeTracks. But
402 // if audioTrackPresent is false, we must have no input device.
403 MOZ_DIAGNOSTIC_ASSERT_IF(
404 !audioTrackPresent,
405 !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
407 return audioTrackPresent;
410 void MediaTrackGraphImpl::CheckDriver() {
411 MOZ_ASSERT(OnGraphThread());
412 // An offline graph has only one driver.
413 // Otherwise, if a switch is already pending, let that happen.
414 if (!mRealtime || Switching()) {
415 return;
418 AudioCallbackDriver* audioCallbackDriver =
419 CurrentDriver()->AsAudioCallbackDriver();
420 if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
421 for (PendingResumeOperation& op : mPendingResumeOperations) {
422 op.Apply(this);
424 mPendingResumeOperations.Clear();
427 // Note that this looks for any audio tracks, input or output, and switches
428 // to a SystemClockDriver if there are none active or no resume operations
429 // to make any active.
430 bool needAudioCallbackDriver =
431 !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
432 if (!needAudioCallbackDriver) {
433 if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
434 SwitchAtNextIteration(
435 new SystemClockDriver(this, CurrentDriver(), mSampleRate));
437 return;
440 NativeInputTrack* native =
441 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
442 CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
443 uint32_t inputChannelCount =
444 native ? AudioInputChannelCount(native->mDeviceId) : 0;
445 AudioInputType inputPreference =
446 native ? AudioInputDevicePreference(native->mDeviceId)
447 : AudioInputType::Unknown;
449 uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount();
450 if (!audioCallbackDriver) {
451 if (primaryOutputChannelCount > 0) {
452 AudioCallbackDriver* driver = new AudioCallbackDriver(
453 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
454 inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
455 inputPreference);
456 SwitchAtNextIteration(driver);
458 return;
461 // Check if this graph should switch to a different number of output channels.
462 // Generally, a driver switch is explicitly made by an event (e.g., setting
463 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
464 // directly playing back via another HTMLMediaElement, the number of channels
465 // of the media determines how many channels to output, and it can change
466 // dynamically.
467 if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
468 AudioCallbackDriver* driver = new AudioCallbackDriver(
469 this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
470 inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
471 inputPreference);
472 SwitchAtNextIteration(driver);
476 void MediaTrackGraphImpl::UpdateTrackOrder() {
477 if (!mTrackOrderDirty) {
478 return;
481 mTrackOrderDirty = false;
483 // The algorithm for finding cycles is based on Tim Leslie's iterative
484 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
485 // connected components (SCC) algorithm. There are variations (a) to
486 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
487 // re-run the algorithm over SCCs with breaks at DelayNodes.
489 // [1] http://www.timl.id.au/?p=327
490 // [2]
491 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
492 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
494 // There are two stacks. One for the depth-first search (DFS),
495 mozilla::LinkedList<MediaTrack> dfsStack;
496 // and another for tracks popped from the DFS stack, but still being
497 // considered as part of SCCs involving tracks on the stack.
498 mozilla::LinkedList<MediaTrack> sccStack;
500 // An index into mTracks for the next track found with no unsatisfied
501 // upstream dependencies.
502 uint32_t orderedTrackCount = 0;
504 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
505 MediaTrack* t = mTracks[i];
506 ProcessedMediaTrack* pt = t->AsProcessedTrack();
507 if (pt) {
508 // The dfsStack initially contains a list of all processed tracks in
509 // unchanged order.
510 dfsStack.insertBack(t);
511 pt->mCycleMarker = NOT_VISITED;
512 } else {
513 // SourceMediaTracks have no inputs and so can be ordered now.
514 mTracks[orderedTrackCount] = t;
515 ++orderedTrackCount;
519 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
520 // counter to label mCycleMarker on the next visited track in the DFS
521 // uniquely in the set of visited tracks that are still being considered.
523 // In this implementation, the counter descends so that the values are
524 // strictly greater than the values that mCycleMarker takes when the track
525 // has been ordered (0 or IN_MUTED_CYCLE).
527 // Each new track labelled, as the DFS searches upstream, receives a value
528 // less than those used for all other tracks being considered.
529 uint32_t nextStackMarker = NOT_VISITED - 1;
530 // Reset list of DelayNodes in cycles stored at the tail of mTracks.
531 mFirstCycleBreaker = mTracks.Length();
533 // Rearrange dfsStack order as required to DFS upstream and pop tracks
534 // in processing order to place in mTracks.
535 while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
536 const auto& inputs = pt->mInputs;
537 MOZ_ASSERT(pt->AsProcessedTrack());
538 if (pt->mCycleMarker == NOT_VISITED) {
539 // Record the position on the visited stack, so that any searches
540 // finding this track again know how much of the stack is in the cycle.
541 pt->mCycleMarker = nextStackMarker;
542 --nextStackMarker;
543 // Not-visited input tracks should be processed first.
544 // SourceMediaTracks have already been ordered.
545 for (uint32_t i = inputs.Length(); i--;) {
546 if (inputs[i]->GetSource()->IsSuspended()) {
547 continue;
549 auto input = inputs[i]->GetSource()->AsProcessedTrack();
550 if (input && input->mCycleMarker == NOT_VISITED) {
551 // It can be that this track has an input which is from a suspended
552 // AudioContext.
553 if (input->isInList()) {
554 input->remove();
555 dfsStack.insertFront(input);
559 continue;
562 // Returning from DFS. Pop from dfsStack.
563 pt->remove();
565 // cycleStackMarker keeps track of the highest marker value on any
566 // upstream track, if any, found receiving input, directly or indirectly,
567 // from the visited stack (and so from |ps|, making a cycle). In a
568 // variation from Tarjan's SCC algorithm, this does not include |ps|
569 // unless it is part of the cycle.
570 uint32_t cycleStackMarker = 0;
571 for (uint32_t i = inputs.Length(); i--;) {
572 if (inputs[i]->GetSource()->IsSuspended()) {
573 continue;
575 auto input = inputs[i]->GetSource()->AsProcessedTrack();
576 if (input) {
577 cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
581 if (cycleStackMarker <= IN_MUTED_CYCLE) {
582 // All inputs have been ordered and their stack markers have been removed.
583 // This track is not part of a cycle. It can be processed next.
584 pt->mCycleMarker = 0;
585 mTracks[orderedTrackCount] = pt;
586 ++orderedTrackCount;
587 continue;
590 // A cycle has been found. Record this track for ordering when all
591 // tracks in this SCC have been popped from the DFS stack.
592 sccStack.insertFront(pt);
594 if (cycleStackMarker > pt->mCycleMarker) {
595 // Cycles have been found that involve tracks that remain on the stack.
596 // Leave mCycleMarker indicating the most downstream (last) track on
597 // the stack known to be part of this SCC. In this way, any searches on
598 // other paths that find |ps| will know (without having to traverse from
599 // this track again) that they are part of this SCC (i.e. part of an
600 // intersecting cycle).
601 pt->mCycleMarker = cycleStackMarker;
602 continue;
605 // |pit| is the root of an SCC involving no other tracks on dfsStack, the
606 // complete SCC has been recorded, and tracks in this SCC are part of at
607 // least one cycle.
608 MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
609 // If there are DelayNodes in this SCC, then they may break the cycles.
610 bool haveDelayNode = false;
611 auto next = sccStack.getFirst();
612 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
613 // (There may be other tracks later in sccStack from other incompletely
614 // searched SCCs, involving tracks still on dfsStack.)
616 // DelayNodes in cycles must behave differently from those not in cycles,
617 // so all DelayNodes in the SCC must be identified.
618 while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
619 cycleStackMarker) {
620 auto nt = next->AsAudioNodeTrack();
621 // Get next before perhaps removing from list below.
622 next = next->getNext();
623 if (nt && nt->Engine()->AsDelayNodeEngine()) {
624 haveDelayNode = true;
625 // DelayNodes break cycles by producing their output in a
626 // preprocessing phase; they do not need to be ordered before their
627 // consumers. Order them at the tail of mTracks so that they can be
628 // handled specially. Do so now, so that DFS ignores them.
629 nt->remove();
630 nt->mCycleMarker = 0;
631 --mFirstCycleBreaker;
632 mTracks[mFirstCycleBreaker] = nt;
635 auto after_scc = next;
636 while ((next = sccStack.getFirst()) != after_scc) {
637 next->remove();
638 auto removed = static_cast<ProcessedMediaTrack*>(next);
639 if (haveDelayNode) {
640 // Return tracks to the DFS stack again (to order and detect cycles
641 // without delayNodes). Any of these tracks that are still inputs
642 // for tracks on the visited stack must be returned to the front of
643 // the stack to be ordered before their dependents. We know that none
644 // of these tracks need input from tracks on the visited stack, so
645 // they can all be searched and ordered before the current stack head
646 // is popped.
647 removed->mCycleMarker = NOT_VISITED;
648 dfsStack.insertFront(removed);
649 } else {
650 // Tracks in cycles without any DelayNodes must be muted, and so do
651 // not need input and can be ordered now. They must be ordered before
652 // their consumers so that their muted output is available.
653 removed->mCycleMarker = IN_MUTED_CYCLE;
654 mTracks[orderedTrackCount] = removed;
655 ++orderedTrackCount;
660 MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
663 TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput,
664 GraphTime aPlayedTime,
665 uint32_t aOutputChannelCount) {
666 MOZ_ASSERT(OnGraphThread());
667 MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
669 TrackTime ticksWritten = 0;
671 ticksWritten = 0;
672 MediaTrack* track = aOutput.mTrack;
673 AudioSegment* audio = track->GetData<AudioSegment>();
674 AudioSegment output;
676 TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
678 // We don't update Track->mTracksStartTime here to account for time spent
679 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
680 // the blocked period has completed. But we do need to make sure we play
681 // from the right offsets in the track buffer, even if we've already
682 // written silence for some amount of blocked time after the current time.
683 GraphTime t = aPlayedTime;
684 while (t < mStateComputedTime) {
685 bool blocked = t >= track->mStartBlocking;
686 GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
687 NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
689 // Check how many ticks of sound we can provide if we are blocked some
690 // time in the middle of this cycle.
691 TrackTime toWrite = end - t;
693 if (blocked) {
694 output.InsertNullDataAtStart(toWrite);
695 ticksWritten += toWrite;
696 LOG(LogLevel::Verbose,
697 ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
698 "%f to %f (%" PRId64 " to %" PRId64 ")",
699 this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
700 offset, offset + toWrite));
701 } else {
702 TrackTime endTicksNeeded = offset + toWrite;
703 TrackTime endTicksAvailable = audio->GetDuration();
705 if (endTicksNeeded <= endTicksAvailable) {
706 LOG(LogLevel::Verbose,
707 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
708 "(samples %" PRId64 " to %" PRId64 ")",
709 this, track, toWrite, MediaTimeToSeconds(t),
710 MediaTimeToSeconds(end), offset, endTicksNeeded));
711 output.AppendSlice(*audio, offset, endTicksNeeded);
712 ticksWritten += toWrite;
713 offset = endTicksNeeded;
714 } else {
715 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
716 // ended."); If we are at the end of the track, maybe write the
717 // remaining samples, and pad with/output silence.
718 if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
719 output.AppendSlice(*audio, offset, endTicksAvailable);
721 LOG(LogLevel::Verbose,
722 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
723 "(samples %" PRId64 " to %" PRId64 ")",
724 this, track, toWrite, MediaTimeToSeconds(t),
725 MediaTimeToSeconds(end), offset, endTicksNeeded));
726 uint32_t available = endTicksAvailable - offset;
727 ticksWritten += available;
728 toWrite -= available;
729 offset = endTicksAvailable;
731 output.AppendNullData(toWrite);
732 LOG(LogLevel::Verbose,
733 ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
734 "%f (samples %" PRId64 " to %" PRId64 ")",
735 this, track, toWrite, MediaTimeToSeconds(t),
736 MediaTimeToSeconds(end), offset, endTicksNeeded));
737 ticksWritten += toWrite;
739 output.ApplyVolume(mGlobalVolume * aOutput.mVolume);
741 t = end;
743 output.Mix(mMixer, aOutputChannelCount, mSampleRate);
745 return ticksWritten;
748 DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread(
749 CubebUtils::AudioDeviceID aID) {
750 MOZ_ASSERT(NS_IsMainThread());
751 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
752 return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
755 NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() {
756 MOZ_ASSERT(NS_IsMainThread());
757 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
758 return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
761 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
762 MOZ_ASSERT(OnGraphThread());
763 LOG(LogLevel::Debug,
764 ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
766 mDeviceInputTrackManagerGraphThread.Add(aTrack);
768 if (aTrack->AsNativeInputTrack()) {
769 // Switch Drivers since we're adding input (to input-only or full-duplex)
770 AudioCallbackDriver* driver = new AudioCallbackDriver(
771 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
772 AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
773 aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId));
774 LOG(LogLevel::Debug,
775 ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
776 this, driver));
777 SwitchAtNextIteration(driver);
778 } else {
779 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
780 MOZ_ASSERT(nonNative);
781 // Start non-native input right away.
782 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
783 MakeRefPtr<AudioInputSourceListener>(nonNative),
784 nonNative->GenerateSourceId(), nonNative->mDeviceId,
785 AudioInputChannelCount(nonNative->mDeviceId),
786 AudioInputDevicePreference(nonNative->mDeviceId) ==
787 AudioInputType::Voice,
788 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
792 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
793 MOZ_ASSERT(NS_IsMainThread());
794 MOZ_ASSERT(aTrack);
796 LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
797 aTrack, aTrack->mDeviceId));
799 class Message : public ControlMessage {
800 public:
801 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
802 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
803 void Run() override {
804 TRACE("MTG::OpenAudioInputImpl ControlMessage");
805 mGraph->OpenAudioInputImpl(mInputTrack);
807 MediaTrackGraphImpl* mGraph;
808 DeviceInputTrack* mInputTrack;
811 mDeviceInputTrackManagerMainThread.Add(aTrack);
813 this->AppendMessage(MakeUnique<Message>(this, aTrack));
816 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
817 MOZ_ASSERT(OnGraphThread());
819 LOG(LogLevel::Debug,
820 ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
822 if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
823 nonNative->StopAudio();
824 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
825 return;
828 MOZ_ASSERT(aTrack->AsNativeInputTrack());
830 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
832 // Switch Drivers since we're adding or removing an input (to nothing/system
833 // or output only)
834 bool audioTrackPresent = AudioTrackPresent();
836 GraphDriver* driver;
837 if (audioTrackPresent) {
838 // We still have audio output
839 LOG(LogLevel::Debug,
840 ("%p: CloseInput: output present (AudioCallback)", this));
842 driver = new AudioCallbackDriver(
843 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
844 AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
845 nullptr, AudioInputDevicePreference(aTrack->mDeviceId));
846 SwitchAtNextIteration(driver);
847 } else if (CurrentDriver()->AsAudioCallbackDriver()) {
848 LOG(LogLevel::Debug,
849 ("%p: CloseInput: no output present (SystemClockCallback)", this));
851 driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
852 SwitchAtNextIteration(driver);
853 } // else SystemClockDriver->SystemClockDriver, no switch
856 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
857 MOZ_ASSERT(OnGraphThreadOrNotRunning());
858 mOutputDevices.RemoveElementsBy([&](OutputDeviceEntry& aDeviceRef) {
859 aDeviceRef.mTrackOutputs.RemoveElement(aTrack);
860 // mReceiver is null for the primary output device, which is retained for
861 // AudioCallbackDriver output even when no tracks have audio outputs.
862 return aDeviceRef.mTrackOutputs.IsEmpty() && aDeviceRef.mReceiver;
866 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
867 MOZ_ASSERT(NS_IsMainThread());
868 MOZ_ASSERT(aTrack);
870 LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
871 this, aTrack, aTrack->mDeviceId));
873 class Message : public ControlMessage {
874 public:
875 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
876 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
877 void Run() override {
878 TRACE("MTG::CloseAudioInputImpl ControlMessage");
879 mGraph->CloseAudioInputImpl(mInputTrack);
881 MediaTrackGraphImpl* mGraph;
882 DeviceInputTrack* mInputTrack;
885 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
886 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
887 // for more details.
888 mDeviceInputTrackManagerMainThread.Remove(aTrack);
890 this->AppendMessage(MakeUnique<Message>(this, aTrack));
892 if (aTrack->AsNativeInputTrack()) {
893 LOG(LogLevel::Debug,
894 ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
895 SetNewNativeInput();
899 // All AudioInput listeners get the same speaker data (at least for now).
900 void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk& aChunk) {
901 if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
902 return;
905 #if defined(MOZ_WEBRTC)
906 for (const auto& track : mTracks) {
907 if (const auto& t = track->AsAudioProcessingTrack()) {
908 t->NotifyOutputData(this, aChunk);
911 #endif
914 void MediaTrackGraphImpl::NotifyInputStopped() {
915 NativeInputTrack* native =
916 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
917 if (!native) {
918 return;
920 native->NotifyInputStopped(this);
923 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
924 size_t aFrames, TrackRate aRate,
925 uint32_t aChannels,
926 uint32_t aAlreadyBuffered) {
927 // Either we have an audio input device, or we just removed the audio input
928 // this iteration, and we're switching back to an output-only driver next
929 // iteration.
930 NativeInputTrack* native =
931 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
932 MOZ_ASSERT(native || Switching());
933 if (!native) {
934 return;
936 native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
937 aAlreadyBuffered);
940 void MediaTrackGraphImpl::DeviceChangedImpl() {
941 MOZ_ASSERT(OnGraphThread());
942 NativeInputTrack* native =
943 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
944 if (!native) {
945 return;
947 native->DeviceChanged(this);
950 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
951 MOZ_ASSERT(OnGraphThread());
952 mMaxOutputChannelCount = aMaxChannelCount;
955 void MediaTrackGraphImpl::DeviceChanged() {
956 // This is safe to be called from any thread: this message comes from an
957 // underlying platform API, and we don't have much guarantees. If it is not
958 // called from the main thread (and it probably will rarely be), it will post
959 // itself to the main thread, and the actual device change message will be ran
960 // and acted upon on the graph thread.
961 if (!NS_IsMainThread()) {
962 RefPtr<nsIRunnable> runnable = WrapRunnable(
963 RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
964 mMainThread->Dispatch(runnable.forget());
965 return;
968 class Message : public ControlMessage {
969 public:
970 explicit Message(MediaTrackGraph* aGraph)
971 : ControlMessage(nullptr),
972 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
973 void Run() override {
974 TRACE("MTG::DeviceChangeImpl ControlMessage");
975 mGraphImpl->DeviceChangedImpl();
977 // We know that this is valid, because the graph can't shutdown if it has
978 // messages.
979 MediaTrackGraphImpl* mGraphImpl;
982 if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
983 // This is a special case where the origin of this event cannot control the
984 // lifetime of the graph, because the graph is controling the lifetime of
985 // the AudioCallbackDriver where the event originated.
986 // We know the graph is soon going away, so there's no need to notify about
987 // this device change.
988 return;
991 // Reset the latency, it will get fetched again next time it's queried.
992 MOZ_ASSERT(NS_IsMainThread());
993 mAudioOutputLatency = 0.0;
995 // Dispatch to the bg thread to do the (potentially expensive) query of the
996 // maximum channel count, and then dispatch back to the main thread, then to
997 // the graph, with the new info.
998 RefPtr<MediaTrackGraphImpl> self = this;
999 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1000 "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
1001 uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
1002 self->Dispatch(NS_NewRunnableFunction(
1003 "MaxChannelCountUpdateToMainThread",
1004 [self{self}, maxChannelCount]() {
1005 class MessageToGraph : public ControlMessage {
1006 public:
1007 explicit MessageToGraph(MediaTrackGraph* aGraph,
1008 uint32_t aMaxChannelCount)
1009 : ControlMessage(nullptr),
1010 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
1011 mMaxChannelCount(aMaxChannelCount) {}
1012 void Run() override {
1013 TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1014 mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
1016 MediaTrackGraphImpl* mGraphImpl;
1017 uint32_t mMaxChannelCount;
1019 self->AppendMessage(
1020 MakeUnique<MessageToGraph>(self, maxChannelCount));
1021 }));
1022 }));
1024 AppendMessage(MakeUnique<Message>(this));
1027 static const char* GetAudioInputTypeString(const AudioInputType& aType) {
1028 return aType == AudioInputType::Voice ? "Voice" : "Unknown";
1031 void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1032 MOZ_ASSERT(OnGraphThread());
1033 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
1034 impl->ReevaluateInputDevice(aID);
1037 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1038 MOZ_ASSERT(OnGraphThread());
1039 LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
1041 DeviceInputTrack* track =
1042 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
1043 if (!track) {
1044 LOG(LogLevel::Debug,
1045 ("%p: No DeviceInputTrack for this device. Ignore", this));
1046 return;
1049 bool needToSwitch = false;
1051 if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
1052 if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
1053 LOG(LogLevel::Debug,
1054 ("%p: %u-channel non-native input device %p (track %p) is "
1055 "re-configured to %d-channel",
1056 this, nonNative->NumberOfChannels(), aID, track,
1057 AudioInputChannelCount(aID)));
1058 needToSwitch = true;
1060 if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
1061 LOG(LogLevel::Debug,
1062 ("%p: %s-type non-native input device %p (track %p) is re-configured "
1063 "to %s-type",
1064 this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
1065 track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1066 needToSwitch = true;
1069 if (needToSwitch) {
1070 nonNative->StopAudio();
1071 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
1072 MakeRefPtr<AudioInputSourceListener>(nonNative),
1073 nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
1074 AudioInputDevicePreference(aID) == AudioInputType::Voice,
1075 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
1078 return;
1081 MOZ_ASSERT(track->AsNativeInputTrack());
1083 if (AudioCallbackDriver* audioCallbackDriver =
1084 CurrentDriver()->AsAudioCallbackDriver()) {
1085 if (audioCallbackDriver->InputChannelCount() !=
1086 AudioInputChannelCount(aID)) {
1087 LOG(LogLevel::Debug,
1088 ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1089 "re-configured to %d-channel",
1090 this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
1091 AudioInputChannelCount(aID)));
1092 needToSwitch = true;
1094 if (audioCallbackDriver->InputDevicePreference() !=
1095 AudioInputDevicePreference(aID)) {
1096 LOG(LogLevel::Debug,
1097 ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1098 "re-configured to %s-type",
1099 this,
1100 GetAudioInputTypeString(
1101 audioCallbackDriver->InputDevicePreference()),
1102 audioCallbackDriver,
1103 GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1104 needToSwitch = true;
1106 } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1107 // We're already in the process of switching to a audio callback driver,
1108 // which will happen at the next iteration.
1109 // However, maybe it's not the correct number of channels. Re-query the
1110 // correct channel amount at this time.
1111 needToSwitch = true;
1114 if (needToSwitch) {
1115 AudioCallbackDriver* newDriver = new AudioCallbackDriver(
1116 this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
1117 AudioInputChannelCount(aID), PrimaryOutputDeviceID(), aID,
1118 AudioInputDevicePreference(aID));
1119 SwitchAtNextIteration(newDriver);
1123 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1124 // either we're on the right thread (and calling CurrentDriver() is safe),
1125 // or we're going to fail the assert anyway, so don't cross-check
1126 // via CurrentDriver().
1127 return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
1130 bool MediaTrackGraphImpl::OnGraphThread() const {
1131 // we're on the right thread (and calling mDriver is safe),
1132 MOZ_ASSERT(mDriver);
1133 if (mGraphRunner && mGraphRunner->OnThread()) {
1134 return true;
1136 return mDriver->OnThread();
1139 bool MediaTrackGraphImpl::Destroyed() const {
1140 MOZ_ASSERT(NS_IsMainThread());
1141 return !mSelfRef;
1144 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1145 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1146 if (mRealtime) {
1147 return true;
1150 TimeStamp now = TimeStamp::Now();
1151 // For offline graphs, update now if it has been long enough since the last
1152 // update, or if it has reached the end.
1153 if ((now - mLastMainThreadUpdate).ToMilliseconds() >
1154 CurrentDriver()->IterationDuration() ||
1155 mStateComputedTime >= mEndTime) {
1156 mLastMainThreadUpdate = now;
1157 return true;
1159 return false;
1162 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
1163 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1164 mMonitor.AssertCurrentThreadOwns();
1166 // We don't want to frequently update the main thread about timing update
1167 // when we are not running in realtime.
1168 if (aFinalUpdate || ShouldUpdateMainThread()) {
1169 // Strip updates that will be obsoleted below, so as to keep the length of
1170 // mTrackUpdates sane.
1171 size_t keptUpdateCount = 0;
1172 for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
1173 MediaTrack* track = mTrackUpdates[i].mTrack;
1174 // RemoveTrackGraphThread() clears mTrack in updates for
1175 // tracks that are removed from the graph.
1176 MOZ_ASSERT(!track || track->GraphImpl() == this);
1177 if (!track || track->MainThreadNeedsUpdates()) {
1178 // Discard this update as it has either been cleared when the track
1179 // was destroyed or there will be a newer update below.
1180 continue;
1182 if (keptUpdateCount != i) {
1183 mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
1184 MOZ_ASSERT(!mTrackUpdates[i].mTrack);
1186 ++keptUpdateCount;
1188 mTrackUpdates.TruncateLength(keptUpdateCount);
1190 mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
1191 mSuspendedTracks.Length());
1192 for (MediaTrack* track : AllTracks()) {
1193 if (!track->MainThreadNeedsUpdates()) {
1194 continue;
1196 TrackUpdate* update = mTrackUpdates.AppendElement();
1197 update->mTrack = track;
1198 // No blocking to worry about here, since we've passed
1199 // UpdateCurrentTimeForTracks.
1200 update->mNextMainThreadCurrentTime =
1201 track->GraphTimeToTrackTime(mProcessedTime);
1202 update->mNextMainThreadEnded = track->mNotifiedEnded;
1204 mNextMainThreadGraphTime = mProcessedTime;
1205 if (!mPendingUpdateRunnables.IsEmpty()) {
1206 mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
1210 // If this is the final update, then a stable state event will soon be
1211 // posted just before this thread finishes, and so there is no need to also
1212 // post here.
1213 if (!aFinalUpdate &&
1214 // Don't send the message to the main thread if it's not going to have
1215 // any work to do.
1216 !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
1217 EnsureStableStateEventPosted();
1221 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
1222 if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
1223 return aTime;
1225 return RoundUpToNextAudioBlock(aTime);
1228 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
1229 uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
1230 uint64_t nextBlock = block + 1;
1231 GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1232 return nextTime;
1235 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1236 uint32_t aTrackIndex, TrackRate aSampleRate) {
1237 MOZ_ASSERT(OnGraphThread());
1238 MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
1239 "Cycle breaker is not AudioNodeTrack?");
1241 while (mProcessedTime < mStateComputedTime) {
1242 // Microtask checkpoints are in between render quanta.
1243 nsAutoMicroTask mt;
1245 GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
1246 for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
1247 auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
1248 MOZ_ASSERT(nt->AsAudioNodeTrack());
1249 nt->ProduceOutputBeforeInput(mProcessedTime);
1251 for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
1252 ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
1253 if (pt) {
1254 pt->ProcessInput(
1255 mProcessedTime, next,
1256 (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
1259 mProcessedTime = next;
1261 NS_ASSERTION(mProcessedTime == mStateComputedTime,
1262 "Something went wrong with rounding to block boundaries");
1265 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1266 UniquePtr<ControlMessageInterface> aMessage) {
1267 MOZ_ASSERT(OnGraphThread());
1269 if (mFrontMessageQueue.IsEmpty()) {
1270 mFrontMessageQueue.AppendElement();
1273 // Only one block is used for messages from the graph thread.
1274 MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1275 mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
1278 void MediaTrackGraphImpl::RunMessagesInQueue() {
1279 TRACE("MTG::RunMessagesInQueue");
1280 MOZ_ASSERT(OnGraphThread());
1281 // Calculate independent action times for each batch of messages (each
1282 // batch corresponding to an event loop task). This isolates the performance
1283 // of different scripts to some extent.
1284 for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1285 nsTArray<UniquePtr<ControlMessageInterface>>& messages =
1286 mFrontMessageQueue[i].mMessages;
1288 for (uint32_t j = 0; j < messages.Length(); ++j) {
1289 TRACE("ControlMessage::Run");
1290 messages[j]->Run();
1293 mFrontMessageQueue.Clear();
1296 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
1297 TRACE("MTG::UpdateGraph");
1298 MOZ_ASSERT(OnGraphThread());
1299 MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1300 // The next state computed time can be the same as the previous: it
1301 // means the driver would have been blocking indefinitly, but the graph has
1302 // been woken up right after having been to sleep.
1303 MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1305 CheckDriver();
1306 UpdateTrackOrder();
1308 // Always do another iteration if there are tracks waiting to resume.
1309 bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
1311 for (MediaTrack* track : mTracks) {
1312 if (SourceMediaTrack* is = track->AsSourceTrack()) {
1313 ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
1314 is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
1316 if (track->mEnded) {
1317 // The track's not suspended, and since it's ended, underruns won't
1318 // stop it playing out. So there's no blocking other than what we impose
1319 // here.
1320 GraphTime endTime = track->GetEnd() + track->mStartTime;
1321 if (endTime <= mStateComputedTime) {
1322 LOG(LogLevel::Verbose,
1323 ("%p: MediaTrack %p is blocked due to being ended", this, track));
1324 track->mStartBlocking = mStateComputedTime;
1325 } else {
1326 LOG(LogLevel::Verbose,
1327 ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1328 "time %f, end at %f)",
1329 this, track, MediaTimeToSeconds(mStateComputedTime),
1330 MediaTimeToSeconds(endTime)));
1331 // Data can't be added to a ended track, so underruns are irrelevant.
1332 MOZ_ASSERT(endTime <= aEndBlockingDecisions);
1333 track->mStartBlocking = endTime;
1335 } else {
1336 track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
1338 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1339 if (SourceMediaTrack* s = track->AsSourceTrack()) {
1340 if (s->Ended()) {
1341 continue;
1344 MutexAutoLock lock(s->mMutex);
1345 if (!s->mUpdateTrack->mPullingEnabled) {
1346 // The invariant that data must be provided is only enforced when
1347 // pulling.
1348 continue;
1351 if (track->GetEnd() <
1352 track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
1353 LOG(LogLevel::Error,
1354 ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1355 "but wasn't fed "
1356 "enough data. TrackListeners=%zu. Track-end=%f, "
1357 "Iteration-end=%f",
1358 this, track,
1359 (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
1360 track->mTrackListeners.Length(),
1361 MediaTimeToSeconds(track->GetEnd()),
1362 MediaTimeToSeconds(
1363 track->GraphTimeToTrackTime(aEndBlockingDecisions))));
1364 MOZ_DIAGNOSTIC_ASSERT(false,
1365 "A non-ended SourceMediaTrack wasn't fed "
1366 "enough data by NotifyPull");
1369 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1373 for (MediaTrack* track : mSuspendedTracks) {
1374 track->mStartBlocking = mStateComputedTime;
1377 // If the loop is woken up so soon that IterationEnd() barely advances or
1378 // if an offline graph is not currently rendering, we end up having
1379 // aEndBlockingDecisions == mStateComputedTime.
1380 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1381 // empty, Process() will not find any unblocked track and so will not
1382 // ensure another iteration. If the graph should be rendering, then ensure
1383 // another iteration to render.
1384 if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
1385 mStateComputedTime < mEndTime)) {
1386 EnsureNextIteration();
1390 void MediaTrackGraphImpl::SelectOutputDeviceForAEC() {
1391 MOZ_ASSERT(OnGraphThread());
1392 size_t currentDeviceIndex = mOutputDevices.IndexOf(mOutputDeviceForAEC);
1393 if (currentDeviceIndex == mOutputDevices.NoIndex) {
1394 // Outputs for this device have been removed.
1395 // Fall back to the primary output device.
1396 mOutputDeviceForAEC = PrimaryOutputDeviceID();
1397 currentDeviceIndex = 0;
1398 MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC);
1400 if (mOutputDevices.Length() == 1) {
1401 // No other output devices so there is no choice.
1402 return;
1405 // The output is considered silent intentionally only if the whole duration
1406 // (often more than just this processing interval) of audio data in the
1407 // MediaSegment is null so as to reduce switching between output devices
1408 // should there be short durations of silence.
1409 auto HasNonNullAudio = [](const TrackAndVolume& aTV) {
1410 return aTV.mVolume != 0 && !aTV.mTrack->IsSuspended() &&
1411 !aTV.mTrack->GetData()->IsNull();
1413 // Keep using the same output device stream if it has non-null data,
1414 // so as to stay with a stream having ongoing audio. If the output stream
1415 // is switched, the echo cancellation algorithm can take some time to adjust
1416 // to the change in delay, so there is less value in switching back and
1417 // forth between output devices for very short sounds.
1418 for (const auto& output : mOutputDevices[currentDeviceIndex].mTrackOutputs) {
1419 if (HasNonNullAudio(output)) {
1420 return;
1423 // The current output device is silent. Use another if it has non-null data.
1424 for (const auto& outputDeviceEntry : mOutputDevices) {
1425 for (const auto& output : outputDeviceEntry.mTrackOutputs) {
1426 if (HasNonNullAudio(output)) {
1427 // Switch to this device.
1428 mOutputDeviceForAEC = outputDeviceEntry.mDeviceID;
1429 return;
1433 // Null data for all outputs. Keep using the same device.
1436 void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) {
1437 TRACE("MTG::Process");
1438 MOZ_ASSERT(OnGraphThread());
1439 // Play track contents.
1440 bool allBlockedForever = true;
1441 // True when we've done ProcessInput for all processed tracks.
1442 bool doneAllProducing = false;
1443 const GraphTime oldProcessedTime = mProcessedTime;
1445 // Figure out what each track wants to do
1446 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
1447 MediaTrack* track = mTracks[i];
1448 if (!doneAllProducing) {
1449 ProcessedMediaTrack* pt = track->AsProcessedTrack();
1450 if (pt) {
1451 AudioNodeTrack* n = track->AsAudioNodeTrack();
1452 if (n) {
1453 #ifdef DEBUG
1454 // Verify that the sampling rate for all of the following tracks is
1455 // the same
1456 for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
1457 AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
1458 if (nextTrack) {
1459 MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
1460 "All AudioNodeTracks in the graph must have the same "
1461 "sampling rate");
1464 #endif
1465 // Since an AudioNodeTrack is present, go ahead and
1466 // produce audio block by block for all the rest of the tracks.
1467 ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
1468 doneAllProducing = true;
1469 } else {
1470 pt->ProcessInput(mProcessedTime, mStateComputedTime,
1471 ProcessedMediaTrack::ALLOW_END);
1472 // Assert that a live track produced enough data
1473 MOZ_ASSERT_IF(!track->mEnded,
1474 track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1475 track, mStateComputedTime));
1479 if (track->mStartBlocking > oldProcessedTime) {
1480 allBlockedForever = false;
1483 mProcessedTime = mStateComputedTime;
1485 SelectOutputDeviceForAEC();
1486 for (const auto& outputDeviceEntry : mOutputDevices) {
1487 uint32_t outputChannelCount;
1488 if (!outputDeviceEntry.mReceiver) { // primary output
1489 if (!aMixerReceiver) {
1490 // Running off a system clock driver. No need to mix output.
1491 continue;
1493 MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(),
1494 "Driver must be AudioCallbackDriver if aMixerReceiver");
1495 // Use the number of channel the driver expects: this is the number of
1496 // channel that can be output by the underlying system level audio stream.
1497 outputChannelCount =
1498 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
1499 } else {
1500 outputChannelCount = AudioOutputChannelCount(outputDeviceEntry);
1502 MOZ_ASSERT(mRealtime,
1503 "If there's an output device, this graph must be realtime");
1504 mMixer.StartMixing();
1505 // This is the number of frames that are written to the output buffer, for
1506 // this iteration.
1507 TrackTime ticksPlayed = 0;
1508 for (const auto& t : outputDeviceEntry.mTrackOutputs) {
1509 TrackTime ticksPlayedForThisTrack =
1510 PlayAudio(t, oldProcessedTime, outputChannelCount);
1511 if (ticksPlayed == 0) {
1512 ticksPlayed = ticksPlayedForThisTrack;
1513 } else {
1514 MOZ_ASSERT(
1515 !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
1516 "Each track should have the same number of frames.");
1520 if (ticksPlayed == 0) {
1521 // Nothing was played, so the mixer doesn't know how many frames were
1522 // processed. We still tell it so AudioCallbackDriver knows how much has
1523 // been processed. (bug 1406027)
1524 mMixer.Mix(nullptr, outputChannelCount,
1525 mStateComputedTime - oldProcessedTime, mSampleRate);
1527 AudioChunk* outputChunk = mMixer.MixedChunk();
1528 if (outputDeviceEntry.mDeviceID == mOutputDeviceForAEC) {
1529 // Callback any observers for the AEC speaker data. Note that one
1530 // (maybe) of these will be full-duplex, the others will get their input
1531 // data off separate cubeb callbacks.
1532 NotifyOutputData(*outputChunk);
1534 if (!outputDeviceEntry.mReceiver) { // primary output
1535 aMixerReceiver->MixerCallback(outputChunk, mSampleRate);
1536 } else {
1537 outputDeviceEntry.mReceiver->EnqueueAudio(*outputChunk);
1541 if (!allBlockedForever) {
1542 EnsureNextIteration();
1546 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1547 MOZ_ASSERT(OnGraphThread());
1548 if (mForceShutDownReceived) {
1549 for (MediaTrack* track : AllTracks()) {
1550 track->OnGraphThreadDone();
1554 MonitorAutoLock lock(mMonitor);
1555 bool finalUpdate =
1556 mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
1557 PrepareUpdatesToMainThreadState(finalUpdate);
1558 if (!finalUpdate) {
1559 SwapMessageQueues();
1560 return true;
1562 // The JSContext will not be used again.
1563 // Clear main thread access while under monitor.
1564 mJSContext = nullptr;
1566 dom::WorkletThread::DeleteCycleCollectedJSContext();
1567 // Enter shutdown mode when this iteration is completed.
1568 // No need to Destroy tracks here. The main-thread owner of each
1569 // track is responsible for calling Destroy on them.
1570 return false;
1573 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime,
1574 GraphTime aIterationEnd,
1575 MixerCallbackReceiver* aMixerReceiver)
1576 -> IterationResult {
1577 if (mGraphRunner) {
1578 return mGraphRunner->OneIteration(aStateTime, aIterationEnd,
1579 aMixerReceiver);
1582 return OneIterationImpl(aStateTime, aIterationEnd, aMixerReceiver);
1585 auto MediaTrackGraphImpl::OneIterationImpl(
1586 GraphTime aStateTime, GraphTime aIterationEnd,
1587 MixerCallbackReceiver* aMixerReceiver) -> IterationResult {
1588 TRACE("MTG::OneIterationImpl");
1590 mIterationEndTime = aIterationEnd;
1592 if (SoftRealTimeLimitReached()) {
1593 TRACE("MTG::Demoting real-time thread!");
1594 DemoteThreadFromRealTime();
1597 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1598 // thread, and so the monitor need not be held to check mLifecycleState.
1599 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1600 // graphs that have not started.
1602 // While changes occur on mainthread, this assert confirms that
1603 // this code shouldn't run if mainthread might be changing the state (to
1604 // > LIFECYCLE_RUNNING)
1606 // Ignore mutex warning: static during execution of the graph
1607 MOZ_PUSH_IGNORE_THREAD_SAFETY
1608 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
1609 MOZ_POP_THREAD_SAFETY
1611 MOZ_ASSERT(OnGraphThread());
1613 WebCore::DenormalDisabler disabler;
1615 // Process graph message from the main thread for this iteration.
1616 RunMessagesInQueue();
1618 // Process MessagePort events.
1619 // These require a single thread, which has an nsThread with an event queue.
1620 if (mGraphRunner || !mRealtime) {
1621 TRACE("MTG::MessagePort events");
1622 NS_ProcessPendingEvents(nullptr);
1625 GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
1626 UpdateGraph(stateTime);
1628 mStateComputedTime = stateTime;
1630 GraphTime oldProcessedTime = mProcessedTime;
1631 Process(aMixerReceiver);
1632 MOZ_ASSERT(mProcessedTime == stateTime);
1634 UpdateCurrentTimeForTracks(oldProcessedTime);
1636 ProcessChunkMetadata(oldProcessedTime);
1638 // Process graph messages queued from RunMessageAfterProcessing() on this
1639 // thread during the iteration.
1640 RunMessagesInQueue();
1642 if (!UpdateMainThreadState()) {
1643 if (Switching()) {
1644 // We'll never get to do this switch. Clear mNextDriver to break the
1645 // ref-cycle graph->nextDriver->currentDriver->graph.
1646 SwitchAtNextIteration(nullptr);
1648 return IterationResult::CreateStop(
1649 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1650 &MediaTrackGraphImpl::SignalMainThreadCleanup));
1653 if (Switching()) {
1654 RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
1655 return IterationResult::CreateSwitchDriver(
1656 nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
1657 "MediaTrackGraphImpl::SetCurrentDriver", this,
1658 &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
1661 return IterationResult::CreateStillProcessing();
1664 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
1665 MOZ_ASSERT(NS_IsMainThread());
1666 mMonitor.AssertCurrentThreadOwns();
1668 MediaTrack* track = aUpdate->mTrack;
1669 if (!track) return;
1670 track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1671 track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
1673 if (track->ShouldNotifyTrackEnded()) {
1674 track->NotifyMainThreadListeners();
1678 void MediaTrackGraphImpl::ForceShutDown() {
1679 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1680 LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
1682 if (mShutdownBlocker) {
1683 // Avoid waiting forever for a graph to shut down
1684 // synchronously. Reports are that some 3rd-party audio drivers
1685 // occasionally hang in shutdown (both for us and Chrome).
1686 NS_NewTimerWithCallback(
1687 getter_AddRefs(mShutdownTimer), this,
1688 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1689 nsITimer::TYPE_ONE_SHOT);
1692 class Message final : public ControlMessage {
1693 public:
1694 explicit Message(MediaTrackGraphImpl* aGraph)
1695 : ControlMessage(nullptr), mGraph(aGraph) {}
1696 void Run() override {
1697 TRACE("MTG::ForceShutdown ControlMessage");
1698 mGraph->mForceShutDownReceived = true;
1700 // The graph owns this message.
1701 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
1704 if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
1705 // If both the track and port counts are zero, the regular shutdown
1706 // sequence will progress shortly to shutdown threads and destroy the graph.
1707 AppendMessage(MakeUnique<Message>(this));
1708 InterruptJS();
1712 NS_IMETHODIMP
1713 MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
1714 MOZ_ASSERT(NS_IsMainThread());
1715 MOZ_ASSERT(!mShutdownBlocker, "MediaTrackGraph took too long to shut down!");
1716 // Sigh, graph took too long to shut down. Stop blocking system
1717 // shutdown and hope all is well.
1718 RemoveShutdownBlocker();
1719 return NS_OK;
1722 static nsCString GetDocumentTitle(uint64_t aWindowID) {
1723 MOZ_ASSERT(NS_IsMainThread());
1724 nsCString title;
1725 auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID);
1726 if (!win) {
1727 return title;
1729 Document* doc = win->GetExtantDoc();
1730 if (!doc) {
1731 return title;
1733 nsAutoString titleUTF16;
1734 doc->GetTitle(titleUTF16);
1735 CopyUTF16toUTF8(titleUTF16, title);
1736 return title;
1739 NS_IMETHODIMP
1740 MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic,
1741 const char16_t* aData) {
1742 MOZ_ASSERT(NS_IsMainThread());
1743 MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0);
1744 nsCString streamName = GetDocumentTitle(mWindowID);
1745 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
1746 if (streamName.IsEmpty()) {
1747 return NS_OK;
1749 QueueControlMessageWithNoShutdown(
1750 [self = RefPtr{this}, this, streamName = std::move(streamName)] {
1751 CurrentDriver()->SetStreamName(streamName);
1753 return NS_OK;
1756 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1757 MOZ_ASSERT(NS_IsMainThread());
1758 MOZ_ASSERT(!mShutdownBlocker);
1760 class Blocker : public media::ShutdownBlocker {
1761 const RefPtr<MediaTrackGraphImpl> mGraph;
1763 public:
1764 Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
1765 : media::ShutdownBlocker(aName), mGraph(aGraph) {}
1767 NS_IMETHOD
1768 BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
1769 mGraph->ForceShutDown();
1770 return NS_OK;
1774 nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
1775 if (!barrier) {
1776 // We're already shutting down, we won't be able to add a blocker, bail.
1777 LOG(LogLevel::Error,
1778 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1779 this));
1780 return false;
1783 // Blocker names must be distinct.
1784 nsString blockerName;
1785 blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
1786 mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
1787 nsresult rv = barrier->AddBlocker(mShutdownBlocker,
1788 NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
1789 __LINE__, u"MediaTrackGraph shutdown"_ns);
1790 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
1791 return true;
1794 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1795 if (!mShutdownBlocker) {
1796 return;
1798 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
1799 mShutdownBlocker = nullptr;
1802 NS_IMETHODIMP
1803 MediaTrackGraphImpl::GetName(nsACString& aName) {
1804 aName.AssignLiteral("MediaTrackGraphImpl");
1805 return NS_OK;
1808 namespace {
1810 class MediaTrackGraphShutDownRunnable : public Runnable {
1811 public:
1812 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
1813 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
1814 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1815 // See bug 1535398.
1816 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
1817 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1818 MOZ_ASSERT(NS_IsMainThread());
1819 MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
1820 "We should know the graph thread control loop isn't running!");
1822 LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
1824 // We've asserted the graph isn't running. Use mDriver instead of
1825 // CurrentDriver to avoid thread-safety checks
1826 #if 0 // AudioCallbackDrivers are released asynchronously anyways
1827 // XXX a better test would be have setting mGraphDriverRunning make sure
1828 // any current callback has finished and block future ones -- or just
1829 // handle it all in Shutdown()!
1830 if (mGraph->mDriver->AsAudioCallbackDriver()) {
1831 MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1833 #endif
1835 for (MediaTrackGraphImpl::PendingResumeOperation& op :
1836 mGraph->mPendingResumeOperations) {
1837 op.Abort();
1840 if (mGraph->mGraphRunner) {
1841 RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
1844 RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
1846 // Release the driver now so that an AudioCallbackDriver will release its
1847 // SharedThreadPool reference. Each SharedThreadPool reference must be
1848 // released before SharedThreadPool::SpinUntilEmpty() runs on
1849 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
1850 // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1851 // which won't otherwise release its reference on the graph until
1852 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1853 mGraph->SetCurrentDriver(nullptr);
1855 // Safe to access these without the monitor since the graph isn't running.
1856 // We may be one of several graphs. Drop ticket to eventually unblock
1857 // shutdown.
1858 if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
1859 MOZ_ASSERT(
1860 false,
1861 "AudioCallbackDriver took too long to shut down and we let shutdown"
1862 " continue - freezing and leaking");
1864 // The timer fired, so we may be deeper in shutdown now. Block any
1865 // further teardown and just leak, for safety.
1866 return NS_OK;
1869 // mGraph's thread is not running so it's OK to do whatever here
1870 for (MediaTrack* track : mGraph->AllTracks()) {
1871 // Clean up all MediaSegments since we cannot release Images too
1872 // late during shutdown. Also notify listeners that they were removed
1873 // so they can clean up any gfx resources.
1874 track->RemoveAllResourcesAndListenersImpl();
1877 #ifdef DEBUG
1879 MonitorAutoLock lock(mGraph->mMonitor);
1880 MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
1882 #endif
1883 mGraph->mPendingUpdateRunnables.Clear();
1885 mGraph->RemoveShutdownBlocker();
1887 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1888 // stage, since completion of that stage requires all tracks to be freed,
1889 // which requires shutdown to proceed.
1891 if (mGraph->IsEmpty()) {
1892 // mGraph is no longer needed, so delete it.
1893 mGraph->Destroy();
1894 } else {
1895 // The graph is not empty. We must be in a forced shutdown.
1896 // Some later AppendMessage will detect that the graph has
1897 // been emptied, and delete it.
1898 NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
1899 mGraph->LifecycleStateRef() =
1900 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
1902 return NS_OK;
1905 private:
1906 RefPtr<MediaTrackGraphImpl> mGraph;
1909 class MediaTrackGraphStableStateRunnable : public Runnable {
1910 public:
1911 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
1912 bool aSourceIsMTG)
1913 : Runnable("MediaTrackGraphStableStateRunnable"),
1914 mGraph(aGraph),
1915 mSourceIsMTG(aSourceIsMTG) {}
1916 NS_IMETHOD Run() override {
1917 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1918 if (mGraph) {
1919 mGraph->RunInStableState(mSourceIsMTG);
1921 return NS_OK;
1924 private:
1925 RefPtr<MediaTrackGraphImpl> mGraph;
1926 bool mSourceIsMTG;
1930 * Control messages forwarded from main thread to graph manager thread
1932 class CreateMessage : public ControlMessage {
1933 public:
1934 explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
1935 void Run() override {
1936 TRACE("MTG::AddTrackGraphThread ControlMessage");
1937 mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
1939 void RunDuringShutdown() override {
1940 // Make sure to run this message during shutdown too, to make sure
1941 // that we balance the number of tracks registered with the graph
1942 // as they're destroyed during shutdown.
1943 Run();
1947 } // namespace
1949 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
1950 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1952 nsTArray<nsCOMPtr<nsIRunnable>> runnables;
1953 // When we're doing a forced shutdown, pending control messages may be
1954 // run on the main thread via RunDuringShutdown. Those messages must
1955 // run without the graph monitor being held. So, we collect them here.
1956 nsTArray<UniquePtr<ControlMessageInterface>>
1957 controlMessagesToRunDuringShutdown;
1960 MonitorAutoLock lock(mMonitor);
1961 if (aSourceIsMTG) {
1962 MOZ_ASSERT(mPostedRunInStableStateEvent);
1963 mPostedRunInStableStateEvent = false;
1966 // This should be kept in sync with the LifecycleState enum in
1967 // MediaTrackGraphImpl.h
1968 const char* LifecycleState_str[] = {
1969 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
1970 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1971 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1972 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
1974 if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
1975 LOG(LogLevel::Debug,
1976 ("%p: Running stable state callback. Current state: %s", this,
1977 LifecycleState_str[LifecycleStateRef()]));
1980 runnables = std::move(mUpdateRunnables);
1981 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
1982 TrackUpdate* update = &mTrackUpdates[i];
1983 if (update->mTrack) {
1984 ApplyTrackUpdate(update);
1987 mTrackUpdates.Clear();
1989 mMainThreadGraphTime = mNextMainThreadGraphTime;
1991 if (mCurrentTaskMessageQueue.IsEmpty()) {
1992 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1993 IsEmpty()) {
1994 // Complete shutdown. First, ensure that this graph is no longer used.
1995 // A new graph graph will be created if one is needed.
1996 // Asynchronously clean up old graph. We don't want to do this
1997 // synchronously because it spins the event loop waiting for threads
1998 // to shut down, and we don't want to do that in a stable state handler.
1999 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2000 LOG(LogLevel::Debug,
2001 ("%p: Sending MediaTrackGraphShutDownRunnable", this));
2002 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
2003 mMainThread->Dispatch(event.forget());
2005 } else {
2006 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2007 MessageBlock* block = mBackMessageQueue.AppendElement();
2008 block->mMessages = std::move(mCurrentTaskMessageQueue);
2009 EnsureNextIteration();
2012 // If this MediaTrackGraph has entered regular (non-forced) shutdown it
2013 // is not able to process any more messages. Those messages being added to
2014 // the graph in the first place is an error.
2015 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
2016 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
2017 mForceShutDownReceived);
2020 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
2021 // Start the driver now. We couldn't start it earlier because the graph
2022 // might exit immediately on finding it has no tracks. The first message
2023 // for a new graph must create a track. Ensure that his message runs on
2024 // the first iteration.
2025 MOZ_ASSERT(MessagesQueued());
2026 SwapMessageQueues();
2028 LOG(LogLevel::Debug,
2029 ("%p: Starting a graph with a %s", this,
2030 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
2031 : "SystemClockDriver"));
2032 LifecycleStateRef() = LIFECYCLE_RUNNING;
2033 mGraphDriverRunning = true;
2034 RefPtr<GraphDriver> driver = CurrentDriver();
2035 driver->Start();
2036 // It's not safe to Shutdown() a thread from StableState, and
2037 // releasing this may shutdown a SystemClockDriver thread.
2038 // Proxy the release to outside of StableState.
2039 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
2040 driver.forget(),
2041 true); // always proxy
2044 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
2045 mForceShutDownReceived) {
2046 // Defer calls to RunDuringShutdown() to happen while mMonitor is not
2047 // held.
2048 for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
2049 MessageBlock& mb = mBackMessageQueue[i];
2050 controlMessagesToRunDuringShutdown.AppendElements(
2051 std::move(mb.mMessages));
2053 mBackMessageQueue.Clear();
2054 MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
2055 // Stop MediaTrackGraph threads.
2056 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2057 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
2058 mMainThread->Dispatch(event.forget());
2061 mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
2064 // Make sure we get a new current time in the next event loop task
2065 if (!aSourceIsMTG) {
2066 MOZ_ASSERT(mPostedRunInStableState);
2067 mPostedRunInStableState = false;
2070 for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
2071 controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
2074 #ifdef DEBUG
2075 mCanRunMessagesSynchronously =
2076 !mGraphDriverRunning &&
2077 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
2078 #endif
2080 for (uint32_t i = 0; i < runnables.Length(); ++i) {
2081 runnables[i]->Run();
2085 void MediaTrackGraphImpl::EnsureRunInStableState() {
2086 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2088 if (mPostedRunInStableState) return;
2089 mPostedRunInStableState = true;
2090 nsCOMPtr<nsIRunnable> event =
2091 new MediaTrackGraphStableStateRunnable(this, false);
2092 nsContentUtils::RunInStableState(event.forget());
2095 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
2096 MOZ_ASSERT(OnGraphThread());
2097 mMonitor.AssertCurrentThreadOwns();
2099 if (mPostedRunInStableStateEvent) return;
2100 mPostedRunInStableStateEvent = true;
2101 nsCOMPtr<nsIRunnable> event =
2102 new MediaTrackGraphStableStateRunnable(this, true);
2103 mMainThread->Dispatch(event.forget());
2106 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2107 MOZ_ASSERT(mDriver->OnThread());
2109 MonitorAutoLock lock(mMonitor);
2110 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2111 // graphs that have not started.
2112 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
2113 LOG(LogLevel::Debug,
2114 ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2115 LifecycleStateRef() =
2116 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
2117 EnsureStableStateEventPosted();
2120 void MediaTrackGraphImpl::AppendMessage(
2121 UniquePtr<ControlMessageInterface> aMessage) {
2122 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2123 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
2125 if (!mGraphDriverRunning &&
2126 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2127 // The graph control loop is not running and main thread cleanup has
2128 // happened. From now on we can't append messages to
2129 // mCurrentTaskMessageQueue, because that will never be processed again, so
2130 // just RunDuringShutdown this message. This should only happen during
2131 // forced shutdown, or after a non-realtime graph has finished processing.
2132 #ifdef DEBUG
2133 MOZ_ASSERT(mCanRunMessagesSynchronously);
2134 mCanRunMessagesSynchronously = false;
2135 #endif
2136 aMessage->RunDuringShutdown();
2137 #ifdef DEBUG
2138 mCanRunMessagesSynchronously = true;
2139 #endif
2140 if (IsEmpty() &&
2141 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
2142 Destroy();
2144 return;
2147 mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
2148 EnsureRunInStableState();
2151 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
2152 mMainThread->Dispatch(std::move(aRunnable));
2155 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
2156 MediaSegment* aSegment)
2157 : mSampleRate(aSampleRate),
2158 mType(aType),
2159 mSegment(aSegment),
2160 mStartTime(0),
2161 mForgottenTime(0),
2162 mEnded(false),
2163 mNotifiedEnded(false),
2164 mDisabledMode(DisabledTrackMode::ENABLED),
2165 mStartBlocking(GRAPH_TIME_MAX),
2166 mSuspendedCount(0),
2167 mMainThreadCurrentTime(0),
2168 mMainThreadEnded(false),
2169 mEndedNotificationSent(false),
2170 mMainThreadDestroyed(false),
2171 mGraph(nullptr) {
2172 MOZ_COUNT_CTOR(MediaTrack);
2173 MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
2176 MediaTrack::~MediaTrack() {
2177 MOZ_COUNT_DTOR(MediaTrack);
2178 NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
2179 NS_ASSERTION(mMainThreadListeners.IsEmpty(),
2180 "All main thread listeners should have been removed");
2183 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
2184 size_t amount = 0;
2186 // Not owned:
2187 // - mGraph - Not reported here
2188 // - mConsumers - elements
2189 // Future:
2190 // - mLastPlayedVideoFrame
2191 // - mTrackListeners - elements
2193 amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2194 amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2195 amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
2197 return amount;
2200 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
2201 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
2204 void MediaTrack::IncrementSuspendCount() {
2205 ++mSuspendedCount;
2206 if (mSuspendedCount != 1 || !mGraph) {
2207 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2208 return;
2210 AssertOnGraphThreadOrNotRunning();
2211 auto* graph = GraphImpl();
2212 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2213 mConsumers[i]->Suspended();
2215 MOZ_ASSERT(graph->mTracks.Contains(this));
2216 graph->mTracks.RemoveElement(this);
2217 graph->mSuspendedTracks.AppendElement(this);
2218 graph->SetTrackOrderDirty();
2221 void MediaTrack::DecrementSuspendCount() {
2222 MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
2223 --mSuspendedCount;
2224 if (mSuspendedCount != 0 || !mGraph) {
2225 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2226 return;
2228 AssertOnGraphThreadOrNotRunning();
2229 auto* graph = GraphImpl();
2230 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2231 mConsumers[i]->Resumed();
2233 MOZ_ASSERT(graph->mSuspendedTracks.Contains(this));
2234 graph->mSuspendedTracks.RemoveElement(this);
2235 graph->mTracks.AppendElement(this);
2236 graph->SetTrackOrderDirty();
2239 void ProcessedMediaTrack::DecrementSuspendCount() {
2240 mCycleMarker = NOT_VISITED;
2241 MediaTrack::DecrementSuspendCount();
2244 MediaTrackGraphImpl* MediaTrack::GraphImpl() {
2245 return static_cast<MediaTrackGraphImpl*>(mGraph);
2248 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const {
2249 return static_cast<MediaTrackGraphImpl*>(mGraph);
2252 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
2253 MOZ_ASSERT(!mGraph, "Should only be called once");
2254 MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
2255 mGraph = aGraph;
2258 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
2259 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
2260 SetGraphImpl(graph);
2263 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
2264 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2265 aTime <= mStartBlocking,
2266 "Incorrectly ignoring blocking!");
2267 return aTime - mStartTime;
2270 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
2271 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2272 aTime + mStartTime <= mStartBlocking,
2273 "Incorrectly ignoring blocking!");
2274 return aTime + mStartTime;
2277 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
2278 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
2281 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2282 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2284 for (auto& l : mTrackListeners.Clone()) {
2285 l->NotifyRemoved(Graph());
2287 mTrackListeners.Clear();
2289 RemoveAllDirectListenersImpl();
2291 if (mSegment) {
2292 mSegment->Clear();
2296 void MediaTrack::DestroyImpl() {
2297 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2298 mConsumers[i]->Disconnect();
2300 if (mSegment) {
2301 mSegment->Clear();
2303 mGraph = nullptr;
2306 void MediaTrack::Destroy() {
2307 // Keep this track alive until we leave this method
2308 RefPtr<MediaTrack> kungFuDeathGrip = this;
2309 // Keep a reference to the graph, since Message might RunDuringShutdown()
2310 // synchronously and make GraphImpl() invalid.
2311 RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
2313 QueueControlOrShutdownMessage(
2314 [self = RefPtr{this}, this](IsInShutdown aInShutdown) {
2315 if (aInShutdown == IsInShutdown::No) {
2316 OnGraphThreadDone();
2318 TRACE("MediaTrack::Destroy ControlMessage");
2319 RemoveAllResourcesAndListenersImpl();
2320 auto* graph = GraphImpl();
2321 DestroyImpl();
2322 graph->RemoveTrackGraphThread(this);
2324 graph->RemoveTrack(this);
2325 // Message::RunDuringShutdown may have removed this track from the graph,
2326 // but our kungFuDeathGrip above will have kept this track alive if
2327 // necessary.
2328 mMainThreadDestroyed = true;
2331 TrackTime MediaTrack::GetEnd() const {
2332 return mSegment ? mSegment->GetDuration() : 0;
2335 void MediaTrack::AddAudioOutput(void* aKey, const AudioDeviceInfo* aSink) {
2336 MOZ_ASSERT(NS_IsMainThread());
2337 AudioDeviceID deviceID = nullptr;
2338 TrackRate preferredSampleRate = 0;
2339 if (aSink) {
2340 deviceID = aSink->DeviceID();
2341 preferredSampleRate = static_cast<TrackRate>(aSink->DefaultRate());
2343 AddAudioOutput(aKey, deviceID, preferredSampleRate);
2346 void MediaTrack::AddAudioOutput(void* aKey, CubebUtils::AudioDeviceID aDeviceID,
2347 TrackRate aPreferredSampleRate) {
2348 MOZ_ASSERT(NS_IsMainThread());
2349 if (mMainThreadDestroyed) {
2350 return;
2352 LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
2353 GraphImpl()->RegisterAudioOutput(this, aKey, aDeviceID, aPreferredSampleRate);
2356 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
2357 float aVolume) {
2358 MOZ_ASSERT(NS_IsMainThread());
2359 for (auto& params : mAudioOutputParams) {
2360 if (params.mKey == aKey && aTrack == params.mTrack) {
2361 params.mVolume = aVolume;
2362 UpdateAudioOutput(aTrack, params.mDeviceID);
2363 return;
2366 MOZ_CRASH("Audio output key not found when setting the volume.");
2369 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
2370 if (mMainThreadDestroyed) {
2371 return;
2373 GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
2376 void MediaTrack::RemoveAudioOutput(void* aKey) {
2377 MOZ_ASSERT(NS_IsMainThread());
2378 if (mMainThreadDestroyed) {
2379 return;
2381 LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
2382 GraphImpl()->UnregisterAudioOutput(this, aKey);
2385 void MediaTrackGraphImpl::RegisterAudioOutput(
2386 MediaTrack* aTrack, void* aKey, CubebUtils::AudioDeviceID aDeviceID,
2387 TrackRate aPreferredSampleRate) {
2388 MOZ_ASSERT(NS_IsMainThread());
2389 MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey}));
2391 IncrementOutputDeviceRefCnt(aDeviceID, aPreferredSampleRate);
2393 mAudioOutputParams.EmplaceBack(
2394 TrackKeyDeviceAndVolume{aTrack, aKey, aDeviceID, 1.f});
2396 UpdateAudioOutput(aTrack, aDeviceID);
2399 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
2400 void* aKey) {
2401 MOZ_ASSERT(NS_IsMainThread());
2403 size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey});
2404 MOZ_ASSERT(index != mAudioOutputParams.NoIndex);
2405 AudioDeviceID deviceID = mAudioOutputParams[index].mDeviceID;
2406 mAudioOutputParams.UnorderedRemoveElementAt(index);
2408 UpdateAudioOutput(aTrack, deviceID);
2410 DecrementOutputDeviceRefCnt(deviceID);
2413 void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack,
2414 AudioDeviceID aDeviceID) {
2415 MOZ_ASSERT(NS_IsMainThread());
2416 MOZ_ASSERT(!aTrack->IsDestroyed());
2418 float volume = 0.f;
2419 bool found = false;
2420 for (const auto& params : mAudioOutputParams) {
2421 if (params.mTrack == aTrack && params.mDeviceID == aDeviceID) {
2422 volume += params.mVolume;
2423 found = true;
2427 QueueControlMessageWithNoShutdown(
2428 // track has a strong reference to this.
2429 [track = RefPtr{aTrack}, aDeviceID, volume, found] {
2430 TRACE("MediaTrack::UpdateAudioOutput ControlMessage");
2431 MediaTrackGraphImpl* graph = track->GraphImpl();
2432 auto& outputDevicesRef = graph->mOutputDevices;
2433 size_t deviceIndex = outputDevicesRef.IndexOf(aDeviceID);
2434 MOZ_ASSERT(deviceIndex != outputDevicesRef.NoIndex);
2435 auto& deviceOutputsRef = outputDevicesRef[deviceIndex].mTrackOutputs;
2436 if (found) {
2437 for (auto& outputRef : deviceOutputsRef) {
2438 if (outputRef.mTrack == track) {
2439 outputRef.mVolume = volume;
2440 return;
2443 deviceOutputsRef.EmplaceBack(TrackAndVolume{track, volume});
2444 } else {
2445 DebugOnly<bool> removed = deviceOutputsRef.RemoveElement(track);
2446 MOZ_ASSERT(removed);
2447 // mOutputDevices[0] is retained for AudioCallbackDriver output even
2448 // when no tracks have audio outputs.
2449 if (deviceIndex != 0 && deviceOutputsRef.IsEmpty()) {
2450 // The device is no longer in use.
2451 outputDevicesRef.UnorderedRemoveElementAt(deviceIndex);
2457 void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt(
2458 AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) {
2459 MOZ_ASSERT(NS_IsMainThread());
2461 for (auto& elementRef : mOutputDeviceRefCnts) {
2462 if (elementRef.mDeviceID == aDeviceID) {
2463 ++elementRef.mRefCnt;
2464 return;
2467 MOZ_ASSERT(aDeviceID != mPrimaryOutputDeviceID,
2468 "mOutputDeviceRefCnts should always have the primary device");
2469 // Need to add an output device.
2470 // Output via another graph for this device.
2471 // This sample rate is not exposed to content.
2472 TrackRate sampleRate =
2473 aPreferredSampleRate != 0
2474 ? aPreferredSampleRate
2475 : static_cast<TrackRate>(CubebUtils::PreferredSampleRate(
2476 /*aShouldResistFingerprinting*/ false));
2477 MediaTrackGraph* newGraph = MediaTrackGraphImpl::GetInstance(
2478 MediaTrackGraph::AUDIO_THREAD_DRIVER, mWindowID, sampleRate, aDeviceID,
2479 GetMainThreadSerialEventTarget());
2480 // CreateCrossGraphReceiver wants the sample rate of this graph.
2481 RefPtr receiver = newGraph->CreateCrossGraphReceiver(mSampleRate);
2482 receiver->AddAudioOutput(nullptr, aDeviceID, sampleRate);
2483 mOutputDeviceRefCnts.EmplaceBack(
2484 DeviceReceiverAndCount{aDeviceID, receiver, 1});
2486 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aDeviceID,
2487 receiver = std::move(receiver)]() mutable {
2488 TRACE("MediaTrackGraph add output device ControlMessage");
2489 MOZ_ASSERT(!mOutputDevices.Contains(aDeviceID));
2490 mOutputDevices.EmplaceBack(
2491 OutputDeviceEntry{aDeviceID, std::move(receiver)});
2495 void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID) {
2496 MOZ_ASSERT(NS_IsMainThread());
2498 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
2499 MOZ_ASSERT(index != mOutputDeviceRefCnts.NoIndex);
2500 // mOutputDeviceRefCnts[0] is retained for consistency with
2501 // mOutputDevices[0], which is retained for AudioCallbackDriver output even
2502 // when no tracks have audio outputs.
2503 if (--mOutputDeviceRefCnts[index].mRefCnt == 0 && index != 0) {
2504 mOutputDeviceRefCnts[index].mReceiver->Destroy();
2505 mOutputDeviceRefCnts.UnorderedRemoveElementAt(index);
2509 void MediaTrack::Suspend() {
2510 // This can happen if this method has been called asynchronously, and the
2511 // track has been destroyed since then.
2512 if (mMainThreadDestroyed) {
2513 return;
2515 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2516 TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2517 IncrementSuspendCount();
2521 void MediaTrack::Resume() {
2522 // This can happen if this method has been called asynchronously, and the
2523 // track has been destroyed since then.
2524 if (mMainThreadDestroyed) {
2525 return;
2527 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
2528 TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2529 DecrementSuspendCount();
2533 void MediaTrack::AddListenerImpl(
2534 already_AddRefed<MediaTrackListener> aListener) {
2535 RefPtr<MediaTrackListener> l(aListener);
2536 mTrackListeners.AppendElement(std::move(l));
2538 PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
2539 mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
2540 Graph(), lastPrincipalHandle);
2541 if (mNotifiedEnded) {
2542 mTrackListeners.LastElement()->NotifyEnded(Graph());
2544 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
2545 mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2549 void MediaTrack::AddListener(MediaTrackListener* aListener) {
2550 MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
2551 if (mMainThreadDestroyed) {
2552 return;
2554 QueueControlMessageWithNoShutdown(
2555 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2556 TRACE("MediaTrack::AddListenerImpl ControlMessage");
2557 AddListenerImpl(listener.forget());
2561 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
2562 for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2563 if (mTrackListeners[i] == aListener) {
2564 mTrackListeners[i]->NotifyRemoved(Graph());
2565 mTrackListeners.RemoveElementAt(i);
2566 return;
2571 RefPtr<GenericPromise> MediaTrack::RemoveListener(
2572 MediaTrackListener* aListener) {
2573 MozPromiseHolder<GenericPromise> promiseHolder;
2574 RefPtr<GenericPromise> p = promiseHolder.Ensure(__func__);
2575 if (mMainThreadDestroyed) {
2576 promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
2577 return p;
2579 QueueControlOrShutdownMessage(
2580 [self = RefPtr{this}, this, listener = RefPtr{aListener},
2581 promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable {
2582 TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2583 // During shutdown we still want the listener's NotifyRemoved to be
2584 // called, since not doing that might block shutdown of other modules.
2585 RemoveListenerImpl(listener);
2586 promiseHolder.Resolve(true, __func__);
2588 return p;
2591 void MediaTrack::AddDirectListenerImpl(
2592 already_AddRefed<DirectMediaTrackListener> aListener) {
2593 AssertOnGraphThread();
2594 // Base implementation, for tracks that don't support direct track listeners.
2595 RefPtr<DirectMediaTrackListener> listener = aListener;
2596 listener->NotifyDirectListenerInstalled(
2597 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
2600 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
2601 if (mMainThreadDestroyed) {
2602 return;
2604 QueueControlMessageWithNoShutdown(
2605 [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
2606 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2607 AddDirectListenerImpl(listener.forget());
2611 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
2612 // Base implementation, the listener was never added so nothing to do.
2615 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
2616 if (mMainThreadDestroyed) {
2617 return;
2619 QueueControlOrShutdownMessage(
2620 [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) {
2621 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2622 // During shutdown we still want the listener's
2623 // NotifyDirectListenerUninstalled to be called, since not doing that
2624 // might block shutdown of other modules.
2625 RemoveDirectListenerImpl(listener);
2629 void MediaTrack::RunAfterPendingUpdates(
2630 already_AddRefed<nsIRunnable> aRunnable) {
2631 MOZ_ASSERT(NS_IsMainThread());
2632 if (mMainThreadDestroyed) {
2633 return;
2635 QueueControlOrShutdownMessage(
2636 [self = RefPtr{this}, this,
2637 runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable {
2638 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2639 if (aInShutdown == IsInShutdown::No) {
2640 Graph()->DispatchToMainThreadStableState(runnable.forget());
2641 } else {
2642 // Don't run mRunnable now as it may call AppendMessage() which would
2643 // assume that there are no remaining
2644 // controlMessagesToRunDuringShutdown.
2645 MOZ_ASSERT(NS_IsMainThread());
2646 GraphImpl()->Dispatch(runnable.forget());
2651 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2652 AssertOnGraphThread();
2653 MOZ_DIAGNOSTIC_ASSERT(
2654 aMode == DisabledTrackMode::ENABLED ||
2655 mDisabledMode == DisabledTrackMode::ENABLED,
2656 "Changing disabled track mode for a track is not allowed");
2657 DisabledTrackMode oldMode = CombinedDisabledMode();
2658 mDisabledMode = aMode;
2659 NotifyIfDisabledModeChangedFrom(oldMode);
2662 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
2663 if (mMainThreadDestroyed) {
2664 return;
2666 QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() {
2667 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2668 SetDisabledTrackModeImpl(aMode);
2672 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
2673 MediaSegment* aRawSegment) {
2674 AssertOnGraphThread();
2675 mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment);
2678 void MediaTrack::AddMainThreadListener(
2679 MainThreadMediaTrackListener* aListener) {
2680 MOZ_ASSERT(NS_IsMainThread());
2681 MOZ_ASSERT(aListener);
2682 MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2684 mMainThreadListeners.AppendElement(aListener);
2686 // If it is not yet time to send the notification, then exit here.
2687 if (!mEndedNotificationSent) {
2688 return;
2691 class NotifyRunnable final : public Runnable {
2692 public:
2693 explicit NotifyRunnable(MediaTrack* aTrack)
2694 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
2696 NS_IMETHOD Run() override {
2697 TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2698 MOZ_ASSERT(NS_IsMainThread());
2699 mTrack->NotifyMainThreadListeners();
2700 return NS_OK;
2703 private:
2704 ~NotifyRunnable() = default;
2706 RefPtr<MediaTrack> mTrack;
2709 nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2710 GraphImpl()->Dispatch(runnable.forget());
2713 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2714 GraphTime aBlockedTime) {
2715 mStartTime += aBlockedTime;
2717 if (!mSegment) {
2718 // No data to be forgotten.
2719 return;
2722 TrackTime time = aCurrentTime - mStartTime;
2723 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2724 // spend too much time pruning segments.
2725 const TrackTime minChunkSize = mSampleRate * 50 / 1000;
2726 if (time < mForgottenTime + minChunkSize) {
2727 return;
2730 mForgottenTime = std::min(GetEnd() - 1, time);
2731 mSegment->ForgetUpTo(mForgottenTime);
2734 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
2735 DisabledTrackMode mode = CombinedDisabledMode();
2736 if (aOldMode == mode) {
2737 return;
2740 for (const auto& listener : mTrackListeners) {
2741 listener->NotifyEnabledStateChanged(
2742 Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
2745 for (const auto& c : mConsumers) {
2746 if (c->GetDestination()) {
2747 c->GetDestination()->OnInputDisabledModeChanged(mode);
2752 void MediaTrack::QueueMessage(UniquePtr<ControlMessageInterface> aMessage) {
2753 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
2754 MOZ_RELEASE_ASSERT(!IsDestroyed());
2755 GraphImpl()->AppendMessage(std::move(aMessage));
2758 void MediaTrack::RunMessageAfterProcessing(
2759 UniquePtr<ControlMessageInterface> aMessage) {
2760 AssertOnGraphThread();
2761 GraphImpl()->RunMessageAfterProcessing(std::move(aMessage));
2764 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
2765 TrackRate aSampleRate)
2766 : MediaTrack(aSampleRate, aType,
2767 aType == MediaSegment::AUDIO
2768 ? static_cast<MediaSegment*>(new AudioSegment())
2769 : static_cast<MediaSegment*>(new VideoSegment())),
2770 mMutex("mozilla::media::SourceMediaTrack") {
2771 mUpdateTrack = MakeUnique<TrackData>();
2772 mUpdateTrack->mInputRate = aSampleRate;
2773 mUpdateTrack->mResamplerChannelCount = 0;
2774 mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
2775 mUpdateTrack->mEnded = false;
2776 mUpdateTrack->mPullingEnabled = false;
2777 mUpdateTrack->mGraphThreadDone = false;
2780 void SourceMediaTrack::DestroyImpl() {
2781 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2782 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2783 // Disconnect before we come under mMutex's lock since it can call back
2784 // through RemoveDirectListenerImpl() and deadlock.
2785 mConsumers[i]->Disconnect();
2788 // Hold mMutex while mGraph is reset so that other threads holding mMutex
2789 // can null-check know that the graph will not destroyed.
2790 MutexAutoLock lock(mMutex);
2791 mUpdateTrack = nullptr;
2792 MediaTrack::DestroyImpl();
2795 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
2796 class Message : public ControlMessage {
2797 public:
2798 Message(SourceMediaTrack* aTrack, bool aEnabled)
2799 : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
2800 void Run() override {
2801 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2802 MutexAutoLock lock(mTrack->mMutex);
2803 if (!mTrack->mUpdateTrack) {
2804 // We can't enable pulling for a track that has ended. We ignore
2805 // this if we're disabling pulling, since shutdown sequences are
2806 // complex. If there's truly an issue we'll have issues enabling anyway.
2807 MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
2808 return;
2810 MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
2811 "Pulling is not allowed for video");
2812 mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
2814 SourceMediaTrack* mTrack;
2815 bool mEnabled;
2817 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
2820 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
2821 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2822 TrackTime t;
2823 TrackTime current;
2825 if (mEnded) {
2826 return false;
2828 MutexAutoLock lock(mMutex);
2829 if (mUpdateTrack->mEnded) {
2830 return false;
2832 if (!mUpdateTrack->mPullingEnabled) {
2833 return false;
2835 // Compute how much track time we'll need assuming we don't block
2836 // the track at all.
2837 t = GraphTimeToTrackTime(aDesiredUpToTime);
2838 current = GetEnd() + mUpdateTrack->mData->GetDuration();
2840 if (t <= current) {
2841 return false;
2843 LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2844 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
2845 GraphImpl()->MediaTimeToSeconds(current)));
2846 for (auto& l : mTrackListeners) {
2847 l->NotifyPull(Graph(), current, t);
2849 return true;
2853 * This moves chunks from aIn to aOut. For audio this is simple. For video
2854 * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2856 * We also handle "resetters" from captured media elements. This type of source
2857 * pushes future frames into the track, and should it need to remove some, e.g.,
2858 * because of a seek or pause, it tells us by letting time go backwards. Without
2859 * this, tracks would be live for too long after a seek or pause.
2861 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
2862 MediaSegment* aOut, TrackTime aCurrentTime,
2863 TrackTime aDesiredUpToTime)
2864 MOZ_REQUIRES(aTrack->GetMutex()) {
2865 MOZ_ASSERT(aIn->GetType() == aOut->GetType());
2866 MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
2867 MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
2868 if (aIn->GetType() == MediaSegment::AUDIO) {
2869 AudioSegment* in = static_cast<AudioSegment*>(aIn);
2870 AudioSegment* out = static_cast<AudioSegment*>(aOut);
2871 TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
2872 TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
2874 out->AppendSlice(*in, 0, end);
2875 in->RemoveLeading(end);
2877 aTrack->GetMutex().AssertCurrentThreadOwns();
2878 out->ApplyVolume(aTrack->GetVolumeLocked());
2879 } else {
2880 VideoSegment* in = static_cast<VideoSegment*>(aIn);
2881 VideoSegment* out = static_cast<VideoSegment*>(aOut);
2882 for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
2883 MOZ_ASSERT(!c->mTimeStamp.IsNull());
2884 VideoChunk* last = out->GetLastChunk();
2885 if (!last || last->mTimeStamp.IsNull()) {
2886 // This is the first frame, or the last frame pushed to `out` has been
2887 // all consumed. Just append and we deal with its duration later.
2888 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2889 c->mFrame.GetIntrinsicSize(),
2890 c->mFrame.GetPrincipalHandle(),
2891 c->mFrame.GetForceBlack(), c->mTimeStamp);
2892 if (c->GetDuration() > 0) {
2893 out->ExtendLastFrameBy(c->GetDuration());
2895 continue;
2898 // We now know when this frame starts, aka when the last frame ends.
2900 if (c->mTimeStamp < last->mTimeStamp) {
2901 // Time is going backwards. This is a resetting frame from
2902 // DecodedStream. Clear everything up to currentTime.
2903 out->Clear();
2904 out->AppendNullData(aCurrentTime);
2907 // Append the current frame (will have duration 0).
2908 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2909 c->mFrame.GetIntrinsicSize(),
2910 c->mFrame.GetPrincipalHandle(),
2911 c->mFrame.GetForceBlack(), c->mTimeStamp);
2912 if (c->GetDuration() > 0) {
2913 out->ExtendLastFrameBy(c->GetDuration());
2916 if (out->GetDuration() < aDesiredUpToTime) {
2917 out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
2919 in->Clear();
2920 MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
2924 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
2925 GraphTime aDesiredUpToTime) {
2926 MutexAutoLock lock(mMutex);
2928 if (!mUpdateTrack) {
2929 MOZ_ASSERT(mEnded);
2930 return;
2933 TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
2935 ApplyTrackDisabling(mUpdateTrack->mData.get());
2937 if (!mUpdateTrack->mData->IsEmpty()) {
2938 for (const auto& l : mTrackListeners) {
2939 l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
2942 TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
2943 TrackTime endTime = trackDesiredUpToTime;
2944 if (mUpdateTrack->mEnded) {
2945 endTime = std::min(trackDesiredUpToTime,
2946 GetEnd() + mUpdateTrack->mData->GetDuration());
2948 LOG(LogLevel::Verbose,
2949 ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
2950 GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
2951 MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
2952 trackCurrentTime, endTime);
2953 if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
2954 mEnded = true;
2955 mUpdateTrack = nullptr;
2959 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
2960 mMutex.AssertCurrentThreadOwns();
2961 if (aSegment->GetType() != MediaSegment::AUDIO ||
2962 mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
2963 return;
2965 AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
2966 segment->ResampleChunks(mUpdateTrack->mResampler,
2967 &mUpdateTrack->mResamplerChannelCount,
2968 mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
2971 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
2972 GraphTime aCurrentTime, GraphTime aBlockedTime) {
2973 MutexAutoLock lock(mMutex);
2974 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
2977 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
2978 MutexAutoLock lock(mMutex);
2979 if (!mUpdateTrack) {
2980 return;
2982 MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
2983 // Set the new input rate and reset the resampler.
2984 mUpdateTrack->mInputRate = aRate;
2985 mUpdateTrack->mResampler.own(nullptr);
2986 mUpdateTrack->mResamplerChannelCount = 0;
2989 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
2990 MediaSegment* aRawSegment) {
2991 MutexAutoLock lock(mMutex);
2992 MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
2993 TrackTime appended = 0;
2994 if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
2995 aSegment->Clear();
2996 return appended;
2999 // Data goes into mData, and on the next iteration of the MTG moves
3000 // into the track's segment after NotifyQueuedTrackChanges(). This adds
3001 // 0-10ms of delay before data gets to direct listeners.
3002 // Indirect listeners (via subsequent TrackUnion nodes) are synced to
3003 // playout time, and so can be delayed by buffering.
3005 // Apply track disabling before notifying any consumers directly
3006 // or inserting into the graph
3007 mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment);
3009 ResampleAudioToGraphSampleRate(aSegment);
3011 // Must notify first, since AppendFrom() will empty out aSegment
3012 NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
3013 appended = aSegment->GetDuration();
3014 mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead
3016 auto graph = GraphImpl();
3017 MonitorAutoLock lock(graph->GetMonitor());
3018 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
3019 graph->EnsureNextIteration();
3023 return appended;
3026 TrackTime SourceMediaTrack::ClearFutureData() {
3027 MutexAutoLock lock(mMutex);
3028 auto graph = GraphImpl();
3029 if (!mUpdateTrack || !graph) {
3030 return 0;
3033 TrackTime duration = mUpdateTrack->mData->GetDuration();
3034 mUpdateTrack->mData->Clear();
3035 return duration;
3038 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
3039 mMutex.AssertCurrentThreadOwns();
3041 for (const auto& l : mDirectTrackListeners) {
3042 TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of
3043 // the internal buffer
3044 l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
3045 *aSegment);
3049 void SourceMediaTrack::AddDirectListenerImpl(
3050 already_AddRefed<DirectMediaTrackListener> aListener) {
3051 AssertOnGraphThread();
3052 MutexAutoLock lock(mMutex);
3054 RefPtr<DirectMediaTrackListener> listener = aListener;
3055 LOG(LogLevel::Debug,
3056 ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
3057 listener.get(), this));
3059 MOZ_ASSERT(mType == MediaSegment::VIDEO);
3060 for (const auto& l : mDirectTrackListeners) {
3061 if (l == listener) {
3062 listener->NotifyDirectListenerInstalled(
3063 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
3064 return;
3068 mDirectTrackListeners.AppendElement(listener);
3070 LOG(LogLevel::Debug,
3071 ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
3072 listener->NotifyDirectListenerInstalled(
3073 DirectMediaTrackListener::InstallationResult::SUCCESS);
3075 if (mDisabledMode != DisabledTrackMode::ENABLED) {
3076 listener->IncreaseDisabled(mDisabledMode);
3079 if (mEnded) {
3080 return;
3083 // Pass buffered data to the listener
3084 VideoSegment bufferedData;
3085 size_t videoFrames = 0;
3086 VideoSegment& segment = *GetData<VideoSegment>();
3087 for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
3088 iter.Next()) {
3089 if (iter->mTimeStamp.IsNull()) {
3090 // No timestamp means this is only for the graph's internal book-keeping,
3091 // denoting a late start of the track.
3092 continue;
3094 ++videoFrames;
3095 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
3096 iter->mFrame.GetIntrinsicSize(),
3097 iter->mFrame.GetPrincipalHandle(),
3098 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
3101 VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
3102 for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
3103 iter.Next()) {
3104 ++videoFrames;
3105 MOZ_ASSERT(!iter->mTimeStamp.IsNull());
3106 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
3107 iter->mFrame.GetIntrinsicSize(),
3108 iter->mFrame.GetPrincipalHandle(),
3109 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
3112 LOG(LogLevel::Info,
3113 ("%p: Notifying direct listener %p of %zu video frames and duration "
3114 "%" PRId64,
3115 GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
3116 listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
3119 void SourceMediaTrack::RemoveDirectListenerImpl(
3120 DirectMediaTrackListener* aListener) {
3121 mGraph->AssertOnGraphThreadOrNotRunning();
3122 MutexAutoLock lock(mMutex);
3123 for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
3124 const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
3125 if (l == aListener) {
3126 if (mDisabledMode != DisabledTrackMode::ENABLED) {
3127 aListener->DecreaseDisabled(mDisabledMode);
3129 aListener->NotifyDirectListenerUninstalled();
3130 mDirectTrackListeners.RemoveElementAt(i);
3135 void SourceMediaTrack::End() {
3136 MutexAutoLock lock(mMutex);
3137 if (!mUpdateTrack) {
3138 // Already ended
3139 return;
3141 mUpdateTrack->mEnded = true;
3142 if (auto graph = GraphImpl()) {
3143 MonitorAutoLock lock(graph->GetMonitor());
3144 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
3145 graph->EnsureNextIteration();
3150 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
3151 AssertOnGraphThread();
3153 MutexAutoLock lock(mMutex);
3154 const DisabledTrackMode oldMode = mDirectDisabledMode;
3155 const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
3156 const bool enabled = aMode == DisabledTrackMode::ENABLED;
3157 mDirectDisabledMode = aMode;
3158 for (const auto& l : mDirectTrackListeners) {
3159 if (!oldEnabled && enabled) {
3160 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3161 "direct listener enabled",
3162 GraphImpl(), this));
3163 l->DecreaseDisabled(oldMode);
3164 } else if (oldEnabled && !enabled) {
3165 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3166 "direct listener disabled",
3167 GraphImpl(), this));
3168 l->IncreaseDisabled(aMode);
3172 MediaTrack::SetDisabledTrackModeImpl(aMode);
3175 uint32_t SourceMediaTrack::NumberOfChannels() const {
3176 AudioSegment* audio = GetData<AudioSegment>();
3177 MOZ_DIAGNOSTIC_ASSERT(audio);
3178 if (!audio) {
3179 return 0;
3181 return audio->MaxChannelCount();
3184 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3185 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3186 MutexAutoLock lock(mMutex);
3188 for (auto& l : mDirectTrackListeners.Clone()) {
3189 l->NotifyDirectListenerUninstalled();
3191 mDirectTrackListeners.Clear();
3194 void SourceMediaTrack::SetVolume(float aVolume) {
3195 MutexAutoLock lock(mMutex);
3196 mVolume = aVolume;
3199 float SourceMediaTrack::GetVolumeLocked() {
3200 mMutex.AssertCurrentThreadOwns();
3201 return mVolume;
3204 SourceMediaTrack::~SourceMediaTrack() = default;
3206 void MediaInputPort::Init() {
3207 mGraph->AssertOnGraphThreadOrNotRunning();
3208 LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph,
3209 this, mSource, mDest));
3210 // Only connect the port if it wasn't disconnected on allocation.
3211 if (mSource) {
3212 mSource->AddConsumer(this);
3213 mDest->AddInput(this);
3215 // mPortCount decremented via MediaInputPort::Destroy's message
3216 ++mGraph->mPortCount;
3219 void MediaInputPort::Disconnect() {
3220 mGraph->AssertOnGraphThreadOrNotRunning();
3221 NS_ASSERTION(!mSource == !mDest,
3222 "mSource and mDest must either both be null or both non-null");
3224 if (!mSource) {
3225 return;
3228 mSource->RemoveConsumer(this);
3229 mDest->RemoveInput(this);
3230 mSource = nullptr;
3231 mDest = nullptr;
3233 mGraph->SetTrackOrderDirty();
3236 MediaTrack* MediaInputPort::GetSource() const {
3237 mGraph->AssertOnGraphThreadOrNotRunning();
3238 return mSource;
3241 ProcessedMediaTrack* MediaInputPort::GetDestination() const {
3242 mGraph->AssertOnGraphThreadOrNotRunning();
3243 return mDest;
3246 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
3247 MediaInputPort const* aPort, GraphTime aTime) {
3248 InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
3249 if (!aPort) {
3250 result.mStart = aTime;
3251 result.mInputIsBlocked = true;
3252 return result;
3254 aPort->mGraph->AssertOnGraphThreadOrNotRunning();
3255 if (aTime >= aPort->mDest->mStartBlocking) {
3256 return result;
3258 result.mStart = aTime;
3259 result.mEnd = aPort->mDest->mStartBlocking;
3260 result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
3261 if (!result.mInputIsBlocked) {
3262 result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
3264 return result;
3267 void MediaInputPort::Suspended() {
3268 mGraph->AssertOnGraphThreadOrNotRunning();
3269 mDest->InputSuspended(this);
3272 void MediaInputPort::Resumed() {
3273 mGraph->AssertOnGraphThreadOrNotRunning();
3274 mDest->InputResumed(this);
3277 void MediaInputPort::Destroy() {
3278 class Message : public ControlMessage {
3279 public:
3280 explicit Message(MediaInputPort* aPort)
3281 : ControlMessage(nullptr), mPort(aPort) {}
3282 void Run() override {
3283 TRACE("MediaInputPort::Destroy ControlMessage");
3284 mPort->Disconnect();
3285 --mPort->GraphImpl()->mPortCount;
3286 mPort->SetGraphImpl(nullptr);
3287 NS_RELEASE(mPort);
3289 void RunDuringShutdown() override { Run(); }
3290 MediaInputPort* mPort;
3292 // Keep a reference to the graph, since Message might RunDuringShutdown()
3293 // synchronously and make GraphImpl() invalid.
3294 RefPtr<MediaTrackGraphImpl> graph = mGraph;
3295 graph->AppendMessage(MakeUnique<Message>(this));
3296 --graph->mMainThreadPortCount;
3299 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
3300 mGraph->AssertOnGraphThreadOrNotRunning();
3301 return mGraph;
3304 MediaTrackGraph* MediaInputPort::Graph() const {
3305 mGraph->AssertOnGraphThreadOrNotRunning();
3306 return mGraph;
3309 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
3310 MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3311 DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
3312 MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
3313 mGraph = aGraph;
3316 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
3317 MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
3318 // This method creates two references to the MediaInputPort: one for
3319 // the main thread, and one for the MediaTrackGraph.
3320 class Message : public ControlMessage {
3321 public:
3322 explicit Message(MediaInputPort* aPort)
3323 : ControlMessage(aPort->mDest), mPort(aPort) {}
3324 void Run() override {
3325 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3326 mPort->Init();
3327 // The graph holds its reference implicitly
3328 mPort->GraphImpl()->SetTrackOrderDirty();
3329 Unused << mPort.forget();
3331 void RunDuringShutdown() override { Run(); }
3332 RefPtr<MediaInputPort> mPort;
3335 MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
3336 RefPtr<MediaInputPort> port;
3337 if (aTrack->IsDestroyed()) {
3338 // Create a port that's disconnected, which is what it'd be after its source
3339 // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3340 // this later is fine.
3341 port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
3342 aOutputNumber);
3343 } else {
3344 MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
3345 port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
3346 aOutputNumber);
3348 ++GraphImpl()->mMainThreadPortCount;
3349 GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3350 return port.forget();
3353 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
3354 class Message : public ControlMessage {
3355 public:
3356 Message(ProcessedMediaTrack* aTrack, bool aAutoend)
3357 : ControlMessage(aTrack), mAutoend(aAutoend) {}
3358 void Run() override {
3359 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3360 static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
3362 bool mAutoend;
3364 if (mMainThreadDestroyed) {
3365 return;
3367 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
3370 void ProcessedMediaTrack::DestroyImpl() {
3371 for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3372 mInputs[i]->Disconnect();
3375 for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3376 mSuspendedInputs[i]->Disconnect();
3379 MediaTrack::DestroyImpl();
3380 // The track order is only important if there are connections, in which
3381 // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3382 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3383 // SetTrackOrderDirty(), for other reasons.
3386 MediaTrackGraphImpl::MediaTrackGraphImpl(uint64_t aWindowID,
3387 TrackRate aSampleRate,
3388 AudioDeviceID aPrimaryOutputDeviceID,
3389 nsISerialEventTarget* aMainThread)
3390 : MediaTrackGraph(aSampleRate, aPrimaryOutputDeviceID),
3391 mWindowID(aWindowID),
3392 mFirstCycleBreaker(0)
3393 // An offline graph is not initially processing.
3395 mPortCount(0),
3396 mMonitor("MediaTrackGraphImpl"),
3397 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
3398 mPostedRunInStableStateEvent(false),
3399 mGraphDriverRunning(false),
3400 mPostedRunInStableState(false),
3401 mTrackOrderDirty(false),
3402 mMainThread(aMainThread),
3403 mGlobalVolume(CubebUtils::GetVolumeScale())
3404 #ifdef DEBUG
3406 mCanRunMessagesSynchronously(false)
3407 #endif
3409 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3410 mAudioOutputLatency(0.0),
3411 mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
3414 void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested,
3415 GraphRunType aRunTypeRequested,
3416 uint32_t aChannelCount) {
3417 mSelfRef = this;
3418 mEndTime = aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX;
3419 mRealtime = aDriverRequested != OFFLINE_THREAD_DRIVER;
3420 // The primary output device always exists because an AudioCallbackDriver
3421 // may exist, and want to be fed data, even when no tracks have audio
3422 // outputs.
3423 mOutputDeviceRefCnts.EmplaceBack(
3424 DeviceReceiverAndCount{mPrimaryOutputDeviceID, nullptr, 0});
3425 mOutputDevices.EmplaceBack(OutputDeviceEntry{mPrimaryOutputDeviceID});
3427 bool failedToGetShutdownBlocker = false;
3428 if (!IsNonRealtime()) {
3429 failedToGetShutdownBlocker = !AddShutdownBlocker();
3432 mGraphRunner = aRunTypeRequested == SINGLE_THREAD
3433 ? GraphRunner::Create(this)
3434 : already_AddRefed<GraphRunner>(nullptr);
3436 if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
3437 failedToGetShutdownBlocker) {
3438 MonitorAutoLock lock(mMonitor);
3439 // At least one of the following happened
3440 // - Failed to create thread.
3441 // - Failed to install a shutdown blocker when one is needed.
3442 // Because we have a fail state, jump to last phase of the lifecycle.
3443 mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
3444 RemoveShutdownBlocker(); // No-op if blocker wasn't added.
3445 #ifdef DEBUG
3446 mCanRunMessagesSynchronously = true;
3447 #endif
3448 return;
3450 if (mRealtime) {
3451 if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3452 // Always start with zero input channels, and no particular preferences
3453 // for the input channel.
3454 mDriver = new AudioCallbackDriver(
3455 this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(),
3456 nullptr, AudioInputType::Unknown);
3457 } else {
3458 mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
3460 nsCString streamName = GetDocumentTitle(mWindowID);
3461 LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
3462 mDriver->SetStreamName(streamName);
3463 } else {
3464 mDriver =
3465 new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
3468 mLastMainThreadUpdate = TimeStamp::Now();
3470 RegisterWeakAsyncMemoryReporter(this);
3473 #ifdef DEBUG
3474 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
3475 return aDriver->OnThread() ||
3476 (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
3478 #endif
3480 void MediaTrackGraphImpl::Destroy() {
3481 // First unregister from memory reporting.
3482 UnregisterWeakMemoryReporter(this);
3484 // Clear the self reference which will destroy this instance if all
3485 // associated GraphDrivers are destroyed.
3486 mSelfRef = nullptr;
3489 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3490 // GTests can create a graph without a window.
3491 /* static */
3492 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
3493 uint64_t aWindowID, TrackRate aSampleRate,
3494 AudioDeviceID aPrimaryOutputDeviceID) {
3495 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3496 MOZ_ASSERT(aSampleRate > 0);
3498 GraphHashSet::Ptr p =
3499 Graphs()->lookup({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
3500 return p ? *p : nullptr;
3503 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3504 // window is a real inner Window, not a WindowProxy.
3505 /* static */
3506 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
3507 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
3508 AudioDeviceID aPrimaryOutputDeviceID) {
3509 TrackRate sampleRate =
3510 aSampleRate ? aSampleRate
3511 : CubebUtils::PreferredSampleRate(
3512 aWindow->AsGlobal()->ShouldResistFingerprinting(
3513 RFPTarget::AudioSampleRate));
3514 return MediaTrackGraphImpl::GetInstanceIfExists(
3515 aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID);
3518 /* static */
3519 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
3520 GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
3521 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID,
3522 nsISerialEventTarget* aMainThread) {
3523 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3524 MOZ_ASSERT(aSampleRate > 0);
3525 MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER,
3526 "Use CreateNonRealtimeInstance() for offline graphs");
3528 GraphHashSet* graphs = Graphs();
3529 GraphHashSet::AddPtr addPtr =
3530 graphs->lookupForAdd({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
3531 if (addPtr) { // graph already exists
3532 return *addPtr;
3535 GraphRunType runType = DIRECT_DRIVER;
3536 if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) {
3537 runType = SINGLE_THREAD;
3540 // In a real time graph, the number of output channels is determined by
3541 // the underlying number of channel of the default audio output device, and
3542 // capped to 8.
3543 uint32_t channelCount =
3544 std::min<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
3545 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3546 aWindowID, aSampleRate, aPrimaryOutputDeviceID, aMainThread);
3547 graph->Init(aGraphDriverRequested, runType, channelCount);
3548 MOZ_ALWAYS_TRUE(graphs->add(addPtr, graph));
3550 LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64,
3551 graph, aWindowID));
3553 return graph;
3556 /* static */
3557 MediaTrackGraph* MediaTrackGraph::GetInstance(
3558 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
3559 TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) {
3560 TrackRate sampleRate =
3561 aSampleRate ? aSampleRate
3562 : CubebUtils::PreferredSampleRate(
3563 aWindow->AsGlobal()->ShouldResistFingerprinting(
3564 RFPTarget::AudioSampleRate));
3565 return MediaTrackGraphImpl::GetInstance(
3566 aGraphDriverRequested, aWindow->WindowID(), sampleRate,
3567 aPrimaryOutputDeviceID, GetMainThreadSerialEventTarget());
3570 MediaTrackGraph* MediaTrackGraphImpl::CreateNonRealtimeInstance(
3571 TrackRate aSampleRate) {
3572 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3574 nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
3575 // Offline graphs have 0 output channel count: they write the output to a
3576 // buffer, not an audio output track.
3577 MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
3578 0, aSampleRate, DEFAULT_OUTPUT_DEVICE, mainThread);
3579 graph->Init(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0);
3581 LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
3583 return graph;
3586 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
3587 TrackRate aSampleRate) {
3588 return MediaTrackGraphImpl::CreateNonRealtimeInstance(aSampleRate);
3591 void MediaTrackGraph::ForceShutDown() {
3592 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3594 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3596 graph->ForceShutDown();
3599 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver,
3600 nsIThreadObserver, nsITimerCallback, nsINamed)
3602 NS_IMETHODIMP
3603 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3604 nsISupports* aData, bool aAnonymize) {
3605 MOZ_ASSERT(NS_IsMainThread());
3606 if (mMainThreadTrackCount == 0) {
3607 // No tracks to report.
3608 FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3609 return NS_OK;
3612 class Message final : public ControlMessage {
3613 public:
3614 Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
3615 nsISupports* aHandlerData)
3616 : ControlMessage(nullptr),
3617 mGraph(aGraph),
3618 mHandleReport(aHandleReport),
3619 mHandlerData(aHandlerData) {}
3620 void Run() override {
3621 TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3622 mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3623 mHandlerData.forget());
3625 void RunDuringShutdown() override {
3626 // Run this message during shutdown too, so that endReports is called.
3627 Run();
3629 MediaTrackGraphImpl* mGraph;
3630 // nsMemoryReporterManager keeps the callback and data alive only if it
3631 // does not time out.
3632 nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3633 nsCOMPtr<nsISupports> mHandlerData;
3636 AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3638 return NS_OK;
3641 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3642 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3643 already_AddRefed<nsISupports> aHandlerData) {
3644 class FinishCollectRunnable final : public Runnable {
3645 public:
3646 explicit FinishCollectRunnable(
3647 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3648 already_AddRefed<nsISupports> aHandlerData)
3649 : mozilla::Runnable("FinishCollectRunnable"),
3650 mHandleReport(aHandleReport),
3651 mHandlerData(aHandlerData) {}
3653 NS_IMETHOD Run() override {
3654 TRACE("MTG::FinishCollectReports ControlMessage");
3655 MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3656 std::move(mAudioTrackSizes));
3657 return NS_OK;
3660 nsTArray<AudioNodeSizes> mAudioTrackSizes;
3662 private:
3663 ~FinishCollectRunnable() = default;
3665 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3666 // constructor modifies the ref-count, which cannot be done off main
3667 // thread.
3668 RefPtr<nsIHandleReportCallback> mHandleReport;
3669 RefPtr<nsISupports> mHandlerData;
3672 RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
3673 std::move(aHandleReport), std::move(aHandlerData));
3675 auto audioTrackSizes = &runnable->mAudioTrackSizes;
3677 for (MediaTrack* t : AllTracks()) {
3678 AudioNodeTrack* track = t->AsAudioNodeTrack();
3679 if (track) {
3680 AudioNodeSizes* usage = audioTrackSizes->AppendElement();
3681 track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3685 mMainThread->Dispatch(runnable.forget());
3688 void MediaTrackGraphImpl::FinishCollectReports(
3689 nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3690 const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
3691 MOZ_ASSERT(NS_IsMainThread());
3693 nsCOMPtr<nsIMemoryReporterManager> manager =
3694 do_GetService("@mozilla.org/memory-reporter-manager;1");
3696 if (!manager) return;
3698 #define REPORT(_path, _amount, _desc) \
3699 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3700 nsLiteralCString(_desc), aData);
3702 for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
3703 const AudioNodeSizes& usage = aAudioTrackSizes[i];
3704 const char* const nodeType =
3705 usage.mNodeType ? usage.mNodeType : "<unknown>";
3707 nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3708 nodeType);
3709 REPORT(enginePath, usage.mEngine,
3710 "Memory used by AudioNode engine objects (Web Audio).");
3712 nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
3713 nodeType);
3714 REPORT(trackPath, usage.mTrack,
3715 "Memory used by AudioNode track objects (Web Audio).");
3718 size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3719 if (hrtfLoaders) {
3720 REPORT(nsLiteralCString(
3721 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3722 hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
3725 #undef REPORT
3727 manager->EndReport();
3730 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
3731 SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
3732 AddTrack(track);
3733 return track;
3736 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
3737 MediaSegment::Type aType) {
3738 ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
3739 AddTrack(track);
3740 return track;
3743 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
3744 AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
3745 AddTrack(track);
3746 return track;
3749 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
3750 CrossGraphReceiver* aReceiver) {
3751 CrossGraphTransmitter* track =
3752 new CrossGraphTransmitter(GraphRate(), aReceiver);
3753 AddTrack(track);
3754 return track;
3757 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
3758 TrackRate aTransmitterRate) {
3759 CrossGraphReceiver* track =
3760 new CrossGraphReceiver(GraphRate(), aTransmitterRate);
3761 AddTrack(track);
3762 return track;
3765 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
3766 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3767 MOZ_ASSERT(NS_IsMainThread());
3768 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3769 if (graph->mRealtime) {
3770 GraphHashSet::Ptr p = Graphs()->lookup(*graph);
3771 MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down");
3773 #endif
3774 if (graph->mMainThreadTrackCount == 0) {
3775 nsCOMPtr<nsIObserverService> observerService =
3776 mozilla::services::GetObserverService();
3777 if (observerService) {
3778 observerService->AddObserver(graph, "document-title-changed", false);
3782 NS_ADDREF(aTrack);
3783 aTrack->SetGraphImpl(graph);
3784 ++graph->mMainThreadTrackCount;
3785 graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
3788 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
3789 MOZ_ASSERT(NS_IsMainThread());
3790 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
3792 mAudioOutputParams.RemoveElementsBy(
3793 [&](const TrackKeyDeviceAndVolume& aElement) {
3794 if (aElement.mTrack != aTrack) {
3795 return false;
3797 DecrementOutputDeviceRefCnt(aElement.mDeviceID);
3798 return true;
3801 if (--mMainThreadTrackCount == 0) {
3802 LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
3803 "main thread. Graph will shut down.",
3804 this, aTrack));
3805 if (mRealtime) {
3806 // Find the graph in the hash table and remove it.
3807 GraphHashSet* graphs = Graphs();
3808 GraphHashSet::Ptr p = graphs->lookup(*this);
3809 MOZ_ASSERT(*p == this);
3810 graphs->remove(p);
3812 nsCOMPtr<nsIObserverService> observerService =
3813 mozilla::services::GetObserverService();
3814 if (observerService) {
3815 observerService->RemoveObserver(this, "document-title-changed");
3818 // The graph thread will shut itself down soon, but won't be able to do
3819 // that if JS continues to run.
3820 InterruptJS();
3824 auto MediaTrackGraphImpl::NotifyWhenDeviceStarted(AudioDeviceID aDeviceID)
3825 -> RefPtr<GraphStartedPromise> {
3826 MOZ_ASSERT(NS_IsMainThread());
3828 size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
3829 if (index == decltype(mOutputDeviceRefCnts)::NoIndex) {
3830 return GraphStartedPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__);
3833 MozPromiseHolder<GraphStartedPromise> h;
3834 RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
3836 if (CrossGraphReceiver* receiver = mOutputDeviceRefCnts[index].mReceiver) {
3837 receiver->GraphImpl()->NotifyWhenPrimaryDeviceStarted(std::move(h));
3838 return p;
3841 // aSink corresponds to the primary audio output device of this graph.
3842 NotifyWhenPrimaryDeviceStarted(std::move(h));
3843 return p;
3846 void MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted(
3847 MozPromiseHolder<GraphStartedPromise>&& aHolder) {
3848 MOZ_ASSERT(NS_IsMainThread());
3849 if (mOutputDeviceRefCnts[0].mRefCnt == 0) {
3850 // There are no track outputs that require the device, so the creator of
3851 // this promise no longer needs to know when the graph is running. Don't
3852 // keep the graph alive with another message.
3853 aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
3854 return;
3857 QueueControlOrShutdownMessage(
3858 [self = RefPtr{this}, this,
3859 holder = std::move(aHolder)](IsInShutdown aInShutdown) mutable {
3860 if (aInShutdown == IsInShutdown::Yes) {
3861 holder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
3862 return;
3865 TRACE("MTG::NotifyWhenPrimaryDeviceStarted ControlMessage");
3866 // This runs on the graph thread, so when this runs, and the current
3867 // driver is an AudioCallbackDriver, we know the audio hardware is
3868 // started. If not, we are going to switch soon, keep reposting this
3869 // ControlMessage.
3870 if (CurrentDriver()->AsAudioCallbackDriver() &&
3871 CurrentDriver()->ThreadRunning() &&
3872 !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3873 // Avoid Resolve's locking on the graph thread by doing it on main.
3874 Dispatch(NS_NewRunnableFunction(
3875 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted::Resolver",
3876 [holder = std::move(holder)]() mutable {
3877 holder.Resolve(true, __func__);
3878 }));
3879 } else {
3880 DispatchToMainThreadStableState(
3881 NewRunnableMethod<
3882 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
3883 "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted", this,
3884 &MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted,
3885 std::move(holder)));
3890 class AudioContextOperationControlMessage : public ControlMessage {
3891 using AudioContextOperationPromise =
3892 MediaTrackGraph::AudioContextOperationPromise;
3894 public:
3895 AudioContextOperationControlMessage(
3896 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3897 AudioContextOperation aOperation,
3898 MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
3899 : ControlMessage(aDestinationTrack),
3900 mTracks(std::move(aTracks)),
3901 mAudioContextOperation(aOperation),
3902 mHolder(std::move(aHolder)) {}
3903 void Run() override {
3904 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3905 kAudioContextOptionsStrings[static_cast<uint8_t>(
3906 mAudioContextOperation)]);
3907 mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
3909 void RunDuringShutdown() override {
3910 MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
3911 "We should be reviving the graph?");
3912 mHolder.Reject(false, __func__);
3915 nsTArray<RefPtr<MediaTrack>> mTracks;
3916 AudioContextOperation mAudioContextOperation;
3917 MozPromiseHolder<AudioContextOperationPromise> mHolder;
3920 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3921 AudioContextOperationControlMessage* aMessage) {
3922 MOZ_ASSERT(OnGraphThread());
3923 // Initialize state to zero. This silences a GCC warning about uninitialized
3924 // values, because although the switch below initializes state for all valid
3925 // enum values, the actual value could be any integer that fits in the enum.
3926 AudioContextState state{0};
3927 switch (aMessage->mAudioContextOperation) {
3928 // Suspend and Close operations may be performed immediately because no
3929 // specific kind of GraphDriver is required. CheckDriver() will schedule
3930 // a change to a SystemCallbackDriver if all tracks are suspended.
3931 case AudioContextOperation::Suspend:
3932 state = AudioContextState::Suspended;
3933 break;
3934 case AudioContextOperation::Close:
3935 state = AudioContextState::Closed;
3936 break;
3937 case AudioContextOperation::Resume:
3938 // Resume operations require an AudioCallbackDriver. CheckDriver() will
3939 // schedule an AudioCallbackDriver if necessary and process pending
3940 // operations if and when an AudioCallbackDriver is running.
3941 mPendingResumeOperations.EmplaceBack(aMessage);
3942 return;
3944 // First resolve any pending Resume promises for the same AudioContext so as
3945 // to resolve its associated promises in the same order as they were
3946 // created. These Resume operations are considered complete and immediately
3947 // canceled by the Suspend or Close.
3948 MediaTrack* destinationTrack = aMessage->GetTrack();
3949 bool shrinking = false;
3950 auto moveDest = mPendingResumeOperations.begin();
3951 for (PendingResumeOperation& op : mPendingResumeOperations) {
3952 if (op.DestinationTrack() == destinationTrack) {
3953 op.Apply(this);
3954 shrinking = true;
3955 continue;
3957 if (shrinking) { // Fill-in gaps in the array.
3958 *moveDest = std::move(op);
3960 ++moveDest;
3962 mPendingResumeOperations.TruncateLength(moveDest -
3963 mPendingResumeOperations.begin());
3965 for (MediaTrack* track : aMessage->mTracks) {
3966 track->IncrementSuspendCount();
3968 // Resolve after main thread state is up to date with completed processing.
3969 DispatchToMainThreadStableState(NS_NewRunnableFunction(
3970 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
3971 [holder = std::move(aMessage->mHolder), state]() mutable {
3972 holder.Resolve(state, __func__);
3973 }));
3976 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
3977 AudioContextOperationControlMessage* aMessage)
3978 : mDestinationTrack(aMessage->GetTrack()),
3979 mTracks(std::move(aMessage->mTracks)),
3980 mHolder(std::move(aMessage->mHolder)) {
3981 MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
3984 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
3985 MediaTrackGraphImpl* aGraph) {
3986 MOZ_ASSERT(aGraph->OnGraphThread());
3987 for (MediaTrack* track : mTracks) {
3988 track->DecrementSuspendCount();
3990 // The graph is provided through the parameter so that it is available even
3991 // when the track is destroyed.
3992 aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
3993 "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
3994 holder.Resolve(AudioContextState::Running, __func__);
3995 }));
3998 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
3999 // The graph is shutting down before the operation completed.
4000 MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
4001 mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
4002 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
4003 mHolder.Reject(false, __func__);
4006 auto MediaTrackGraph::ApplyAudioContextOperation(
4007 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
4008 AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
4009 MozPromiseHolder<AudioContextOperationPromise> holder;
4010 RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
4011 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4012 graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
4013 aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
4014 return p;
4017 uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const {
4018 MOZ_ASSERT(!mOutputDevices[0].mReceiver);
4019 return AudioOutputChannelCount(mOutputDevices[0]);
4022 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount(
4023 const OutputDeviceEntry& aDevice) const {
4024 MOZ_ASSERT(OnGraphThread());
4025 // The audio output channel count for a graph is the maximum of the output
4026 // channel count of all the tracks with outputs to this device, or the max
4027 // audio output channel count the machine can do, whichever is smaller.
4028 uint32_t channelCount = 0;
4029 for (const auto& output : aDevice.mTrackOutputs) {
4030 channelCount = std::max(channelCount, output.mTrack->NumberOfChannels());
4032 channelCount = std::min(channelCount, mMaxOutputChannelCount);
4033 if (channelCount) {
4034 return channelCount;
4035 } else {
4036 // null aDevice.mReceiver indicates the primary graph output device.
4037 if (!aDevice.mReceiver && CurrentDriver()->AsAudioCallbackDriver()) {
4038 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
4040 return 2;
4044 double MediaTrackGraph::AudioOutputLatency() {
4045 return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
4048 double MediaTrackGraphImpl::AudioOutputLatency() {
4049 MOZ_ASSERT(NS_IsMainThread());
4050 if (mAudioOutputLatency != 0.0) {
4051 return mAudioOutputLatency;
4053 MonitorAutoLock lock(mMonitor);
4054 if (CurrentDriver()->AsAudioCallbackDriver()) {
4055 mAudioOutputLatency = CurrentDriver()
4056 ->AsAudioCallbackDriver()
4057 ->AudioOutputLatency()
4058 .ToSeconds();
4059 } else {
4060 // Failure mode: return 0.0 if running on a normal thread.
4061 mAudioOutputLatency = 0.0;
4064 return mAudioOutputLatency;
4067 bool MediaTrackGraph::IsNonRealtime() const {
4068 return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
4071 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
4072 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
4074 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
4075 NS_ASSERTION(!graph->mRealtime, "non-realtime only");
4077 class Message : public ControlMessage {
4078 public:
4079 explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
4080 : ControlMessage(nullptr),
4081 mGraph(aGraph),
4082 mTicksToProcess(aTicksToProcess) {}
4083 void Run() override {
4084 TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
4085 MOZ_ASSERT(mGraph->mEndTime == 0,
4086 "StartNonRealtimeProcessing should be called only once");
4087 mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
4088 mGraph->mStateComputedTime + mTicksToProcess);
4090 // The graph owns this message.
4091 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
4092 uint32_t mTicksToProcess;
4095 graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
4098 void MediaTrackGraphImpl::InterruptJS() {
4099 MonitorAutoLock lock(mMonitor);
4100 mInterruptJSCalled = true;
4101 if (mJSContext) {
4102 JS_RequestInterruptCallback(mJSContext);
4106 static bool InterruptCallback(JSContext* aCx) {
4107 // Interrupt future calls also.
4108 JS_RequestInterruptCallback(aCx);
4109 // Stop execution.
4110 return false;
4113 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
4114 MOZ_ASSERT(OnGraphThread());
4115 MOZ_ASSERT(aCx);
4117 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
4118 MonitorAutoLock lock(impl->mMonitor);
4119 if (impl->mJSContext) {
4120 MOZ_ASSERT(impl->mJSContext == aCx);
4121 return;
4123 JS_AddInterruptCallback(aCx, InterruptCallback);
4124 impl->mJSContext = aCx;
4125 if (impl->mInterruptJSCalled) {
4126 JS_RequestInterruptCallback(aCx);
4130 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
4131 MediaTrack* t = aPort->GetSource();
4132 if (!t->IsSuspended()) {
4133 mInputs.AppendElement(aPort);
4134 } else {
4135 mSuspendedInputs.AppendElement(aPort);
4137 GraphImpl()->SetTrackOrderDirty();
4140 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
4141 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4142 mInputs.RemoveElement(aPort);
4143 mSuspendedInputs.AppendElement(aPort);
4144 GraphImpl()->SetTrackOrderDirty();
4147 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
4148 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4149 mSuspendedInputs.RemoveElement(aPort);
4150 mInputs.AppendElement(aPort);
4151 GraphImpl()->SetTrackOrderDirty();
4154 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
4155 MOZ_ASSERT(OnGraphThread());
4156 LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
4157 if (GraphDriver* nextDriver = NextDriver()) {
4158 if (nextDriver != CurrentDriver()) {
4159 LOG(LogLevel::Debug,
4160 ("%p: Discarding previous next driver: %p", this, nextDriver));
4163 mNextDriver = aNextDriver;
4166 void MediaTrackGraph::RegisterCaptureTrackForWindow(
4167 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
4168 MOZ_ASSERT(NS_IsMainThread());
4169 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4170 graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
4173 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
4174 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
4175 MOZ_ASSERT(NS_IsMainThread());
4176 WindowAndTrack winAndTrack;
4177 winAndTrack.mWindowId = aWindowId;
4178 winAndTrack.mCaptureTrackSink = aCaptureTrack;
4179 mWindowCaptureTracks.AppendElement(winAndTrack);
4182 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4183 MOZ_ASSERT(NS_IsMainThread());
4184 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4185 graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
4188 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4189 MOZ_ASSERT(NS_IsMainThread());
4190 mWindowCaptureTracks.RemoveElementsBy(
4191 [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
4194 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
4195 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4196 return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
4197 aMediaTrack);
4200 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
4201 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4202 MOZ_ASSERT(NS_IsMainThread());
4203 for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
4204 if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
4205 ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
4206 return sink->AllocateInputPort(aMediaTrack);
4209 return nullptr;
4212 void MediaTrackGraph::DispatchToMainThreadStableState(
4213 already_AddRefed<nsIRunnable> aRunnable) {
4214 AssertOnGraphThreadOrNotRunning();
4215 static_cast<MediaTrackGraphImpl*>(this)
4216 ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
4219 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
4220 MOZ_ASSERT(NS_IsMainThread());
4221 return mMainThreadGraphTime;
4224 GraphTime MediaTrackGraph::ProcessedTime() const {
4225 AssertOnGraphThreadOrNotRunning();
4226 return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
4229 void* MediaTrackGraph::CurrentDriver() const {
4230 AssertOnGraphThreadOrNotRunning();
4231 return static_cast<const MediaTrackGraphImpl*>(this)->mDriver;
4234 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4235 CubebUtils::AudioDeviceID aID) {
4236 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4237 DeviceInputTrack* t =
4238 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4239 return t ? t->MaxRequestedInputChannels() : 0;
4242 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
4243 CubebUtils::AudioDeviceID aID) {
4244 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4245 DeviceInputTrack* t =
4246 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4247 return t && t->HasVoiceInput() ? AudioInputType::Voice
4248 : AudioInputType::Unknown;
4251 void MediaTrackGraphImpl::SetNewNativeInput() {
4252 MOZ_ASSERT(NS_IsMainThread());
4253 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4255 LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
4257 NonNativeInputTrack* track =
4258 mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
4259 if (!track) {
4260 LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
4261 return;
4264 const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
4265 const PrincipalHandle principal = track->mPrincipalHandle;
4267 LOG(LogLevel::Debug,
4268 ("%p Select device %p as the new native input device", this, deviceId));
4270 struct TrackListener {
4271 DeviceInputConsumerTrack* track;
4272 // Keep its reference so it won't be dropped when after
4273 // DisconnectDeviceInput().
4274 RefPtr<AudioDataListener> listener;
4276 nsTArray<TrackListener> pairs;
4278 for (const auto& t : track->GetConsumerTracks()) {
4279 pairs.AppendElement(
4280 TrackListener{t.get(), t->GetAudioDataListener().get()});
4283 for (TrackListener& pair : pairs) {
4284 pair.track->DisconnectDeviceInput();
4287 for (TrackListener& pair : pairs) {
4288 pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
4289 LOG(LogLevel::Debug,
4290 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4291 pair.track, deviceId));
4294 LOG(LogLevel::Debug,
4295 ("%p Native input device is set to device %p now", this, deviceId));
4297 MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4300 // nsIThreadObserver methods
4302 NS_IMETHODIMP
4303 MediaTrackGraphImpl::OnDispatchedEvent() {
4304 MonitorAutoLock lock(mMonitor);
4305 EnsureNextIteration();
4306 return NS_OK;
4309 NS_IMETHODIMP
4310 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
4311 return NS_OK;
4314 NS_IMETHODIMP
4315 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
4316 return NS_OK;
4318 } // namespace mozilla