Bug 1839170 - Refactor Snap pulling, Add Firefox Snap Core22 and GNOME 42 SDK symbols...
[gecko.git] / dom / media / MediaTrackGraph.cpp
blobd60015a68d746cd76bb1cfa8c1080e7963a9b4b7
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaTrackGraphImpl.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "CrossGraphPort.h"
12 #include "VideoSegment.h"
13 #include "nsContentUtils.h"
14 #include "nsPrintfCString.h"
15 #include "nsServiceManagerUtils.h"
16 #include "prerror.h"
17 #include "mozilla/Logging.h"
18 #include "mozilla/Attributes.h"
19 #include "ForwardedInputTrack.h"
20 #include "ImageContainer.h"
21 #include "AudioCaptureTrack.h"
22 #include "AudioNodeTrack.h"
23 #include "AudioNodeExternalInputTrack.h"
24 #if defined(MOZ_WEBRTC)
25 # include "MediaEngineWebRTCAudio.h"
26 #endif // MOZ_WEBRTC
27 #include "MediaTrackListener.h"
28 #include "mozilla/dom/BaseAudioContextBinding.h"
29 #include "mozilla/dom/WorkletThread.h"
30 #include "mozilla/media/MediaUtils.h"
31 #include <algorithm>
32 #include "GeckoProfiler.h"
33 #include "VideoFrameContainer.h"
34 #include "mozilla/AbstractThread.h"
35 #include "mozilla/StaticPrefs_dom.h"
36 #include "mozilla/StaticPrefs_media.h"
37 #include "transport/runnable_utils.h"
38 #include "VideoUtils.h"
39 #include "GraphRunner.h"
40 #include "Tracing.h"
41 #include "UnderrunHandler.h"
42 #include "mozilla/CycleCollectedJSRuntime.h"
43 #include "mozilla/Preferences.h"
45 #include "webaudio/blink/DenormalDisabler.h"
46 #include "webaudio/blink/HRTFDatabaseLoader.h"
48 using namespace mozilla::layers;
49 using namespace mozilla::dom;
50 using namespace mozilla::gfx;
51 using namespace mozilla::media;
53 namespace mozilla {
55 LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
56 #ifdef LOG
57 # undef LOG
58 #endif // LOG
59 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
61 NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
62 return mNativeInputTrack.get();
65 DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
66 CubebUtils::AudioDeviceID aID) {
67 if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
68 return mNativeInputTrack.get();
70 for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
71 if (t->mDeviceId == aID) {
72 return t.get();
75 return nullptr;
78 NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
79 if (mNonNativeInputTracks.IsEmpty()) {
80 return nullptr;
82 return mNonNativeInputTracks[0].get();
85 void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
86 if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
87 MOZ_ASSERT(!mNativeInputTrack);
88 mNativeInputTrack = native;
89 } else {
90 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
91 MOZ_ASSERT(nonNative);
92 struct DeviceTrackComparator {
93 public:
94 bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
95 CubebUtils::AudioDeviceID aDeviceId) const {
96 return aTrack->mDeviceId == aDeviceId;
99 MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
100 DeviceTrackComparator()));
101 mNonNativeInputTracks.AppendElement(nonNative);
105 void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
106 if (aTrack->AsNativeInputTrack()) {
107 MOZ_ASSERT(mNativeInputTrack);
108 MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
109 mNativeInputTrack = nullptr;
110 } else {
111 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
112 MOZ_ASSERT(nonNative);
113 DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
114 MOZ_ASSERT(removed);
118 namespace {
120 * A hash table containing the graph instances, one per Window ID,
121 * sample rate, and device ID combination.
123 class GraphKey final {
124 public:
125 GraphKey(uint64_t aWindowID, TrackRate aSampleRate,
126 CubebUtils::AudioDeviceID aOutputDeviceID)
127 : mWindowID(aWindowID),
128 mSampleRate(aSampleRate),
129 mOutputDeviceID(aOutputDeviceID) {}
130 GraphKey(const GraphKey&) = default;
131 ~GraphKey() = default;
132 bool operator==(const GraphKey& b) const {
133 return mWindowID == b.mWindowID && mSampleRate == b.mSampleRate &&
134 mOutputDeviceID == b.mOutputDeviceID;
136 PLDHashNumber Hash() const {
137 return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
140 private:
141 uint64_t mWindowID;
142 TrackRate mSampleRate;
143 CubebUtils::AudioDeviceID mOutputDeviceID;
145 nsTHashMap<nsGenericHashKey<GraphKey>, MediaTrackGraphImpl*> gGraphs;
146 } // anonymous namespace
148 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
149 MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
150 "All tracks should have been destroyed by messages from the main "
151 "thread");
152 LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
153 LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
156 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
157 MOZ_ASSERT(OnGraphThreadOrNotRunning());
158 aTrack->mStartTime = mProcessedTime;
160 if (aTrack->IsSuspended()) {
161 mSuspendedTracks.AppendElement(aTrack);
162 LOG(LogLevel::Debug,
163 ("%p: Adding media track %p, in the suspended track array", this,
164 aTrack));
165 } else {
166 mTracks.AppendElement(aTrack);
167 LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack,
168 mTracks.Length()));
171 SetTrackOrderDirty();
174 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
175 MOZ_ASSERT(OnGraphThreadOrNotRunning());
176 // Remove references in mTrackUpdates before we allow aTrack to die.
177 // Pending updates are not needed (since the main thread has already given
178 // up the track) so we will just drop them.
180 MonitorAutoLock lock(mMonitor);
181 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
182 if (mTrackUpdates[i].mTrack == aTrack) {
183 mTrackUpdates[i].mTrack = nullptr;
188 // Ensure that mFirstCycleBreaker is updated when necessary.
189 SetTrackOrderDirty();
191 UnregisterAllAudioOutputs(aTrack);
193 if (aTrack->IsSuspended()) {
194 mSuspendedTracks.RemoveElement(aTrack);
195 } else {
196 mTracks.RemoveElement(aTrack);
199 LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
200 mTracks.Length()));
202 NS_RELEASE(aTrack); // probably destroying it
205 TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
206 const MediaTrack* aTrack, GraphTime aTime) const {
207 MOZ_ASSERT(
208 aTime <= mStateComputedTime,
209 "Don't ask about times where we haven't made blocking decisions yet");
210 return std::max<TrackTime>(
211 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
214 GraphTime MediaTrackGraphImpl::IterationEnd() const {
215 MOZ_ASSERT(OnGraphThread());
216 return mIterationEndTime;
219 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
220 GraphTime aPrevCurrentTime) {
221 MOZ_ASSERT(OnGraphThread());
222 for (MediaTrack* track : AllTracks()) {
223 // Shouldn't have already notified of ended *and* have output!
224 MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
225 !track->mNotifiedEnded);
227 // Calculate blocked time and fire Blocked/Unblocked events
228 GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
229 NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
230 track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
231 blockedTime);
232 LOG(LogLevel::Verbose,
233 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
234 MediaTimeToSeconds(track->mStartTime),
235 MediaTimeToSeconds(blockedTime)));
236 track->mStartBlocking = mStateComputedTime;
238 TrackTime trackCurrentTime =
239 track->GraphTimeToTrackTime(mStateComputedTime);
240 if (track->mEnded) {
241 MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
242 if (!track->mNotifiedEnded) {
243 // Playout of this track ended and listeners have not been notified.
244 track->mNotifiedEnded = true;
245 SetTrackOrderDirty();
246 for (const auto& listener : track->mTrackListeners) {
247 listener->NotifyOutput(this, track->GetEnd());
248 listener->NotifyEnded(this);
251 } else {
252 for (const auto& listener : track->mTrackListeners) {
253 listener->NotifyOutput(this, trackCurrentTime);
259 template <typename C, typename Chunk>
260 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
261 C& aSegment,
262 TrackTime aStart,
263 TrackTime aEnd) {
264 MOZ_ASSERT(OnGraphThreadOrNotRunning());
265 MOZ_ASSERT(aTrack);
267 TrackTime offset = 0;
268 for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
269 chunk.Next()) {
270 if (offset >= aEnd) {
271 break;
273 offset += chunk->GetDuration();
274 if (chunk->IsNull() || offset < aStart) {
275 continue;
277 const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
278 if (principalHandle != aSegment.GetLastPrincipalHandle()) {
279 aSegment.SetLastPrincipalHandle(principalHandle);
280 LOG(LogLevel::Debug,
281 ("%p: MediaTrack %p, principalHandle "
282 "changed in %sChunk with duration %lld",
283 this, aTrack,
284 aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
285 (long long)chunk->GetDuration()));
286 for (const auto& listener : aTrack->mTrackListeners) {
287 listener->NotifyPrincipalHandleChanged(this, principalHandle);
293 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
294 MOZ_ASSERT(OnGraphThreadOrNotRunning());
295 for (MediaTrack* track : AllTracks()) {
296 TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
297 TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
298 if (!track->mSegment) {
299 continue;
301 if (track->mType == MediaSegment::AUDIO) {
302 ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
303 track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
304 } else if (track->mType == MediaSegment::VIDEO) {
305 ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
306 track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
307 } else {
308 MOZ_CRASH("Unknown track type");
313 GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
314 GraphTime aEndBlockingDecisions) {
315 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
316 // underrun currently, since we'll always be able to produce data for them
317 // unless they block on some other track.
318 if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
319 return aEndBlockingDecisions;
321 // This track isn't ended or suspended. We don't need to call
322 // TrackTimeToGraphTime since an underrun is the only thing that can block
323 // it.
324 GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
325 #ifdef DEBUG
326 if (bufferEnd < mProcessedTime) {
327 LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
328 "bufferEnd %f < mProcessedTime %f (%" PRId64
329 " < %" PRId64 "), TrackTime %" PRId64,
330 this, aTrack, MediaTimeToSeconds(bufferEnd),
331 MediaTimeToSeconds(mProcessedTime), bufferEnd,
332 mProcessedTime, aTrack->GetEnd()));
333 NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
335 #endif
336 return std::min(bufferEnd, aEndBlockingDecisions);
339 namespace {
340 // Value of mCycleMarker for unvisited tracks in cycle detection.
341 const uint32_t NOT_VISITED = UINT32_MAX;
342 // Value of mCycleMarker for ordered tracks in muted cycles.
343 const uint32_t IN_MUTED_CYCLE = 1;
344 } // namespace
346 bool MediaTrackGraphImpl::AudioTrackPresent() {
347 MOZ_ASSERT(OnGraphThreadOrNotRunning());
349 bool audioTrackPresent = false;
350 for (MediaTrack* track : mTracks) {
351 if (track->AsAudioNodeTrack()) {
352 audioTrackPresent = true;
353 break;
356 if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
357 audioTrackPresent = true;
358 break;
362 // We may not have audio input device when we only have AudioNodeTracks. But
363 // if audioTrackPresent is false, we must have no input device.
364 MOZ_DIAGNOSTIC_ASSERT_IF(
365 !audioTrackPresent,
366 !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
368 return audioTrackPresent;
371 void MediaTrackGraphImpl::CheckDriver() {
372 MOZ_ASSERT(OnGraphThread());
373 // An offline graph has only one driver.
374 // Otherwise, if a switch is already pending, let that happen.
375 if (!mRealtime || Switching()) {
376 return;
379 AudioCallbackDriver* audioCallbackDriver =
380 CurrentDriver()->AsAudioCallbackDriver();
381 if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
382 for (PendingResumeOperation& op : mPendingResumeOperations) {
383 op.Apply(this);
385 mPendingResumeOperations.Clear();
388 // Note that this looks for any audio tracks, input or output, and switches
389 // to a SystemClockDriver if there are none active or no resume operations
390 // to make any active.
391 bool needAudioCallbackDriver =
392 !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
393 if (!needAudioCallbackDriver) {
394 if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
395 SwitchAtNextIteration(
396 new SystemClockDriver(this, CurrentDriver(), mSampleRate));
398 return;
401 NativeInputTrack* native =
402 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
403 CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
404 uint32_t inputChannelCount =
405 native ? AudioInputChannelCount(native->mDeviceId) : 0;
406 AudioInputType inputPreference =
407 native ? AudioInputDevicePreference(native->mDeviceId)
408 : AudioInputType::Unknown;
410 uint32_t graphOutputChannelCount = AudioOutputChannelCount();
411 if (!audioCallbackDriver) {
412 if (graphOutputChannelCount > 0) {
413 AudioCallbackDriver* driver = new AudioCallbackDriver(
414 this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
415 inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
416 SwitchAtNextIteration(driver);
418 return;
421 // Check if this graph should switch to a different number of output channels.
422 // Generally, a driver switch is explicitly made by an event (e.g., setting
423 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
424 // directly playing back via another HTMLMediaElement, the number of channels
425 // of the media determines how many channels to output, and it can change
426 // dynamically.
427 if (graphOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
428 AudioCallbackDriver* driver = new AudioCallbackDriver(
429 this, CurrentDriver(), mSampleRate, graphOutputChannelCount,
430 inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
431 SwitchAtNextIteration(driver);
435 void MediaTrackGraphImpl::UpdateTrackOrder() {
436 if (!mTrackOrderDirty) {
437 return;
440 mTrackOrderDirty = false;
442 // The algorithm for finding cycles is based on Tim Leslie's iterative
443 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
444 // connected components (SCC) algorithm. There are variations (a) to
445 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
446 // re-run the algorithm over SCCs with breaks at DelayNodes.
448 // [1] http://www.timl.id.au/?p=327
449 // [2]
450 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
451 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
453 // There are two stacks. One for the depth-first search (DFS),
454 mozilla::LinkedList<MediaTrack> dfsStack;
455 // and another for tracks popped from the DFS stack, but still being
456 // considered as part of SCCs involving tracks on the stack.
457 mozilla::LinkedList<MediaTrack> sccStack;
459 // An index into mTracks for the next track found with no unsatisfied
460 // upstream dependencies.
461 uint32_t orderedTrackCount = 0;
463 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
464 MediaTrack* t = mTracks[i];
465 ProcessedMediaTrack* pt = t->AsProcessedTrack();
466 if (pt) {
467 // The dfsStack initially contains a list of all processed tracks in
468 // unchanged order.
469 dfsStack.insertBack(t);
470 pt->mCycleMarker = NOT_VISITED;
471 } else {
472 // SourceMediaTracks have no inputs and so can be ordered now.
473 mTracks[orderedTrackCount] = t;
474 ++orderedTrackCount;
478 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
479 // counter to label mCycleMarker on the next visited track in the DFS
480 // uniquely in the set of visited tracks that are still being considered.
482 // In this implementation, the counter descends so that the values are
483 // strictly greater than the values that mCycleMarker takes when the track
484 // has been ordered (0 or IN_MUTED_CYCLE).
486 // Each new track labelled, as the DFS searches upstream, receives a value
487 // less than those used for all other tracks being considered.
488 uint32_t nextStackMarker = NOT_VISITED - 1;
489 // Reset list of DelayNodes in cycles stored at the tail of mTracks.
490 mFirstCycleBreaker = mTracks.Length();
492 // Rearrange dfsStack order as required to DFS upstream and pop tracks
493 // in processing order to place in mTracks.
494 while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
495 const auto& inputs = pt->mInputs;
496 MOZ_ASSERT(pt->AsProcessedTrack());
497 if (pt->mCycleMarker == NOT_VISITED) {
498 // Record the position on the visited stack, so that any searches
499 // finding this track again know how much of the stack is in the cycle.
500 pt->mCycleMarker = nextStackMarker;
501 --nextStackMarker;
502 // Not-visited input tracks should be processed first.
503 // SourceMediaTracks have already been ordered.
504 for (uint32_t i = inputs.Length(); i--;) {
505 if (inputs[i]->GetSource()->IsSuspended()) {
506 continue;
508 auto input = inputs[i]->GetSource()->AsProcessedTrack();
509 if (input && input->mCycleMarker == NOT_VISITED) {
510 // It can be that this track has an input which is from a suspended
511 // AudioContext.
512 if (input->isInList()) {
513 input->remove();
514 dfsStack.insertFront(input);
518 continue;
521 // Returning from DFS. Pop from dfsStack.
522 pt->remove();
524 // cycleStackMarker keeps track of the highest marker value on any
525 // upstream track, if any, found receiving input, directly or indirectly,
526 // from the visited stack (and so from |ps|, making a cycle). In a
527 // variation from Tarjan's SCC algorithm, this does not include |ps|
528 // unless it is part of the cycle.
529 uint32_t cycleStackMarker = 0;
530 for (uint32_t i = inputs.Length(); i--;) {
531 if (inputs[i]->GetSource()->IsSuspended()) {
532 continue;
534 auto input = inputs[i]->GetSource()->AsProcessedTrack();
535 if (input) {
536 cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
540 if (cycleStackMarker <= IN_MUTED_CYCLE) {
541 // All inputs have been ordered and their stack markers have been removed.
542 // This track is not part of a cycle. It can be processed next.
543 pt->mCycleMarker = 0;
544 mTracks[orderedTrackCount] = pt;
545 ++orderedTrackCount;
546 continue;
549 // A cycle has been found. Record this track for ordering when all
550 // tracks in this SCC have been popped from the DFS stack.
551 sccStack.insertFront(pt);
553 if (cycleStackMarker > pt->mCycleMarker) {
554 // Cycles have been found that involve tracks that remain on the stack.
555 // Leave mCycleMarker indicating the most downstream (last) track on
556 // the stack known to be part of this SCC. In this way, any searches on
557 // other paths that find |ps| will know (without having to traverse from
558 // this track again) that they are part of this SCC (i.e. part of an
559 // intersecting cycle).
560 pt->mCycleMarker = cycleStackMarker;
561 continue;
564 // |pit| is the root of an SCC involving no other tracks on dfsStack, the
565 // complete SCC has been recorded, and tracks in this SCC are part of at
566 // least one cycle.
567 MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
568 // If there are DelayNodes in this SCC, then they may break the cycles.
569 bool haveDelayNode = false;
570 auto next = sccStack.getFirst();
571 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
572 // (There may be other tracks later in sccStack from other incompletely
573 // searched SCCs, involving tracks still on dfsStack.)
575 // DelayNodes in cycles must behave differently from those not in cycles,
576 // so all DelayNodes in the SCC must be identified.
577 while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
578 cycleStackMarker) {
579 auto nt = next->AsAudioNodeTrack();
580 // Get next before perhaps removing from list below.
581 next = next->getNext();
582 if (nt && nt->Engine()->AsDelayNodeEngine()) {
583 haveDelayNode = true;
584 // DelayNodes break cycles by producing their output in a
585 // preprocessing phase; they do not need to be ordered before their
586 // consumers. Order them at the tail of mTracks so that they can be
587 // handled specially. Do so now, so that DFS ignores them.
588 nt->remove();
589 nt->mCycleMarker = 0;
590 --mFirstCycleBreaker;
591 mTracks[mFirstCycleBreaker] = nt;
594 auto after_scc = next;
595 while ((next = sccStack.getFirst()) != after_scc) {
596 next->remove();
597 auto removed = static_cast<ProcessedMediaTrack*>(next);
598 if (haveDelayNode) {
599 // Return tracks to the DFS stack again (to order and detect cycles
600 // without delayNodes). Any of these tracks that are still inputs
601 // for tracks on the visited stack must be returned to the front of
602 // the stack to be ordered before their dependents. We know that none
603 // of these tracks need input from tracks on the visited stack, so
604 // they can all be searched and ordered before the current stack head
605 // is popped.
606 removed->mCycleMarker = NOT_VISITED;
607 dfsStack.insertFront(removed);
608 } else {
609 // Tracks in cycles without any DelayNodes must be muted, and so do
610 // not need input and can be ordered now. They must be ordered before
611 // their consumers so that their muted output is available.
612 removed->mCycleMarker = IN_MUTED_CYCLE;
613 mTracks[orderedTrackCount] = removed;
614 ++orderedTrackCount;
619 MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
622 TrackTime MediaTrackGraphImpl::PlayAudio(AudioMixer* aMixer,
623 const TrackKeyAndVolume& aTkv,
624 GraphTime aPlayedTime) {
625 MOZ_ASSERT(OnGraphThread());
626 MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
627 MOZ_ASSERT(aMixer, "Can only play audio if there's a mixer");
629 TrackTime ticksWritten = 0;
631 ticksWritten = 0;
632 MediaTrack* track = aTkv.mTrack;
633 AudioSegment* audio = track->GetData<AudioSegment>();
634 AudioSegment output;
636 TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
638 // We don't update Track->mTracksStartTime here to account for time spent
639 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
640 // the blocked period has completed. But we do need to make sure we play
641 // from the right offsets in the track buffer, even if we've already
642 // written silence for some amount of blocked time after the current time.
643 GraphTime t = aPlayedTime;
644 while (t < mStateComputedTime) {
645 bool blocked = t >= track->mStartBlocking;
646 GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
647 NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
649 // Check how many ticks of sound we can provide if we are blocked some
650 // time in the middle of this cycle.
651 TrackTime toWrite = end - t;
653 if (blocked) {
654 output.InsertNullDataAtStart(toWrite);
655 ticksWritten += toWrite;
656 LOG(LogLevel::Verbose,
657 ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
658 "%f to %f (%" PRId64 " to %" PRId64 ")",
659 this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
660 offset, offset + toWrite));
661 } else {
662 TrackTime endTicksNeeded = offset + toWrite;
663 TrackTime endTicksAvailable = audio->GetDuration();
665 if (endTicksNeeded <= endTicksAvailable) {
666 LOG(LogLevel::Verbose,
667 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
668 "(samples %" PRId64 " to %" PRId64 ")",
669 this, track, toWrite, MediaTimeToSeconds(t),
670 MediaTimeToSeconds(end), offset, endTicksNeeded));
671 output.AppendSlice(*audio, offset, endTicksNeeded);
672 ticksWritten += toWrite;
673 offset = endTicksNeeded;
674 } else {
675 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
676 // ended."); If we are at the end of the track, maybe write the
677 // remaining samples, and pad with/output silence.
678 if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
679 output.AppendSlice(*audio, offset, endTicksAvailable);
681 LOG(LogLevel::Verbose,
682 ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
683 "(samples %" PRId64 " to %" PRId64 ")",
684 this, track, toWrite, MediaTimeToSeconds(t),
685 MediaTimeToSeconds(end), offset, endTicksNeeded));
686 uint32_t available = endTicksAvailable - offset;
687 ticksWritten += available;
688 toWrite -= available;
689 offset = endTicksAvailable;
691 output.AppendNullData(toWrite);
692 LOG(LogLevel::Verbose,
693 ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
694 "%f (samples %" PRId64 " to %" PRId64 ")",
695 this, track, toWrite, MediaTimeToSeconds(t),
696 MediaTimeToSeconds(end), offset, endTicksNeeded));
697 ticksWritten += toWrite;
699 output.ApplyVolume(mGlobalVolume * aTkv.mVolume);
701 t = end;
703 uint32_t outputChannels;
704 // Use the number of channel the driver expects: this is the number of
705 // channel that can be output by the underlying system level audio stream.
706 // Fall back to something sensible if this graph is being driven by a normal
707 // thread (this can happen when there are no output devices, etc.).
708 if (CurrentDriver()->AsAudioCallbackDriver()) {
709 outputChannels =
710 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
711 } else {
712 outputChannels = AudioOutputChannelCount();
714 output.WriteTo(*aMixer, outputChannels, mSampleRate);
716 return ticksWritten;
719 DeviceInputTrack* MediaTrackGraphImpl::GetDeviceInputTrackMainThread(
720 CubebUtils::AudioDeviceID aID) {
721 MOZ_ASSERT(NS_IsMainThread());
722 return mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
725 NativeInputTrack* MediaTrackGraphImpl::GetNativeInputTrackMainThread() {
726 MOZ_ASSERT(NS_IsMainThread());
727 return mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
730 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
731 MOZ_ASSERT(OnGraphThread());
732 LOG(LogLevel::Debug,
733 ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
735 mDeviceInputTrackManagerGraphThread.Add(aTrack);
737 if (aTrack->AsNativeInputTrack()) {
738 // Switch Drivers since we're adding input (to input-only or full-duplex)
739 AudioCallbackDriver* driver = new AudioCallbackDriver(
740 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
741 AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID,
742 aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId));
743 LOG(LogLevel::Debug,
744 ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
745 this, driver));
746 SwitchAtNextIteration(driver);
747 } else {
748 NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
749 MOZ_ASSERT(nonNative);
750 // Start non-native input right away.
751 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
752 MakeRefPtr<AudioInputSourceListener>(nonNative),
753 nonNative->GenerateSourceId(), nonNative->mDeviceId,
754 AudioInputChannelCount(nonNative->mDeviceId),
755 AudioInputDevicePreference(nonNative->mDeviceId) ==
756 AudioInputType::Voice,
757 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate(),
758 StaticPrefs::media_clockdrift_buffering()));
762 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
763 MOZ_ASSERT(NS_IsMainThread());
764 MOZ_ASSERT(aTrack);
766 LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
767 aTrack, aTrack->mDeviceId));
769 class Message : public ControlMessage {
770 public:
771 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
772 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
773 void Run() override {
774 TRACE("MTG::OpenAudioInputImpl ControlMessage");
775 mGraph->OpenAudioInputImpl(mInputTrack);
777 MediaTrackGraphImpl* mGraph;
778 DeviceInputTrack* mInputTrack;
781 mDeviceInputTrackManagerMainThread.Add(aTrack);
783 this->AppendMessage(MakeUnique<Message>(this, aTrack));
786 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
787 MOZ_ASSERT(OnGraphThread());
789 LOG(LogLevel::Debug,
790 ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
792 if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
793 nonNative->StopAudio();
794 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
795 return;
798 MOZ_ASSERT(aTrack->AsNativeInputTrack());
800 mDeviceInputTrackManagerGraphThread.Remove(aTrack);
802 // Switch Drivers since we're adding or removing an input (to nothing/system
803 // or output only)
804 bool audioTrackPresent = AudioTrackPresent();
806 GraphDriver* driver;
807 if (audioTrackPresent) {
808 // We still have audio output
809 LOG(LogLevel::Debug,
810 ("%p: CloseInput: output present (AudioCallback)", this));
812 driver = new AudioCallbackDriver(
813 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
814 AudioInputChannelCount(aTrack->mDeviceId), mOutputDeviceID, nullptr,
815 AudioInputDevicePreference(aTrack->mDeviceId));
816 SwitchAtNextIteration(driver);
817 } else if (CurrentDriver()->AsAudioCallbackDriver()) {
818 LOG(LogLevel::Debug,
819 ("%p: CloseInput: no output present (SystemClockCallback)", this));
821 driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
822 SwitchAtNextIteration(driver);
823 } // else SystemClockDriver->SystemClockDriver, no switch
826 void MediaTrackGraphImpl::RegisterAudioOutput(MediaTrack* aTrack, void* aKey) {
827 MOZ_ASSERT(OnGraphThread());
828 MOZ_ASSERT(!mAudioOutputs.Contains(TrackAndKey{aTrack, aKey}));
830 TrackKeyAndVolume* tkv = mAudioOutputs.AppendElement();
831 tkv->mTrack = aTrack;
832 tkv->mKey = aKey;
833 tkv->mVolume = 1.0;
835 if (!CurrentDriver()->AsAudioCallbackDriver() && !Switching()) {
836 NativeInputTrack* native =
837 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
838 CubebUtils::AudioDeviceID inputDevice =
839 native ? native->mDeviceId : nullptr;
840 uint32_t inputChannelCount =
841 native ? AudioInputChannelCount(native->mDeviceId) : 0;
842 AudioInputType inputPreference =
843 native ? AudioInputDevicePreference(native->mDeviceId)
844 : AudioInputType::Unknown;
846 AudioCallbackDriver* driver = new AudioCallbackDriver(
847 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
848 inputChannelCount, mOutputDeviceID, inputDevice, inputPreference);
849 SwitchAtNextIteration(driver);
853 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
854 MOZ_ASSERT(OnGraphThreadOrNotRunning());
856 mAudioOutputs.RemoveElementsBy([aTrack](const TrackKeyAndVolume& aTkv) {
857 return aTkv.mTrack == aTrack;
861 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
862 void* aKey) {
863 MOZ_ASSERT(OnGraphThreadOrNotRunning());
865 DebugOnly<bool> removed =
866 mAudioOutputs.RemoveElement(TrackAndKey{aTrack, aKey});
867 MOZ_ASSERT(removed, "Audio output not found");
870 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
871 MOZ_ASSERT(NS_IsMainThread());
872 MOZ_ASSERT(aTrack);
874 LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
875 this, aTrack, aTrack->mDeviceId));
877 class Message : public ControlMessage {
878 public:
879 Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
880 : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
881 void Run() override {
882 TRACE("MTG::CloseAudioInputImpl ControlMessage");
883 mGraph->CloseAudioInputImpl(mInputTrack);
885 MediaTrackGraphImpl* mGraph;
886 DeviceInputTrack* mInputTrack;
889 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
890 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
891 // for more details.
892 mDeviceInputTrackManagerMainThread.Remove(aTrack);
894 this->AppendMessage(MakeUnique<Message>(this, aTrack));
896 if (aTrack->AsNativeInputTrack()) {
897 LOG(LogLevel::Debug,
898 ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
899 SetNewNativeInput();
903 // All AudioInput listeners get the same speaker data (at least for now).
904 void MediaTrackGraphImpl::NotifyOutputData(AudioDataValue* aBuffer,
905 size_t aFrames, TrackRate aRate,
906 uint32_t aChannels) {
907 if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
908 return;
911 #if defined(MOZ_WEBRTC)
912 for (const auto& track : mTracks) {
913 if (const auto& t = track->AsAudioProcessingTrack()) {
914 t->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
917 #endif
920 void MediaTrackGraphImpl::NotifyInputStopped() {
921 NativeInputTrack* native =
922 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
923 if (!native) {
924 return;
926 native->NotifyInputStopped(this);
929 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
930 size_t aFrames, TrackRate aRate,
931 uint32_t aChannels,
932 uint32_t aAlreadyBuffered) {
933 // Either we have an audio input device, or we just removed the audio input
934 // this iteration, and we're switching back to an output-only driver next
935 // iteration.
936 NativeInputTrack* native =
937 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
938 MOZ_ASSERT(native || Switching());
939 if (!native) {
940 return;
942 native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
943 aAlreadyBuffered);
946 void MediaTrackGraphImpl::DeviceChangedImpl() {
947 MOZ_ASSERT(OnGraphThread());
948 NativeInputTrack* native =
949 mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
950 if (!native) {
951 return;
953 native->DeviceChanged(this);
956 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
957 MOZ_ASSERT(OnGraphThread());
958 mMaxOutputChannelCount = aMaxChannelCount;
961 void MediaTrackGraphImpl::DeviceChanged() {
962 // This is safe to be called from any thread: this message comes from an
963 // underlying platform API, and we don't have much guarantees. If it is not
964 // called from the main thread (and it probably will rarely be), it will post
965 // itself to the main thread, and the actual device change message will be ran
966 // and acted upon on the graph thread.
967 if (!NS_IsMainThread()) {
968 RefPtr<nsIRunnable> runnable = WrapRunnable(
969 RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
970 mMainThread->Dispatch(runnable.forget());
971 return;
974 class Message : public ControlMessage {
975 public:
976 explicit Message(MediaTrackGraph* aGraph)
977 : ControlMessage(nullptr),
978 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
979 void Run() override {
980 TRACE("MTG::DeviceChangeImpl ControlMessage");
981 mGraphImpl->DeviceChangedImpl();
983 // We know that this is valid, because the graph can't shutdown if it has
984 // messages.
985 MediaTrackGraphImpl* mGraphImpl;
988 if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
989 // This is a special case where the origin of this event cannot control the
990 // lifetime of the graph, because the graph is controling the lifetime of
991 // the AudioCallbackDriver where the event originated.
992 // We know the graph is soon going away, so there's no need to notify about
993 // this device change.
994 return;
997 // Reset the latency, it will get fetched again next time it's queried.
998 MOZ_ASSERT(NS_IsMainThread());
999 mAudioOutputLatency = 0.0;
1001 // Dispatch to the bg thread to do the (potentially expensive) query of the
1002 // maximum channel count, and then dispatch back to the main thread, then to
1003 // the graph, with the new info.
1004 RefPtr<MediaTrackGraphImpl> self = this;
1005 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1006 "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
1007 uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
1008 self->Dispatch(NS_NewRunnableFunction(
1009 "MaxChannelCountUpdateToMainThread",
1010 [self{self}, maxChannelCount]() {
1011 class MessageToGraph : public ControlMessage {
1012 public:
1013 explicit MessageToGraph(MediaTrackGraph* aGraph,
1014 uint32_t aMaxChannelCount)
1015 : ControlMessage(nullptr),
1016 mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
1017 mMaxChannelCount(aMaxChannelCount) {}
1018 void Run() override {
1019 TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1020 mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
1022 MediaTrackGraphImpl* mGraphImpl;
1023 uint32_t mMaxChannelCount;
1025 self->AppendMessage(
1026 MakeUnique<MessageToGraph>(self, maxChannelCount));
1027 }));
1028 }));
1030 AppendMessage(MakeUnique<Message>(this));
1033 static const char* GetAudioInputTypeString(const AudioInputType& aType) {
1034 return aType == AudioInputType::Voice ? "Voice" : "Unknown";
1037 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
1038 MOZ_ASSERT(OnGraphThread());
1040 LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
1042 DeviceInputTrack* track =
1043 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
1044 if (!track) {
1045 LOG(LogLevel::Debug,
1046 ("%p: No DeviceInputTrack for this device. Ignore", this));
1047 return;
1050 bool needToSwitch = false;
1052 if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
1053 if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
1054 LOG(LogLevel::Debug,
1055 ("%p: %u-channel non-native input device %p (track %p) is "
1056 "re-configured to %d-channel",
1057 this, nonNative->NumberOfChannels(), aID, track,
1058 AudioInputChannelCount(aID)));
1059 needToSwitch = true;
1061 if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
1062 LOG(LogLevel::Debug,
1063 ("%p: %s-type non-native input device %p (track %p) is re-configured "
1064 "to %s-type",
1065 this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
1066 track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1067 needToSwitch = true;
1070 if (needToSwitch) {
1071 nonNative->StopAudio();
1072 nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
1073 MakeRefPtr<AudioInputSourceListener>(nonNative),
1074 nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
1075 AudioInputDevicePreference(aID) == AudioInputType::Voice,
1076 nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate(),
1077 StaticPrefs::media_clockdrift_buffering()));
1080 return;
1083 MOZ_ASSERT(track->AsNativeInputTrack());
1085 if (AudioCallbackDriver* audioCallbackDriver =
1086 CurrentDriver()->AsAudioCallbackDriver()) {
1087 if (audioCallbackDriver->InputChannelCount() !=
1088 AudioInputChannelCount(aID)) {
1089 LOG(LogLevel::Debug,
1090 ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1091 "re-configured to %d-channel",
1092 this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
1093 AudioInputChannelCount(aID)));
1094 needToSwitch = true;
1096 if (audioCallbackDriver->InputDevicePreference() !=
1097 AudioInputDevicePreference(aID)) {
1098 LOG(LogLevel::Debug,
1099 ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1100 "re-configured to %s-type",
1101 this,
1102 GetAudioInputTypeString(
1103 audioCallbackDriver->InputDevicePreference()),
1104 audioCallbackDriver,
1105 GetAudioInputTypeString(AudioInputDevicePreference(aID))));
1106 needToSwitch = true;
1108 } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1109 // We're already in the process of switching to a audio callback driver,
1110 // which will happen at the next iteration.
1111 // However, maybe it's not the correct number of channels. Re-query the
1112 // correct channel amount at this time.
1113 needToSwitch = true;
1116 if (needToSwitch) {
1117 AudioCallbackDriver* newDriver = new AudioCallbackDriver(
1118 this, CurrentDriver(), mSampleRate, AudioOutputChannelCount(),
1119 AudioInputChannelCount(aID), mOutputDeviceID, aID,
1120 AudioInputDevicePreference(aID));
1121 SwitchAtNextIteration(newDriver);
1125 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1126 // either we're on the right thread (and calling CurrentDriver() is safe),
1127 // or we're going to fail the assert anyway, so don't cross-check
1128 // via CurrentDriver().
1129 return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
1132 bool MediaTrackGraphImpl::OnGraphThread() const {
1133 // we're on the right thread (and calling mDriver is safe),
1134 MOZ_ASSERT(mDriver);
1135 if (mGraphRunner && mGraphRunner->OnThread()) {
1136 return true;
1138 return mDriver->OnThread();
1141 bool MediaTrackGraphImpl::Destroyed() const {
1142 MOZ_ASSERT(NS_IsMainThread());
1143 return !mSelfRef;
1146 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1147 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1148 if (mRealtime) {
1149 return true;
1152 TimeStamp now = TimeStamp::Now();
1153 // For offline graphs, update now if it has been long enough since the last
1154 // update, or if it has reached the end.
1155 if ((now - mLastMainThreadUpdate).ToMilliseconds() >
1156 CurrentDriver()->IterationDuration() ||
1157 mStateComputedTime >= mEndTime) {
1158 mLastMainThreadUpdate = now;
1159 return true;
1161 return false;
1164 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
1165 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1166 mMonitor.AssertCurrentThreadOwns();
1168 // We don't want to frequently update the main thread about timing update
1169 // when we are not running in realtime.
1170 if (aFinalUpdate || ShouldUpdateMainThread()) {
1171 // Strip updates that will be obsoleted below, so as to keep the length of
1172 // mTrackUpdates sane.
1173 size_t keptUpdateCount = 0;
1174 for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
1175 MediaTrack* track = mTrackUpdates[i].mTrack;
1176 // RemoveTrackGraphThread() clears mTrack in updates for
1177 // tracks that are removed from the graph.
1178 MOZ_ASSERT(!track || track->GraphImpl() == this);
1179 if (!track || track->MainThreadNeedsUpdates()) {
1180 // Discard this update as it has either been cleared when the track
1181 // was destroyed or there will be a newer update below.
1182 continue;
1184 if (keptUpdateCount != i) {
1185 mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
1186 MOZ_ASSERT(!mTrackUpdates[i].mTrack);
1188 ++keptUpdateCount;
1190 mTrackUpdates.TruncateLength(keptUpdateCount);
1192 mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
1193 mSuspendedTracks.Length());
1194 for (MediaTrack* track : AllTracks()) {
1195 if (!track->MainThreadNeedsUpdates()) {
1196 continue;
1198 TrackUpdate* update = mTrackUpdates.AppendElement();
1199 update->mTrack = track;
1200 // No blocking to worry about here, since we've passed
1201 // UpdateCurrentTimeForTracks.
1202 update->mNextMainThreadCurrentTime =
1203 track->GraphTimeToTrackTime(mProcessedTime);
1204 update->mNextMainThreadEnded = track->mNotifiedEnded;
1206 mNextMainThreadGraphTime = mProcessedTime;
1207 if (!mPendingUpdateRunnables.IsEmpty()) {
1208 mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
1212 // If this is the final update, then a stable state event will soon be
1213 // posted just before this thread finishes, and so there is no need to also
1214 // post here.
1215 if (!aFinalUpdate &&
1216 // Don't send the message to the main thread if it's not going to have
1217 // any work to do.
1218 !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
1219 EnsureStableStateEventPosted();
1223 GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
1224 if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
1225 return aTime;
1227 return RoundUpToNextAudioBlock(aTime);
1230 GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
1231 uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
1232 uint64_t nextBlock = block + 1;
1233 GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
1234 return nextTime;
1237 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1238 uint32_t aTrackIndex, TrackRate aSampleRate) {
1239 MOZ_ASSERT(OnGraphThread());
1240 MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
1241 "Cycle breaker is not AudioNodeTrack?");
1243 while (mProcessedTime < mStateComputedTime) {
1244 // Microtask checkpoints are in between render quanta.
1245 nsAutoMicroTask mt;
1247 GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
1248 for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
1249 auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
1250 MOZ_ASSERT(nt->AsAudioNodeTrack());
1251 nt->ProduceOutputBeforeInput(mProcessedTime);
1253 for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
1254 ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
1255 if (pt) {
1256 pt->ProcessInput(
1257 mProcessedTime, next,
1258 (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
1261 mProcessedTime = next;
1263 NS_ASSERTION(mProcessedTime == mStateComputedTime,
1264 "Something went wrong with rounding to block boundaries");
1267 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1268 UniquePtr<ControlMessage> aMessage) {
1269 MOZ_ASSERT(OnGraphThread());
1271 if (mFrontMessageQueue.IsEmpty()) {
1272 mFrontMessageQueue.AppendElement();
1275 // Only one block is used for messages from the graph thread.
1276 MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
1277 mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
1280 void MediaTrackGraphImpl::RunMessagesInQueue() {
1281 TRACE("MTG::RunMessagesInQueue");
1282 MOZ_ASSERT(OnGraphThread());
1283 // Calculate independent action times for each batch of messages (each
1284 // batch corresponding to an event loop task). This isolates the performance
1285 // of different scripts to some extent.
1286 for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
1287 nsTArray<UniquePtr<ControlMessage>>& messages =
1288 mFrontMessageQueue[i].mMessages;
1290 for (uint32_t j = 0; j < messages.Length(); ++j) {
1291 TRACE("ControlMessage::Run");
1292 messages[j]->Run();
1295 mFrontMessageQueue.Clear();
1298 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
1299 TRACE("MTG::UpdateGraph");
1300 MOZ_ASSERT(OnGraphThread());
1301 MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
1302 // The next state computed time can be the same as the previous: it
1303 // means the driver would have been blocking indefinitly, but the graph has
1304 // been woken up right after having been to sleep.
1305 MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
1307 CheckDriver();
1308 UpdateTrackOrder();
1310 // Always do another iteration if there are tracks waiting to resume.
1311 bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
1313 for (MediaTrack* track : mTracks) {
1314 if (SourceMediaTrack* is = track->AsSourceTrack()) {
1315 ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
1316 is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
1318 if (track->mEnded) {
1319 // The track's not suspended, and since it's ended, underruns won't
1320 // stop it playing out. So there's no blocking other than what we impose
1321 // here.
1322 GraphTime endTime = track->GetEnd() + track->mStartTime;
1323 if (endTime <= mStateComputedTime) {
1324 LOG(LogLevel::Verbose,
1325 ("%p: MediaTrack %p is blocked due to being ended", this, track));
1326 track->mStartBlocking = mStateComputedTime;
1327 } else {
1328 LOG(LogLevel::Verbose,
1329 ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1330 "time %f, end at %f)",
1331 this, track, MediaTimeToSeconds(mStateComputedTime),
1332 MediaTimeToSeconds(endTime)));
1333 // Data can't be added to a ended track, so underruns are irrelevant.
1334 MOZ_ASSERT(endTime <= aEndBlockingDecisions);
1335 track->mStartBlocking = endTime;
1337 } else {
1338 track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
1340 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1341 if (SourceMediaTrack* s = track->AsSourceTrack()) {
1342 if (s->Ended()) {
1343 continue;
1346 MutexAutoLock lock(s->mMutex);
1347 if (!s->mUpdateTrack->mPullingEnabled) {
1348 // The invariant that data must be provided is only enforced when
1349 // pulling.
1350 continue;
1353 if (track->GetEnd() <
1354 track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
1355 LOG(LogLevel::Error,
1356 ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1357 "but wasn't fed "
1358 "enough data. TrackListeners=%zu. Track-end=%f, "
1359 "Iteration-end=%f",
1360 this, track,
1361 (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
1362 track->mTrackListeners.Length(),
1363 MediaTimeToSeconds(track->GetEnd()),
1364 MediaTimeToSeconds(
1365 track->GraphTimeToTrackTime(aEndBlockingDecisions))));
1366 MOZ_DIAGNOSTIC_ASSERT(false,
1367 "A non-ended SourceMediaTrack wasn't fed "
1368 "enough data by NotifyPull");
1371 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1375 for (MediaTrack* track : mSuspendedTracks) {
1376 track->mStartBlocking = mStateComputedTime;
1379 // If the loop is woken up so soon that IterationEnd() barely advances or
1380 // if an offline graph is not currently rendering, we end up having
1381 // aEndBlockingDecisions == mStateComputedTime.
1382 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1383 // empty, Process() will not find any unblocked track and so will not
1384 // ensure another iteration. If the graph should be rendering, then ensure
1385 // another iteration to render.
1386 if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
1387 mStateComputedTime < mEndTime)) {
1388 EnsureNextIteration();
1392 void MediaTrackGraphImpl::Process(AudioMixer* aMixer) {
1393 TRACE("MTG::Process");
1394 MOZ_ASSERT(OnGraphThread());
1395 // Play track contents.
1396 bool allBlockedForever = true;
1397 // True when we've done ProcessInput for all processed tracks.
1398 bool doneAllProducing = false;
1399 const GraphTime oldProcessedTime = mProcessedTime;
1401 // Figure out what each track wants to do
1402 for (uint32_t i = 0; i < mTracks.Length(); ++i) {
1403 MediaTrack* track = mTracks[i];
1404 if (!doneAllProducing) {
1405 ProcessedMediaTrack* pt = track->AsProcessedTrack();
1406 if (pt) {
1407 AudioNodeTrack* n = track->AsAudioNodeTrack();
1408 if (n) {
1409 #ifdef DEBUG
1410 // Verify that the sampling rate for all of the following tracks is
1411 // the same
1412 for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
1413 AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
1414 if (nextTrack) {
1415 MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
1416 "All AudioNodeTracks in the graph must have the same "
1417 "sampling rate");
1420 #endif
1421 // Since an AudioNodeTrack is present, go ahead and
1422 // produce audio block by block for all the rest of the tracks.
1423 ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
1424 doneAllProducing = true;
1425 } else {
1426 pt->ProcessInput(mProcessedTime, mStateComputedTime,
1427 ProcessedMediaTrack::ALLOW_END);
1428 // Assert that a live track produced enough data
1429 MOZ_ASSERT_IF(!track->mEnded,
1430 track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1431 track, mStateComputedTime));
1435 if (track->mStartBlocking > oldProcessedTime) {
1436 allBlockedForever = false;
1439 mProcessedTime = mStateComputedTime;
1441 if (aMixer) {
1442 MOZ_ASSERT(mRealtime, "If there's a mixer, this graph must be realtime");
1443 aMixer->StartMixing();
1444 // This is the number of frames that are written to the output buffer, for
1445 // this iteration.
1446 TrackTime ticksPlayed = 0;
1447 for (auto& t : mAudioOutputs) {
1448 TrackTime ticksPlayedForThisTrack =
1449 PlayAudio(aMixer, t, oldProcessedTime);
1450 if (ticksPlayed == 0) {
1451 ticksPlayed = ticksPlayedForThisTrack;
1452 } else {
1453 MOZ_ASSERT(
1454 !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
1455 "Each track should have the same number of frames.");
1459 if (ticksPlayed == 0) {
1460 // Nothing was played, so the mixer doesn't know how many frames were
1461 // processed. We still tell it so AudioCallbackDriver knows how much has
1462 // been processed. (bug 1406027)
1463 aMixer->Mix(
1464 nullptr,
1465 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(),
1466 mStateComputedTime - oldProcessedTime, mSampleRate);
1468 aMixer->FinishMixing();
1471 if (!allBlockedForever) {
1472 EnsureNextIteration();
1476 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1477 MOZ_ASSERT(OnGraphThread());
1478 if (mForceShutDownReceived) {
1479 for (MediaTrack* track : AllTracks()) {
1480 track->OnGraphThreadDone();
1484 MonitorAutoLock lock(mMonitor);
1485 bool finalUpdate =
1486 mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
1487 PrepareUpdatesToMainThreadState(finalUpdate);
1488 if (!finalUpdate) {
1489 SwapMessageQueues();
1490 return true;
1492 // The JSContext will not be used again.
1493 // Clear main thread access while under monitor.
1494 mJSContext = nullptr;
1496 dom::WorkletThread::DeleteCycleCollectedJSContext();
1497 // Enter shutdown mode when this iteration is completed.
1498 // No need to Destroy tracks here. The main-thread owner of each
1499 // track is responsible for calling Destroy on them.
1500 return false;
1503 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime,
1504 GraphTime aIterationEnd,
1505 AudioMixer* aMixer) -> IterationResult {
1506 if (mGraphRunner) {
1507 return mGraphRunner->OneIteration(aStateTime, aIterationEnd, aMixer);
1510 return OneIterationImpl(aStateTime, aIterationEnd, aMixer);
1513 auto MediaTrackGraphImpl::OneIterationImpl(GraphTime aStateTime,
1514 GraphTime aIterationEnd,
1515 AudioMixer* aMixer)
1516 -> IterationResult {
1517 TRACE("MTG::OneIterationImpl");
1519 mIterationEndTime = aIterationEnd;
1521 if (SoftRealTimeLimitReached()) {
1522 TRACE("MTG::Demoting real-time thread!");
1523 DemoteThreadFromRealTime();
1526 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1527 // thread, and so the monitor need not be held to check mLifecycleState.
1528 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1529 // graphs that have not started.
1531 // While changes occur on mainthread, this assert confirms that
1532 // this code shouldn't run if mainthread might be changing the state (to
1533 // > LIFECYCLE_RUNNING)
1535 // Ignore mutex warning: static during execution of the graph
1536 MOZ_PUSH_IGNORE_THREAD_SAFETY
1537 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
1538 MOZ_POP_THREAD_SAFETY
1540 MOZ_ASSERT(OnGraphThread());
1542 WebCore::DenormalDisabler disabler;
1544 // Process graph message from the main thread for this iteration.
1545 RunMessagesInQueue();
1547 // Process MessagePort events.
1548 // These require a single thread, which has an nsThread with an event queue.
1549 if (mGraphRunner || !mRealtime) {
1550 TRACE("MTG::MessagePort events");
1551 NS_ProcessPendingEvents(nullptr);
1554 GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
1555 UpdateGraph(stateTime);
1557 mStateComputedTime = stateTime;
1559 GraphTime oldProcessedTime = mProcessedTime;
1560 Process(aMixer);
1561 MOZ_ASSERT(mProcessedTime == stateTime);
1563 UpdateCurrentTimeForTracks(oldProcessedTime);
1565 ProcessChunkMetadata(oldProcessedTime);
1567 // Process graph messages queued from RunMessageAfterProcessing() on this
1568 // thread during the iteration.
1569 RunMessagesInQueue();
1571 if (!UpdateMainThreadState()) {
1572 if (Switching()) {
1573 // We'll never get to do this switch. Clear mNextDriver to break the
1574 // ref-cycle graph->nextDriver->currentDriver->graph.
1575 SwitchAtNextIteration(nullptr);
1577 return IterationResult::CreateStop(
1578 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1579 &MediaTrackGraphImpl::SignalMainThreadCleanup));
1582 if (Switching()) {
1583 RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
1584 return IterationResult::CreateSwitchDriver(
1585 nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
1586 "MediaTrackGraphImpl::SetCurrentDriver", this,
1587 &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
1590 return IterationResult::CreateStillProcessing();
1593 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
1594 MOZ_ASSERT(NS_IsMainThread());
1595 mMonitor.AssertCurrentThreadOwns();
1597 MediaTrack* track = aUpdate->mTrack;
1598 if (!track) return;
1599 track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
1600 track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
1602 if (track->ShouldNotifyTrackEnded()) {
1603 track->NotifyMainThreadListeners();
1607 void MediaTrackGraphImpl::ForceShutDown() {
1608 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1609 LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
1611 if (mShutdownBlocker) {
1612 // Avoid waiting forever for a graph to shut down
1613 // synchronously. Reports are that some 3rd-party audio drivers
1614 // occasionally hang in shutdown (both for us and Chrome).
1615 NS_NewTimerWithCallback(
1616 getter_AddRefs(mShutdownTimer), this,
1617 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
1618 nsITimer::TYPE_ONE_SHOT);
1621 class Message final : public ControlMessage {
1622 public:
1623 explicit Message(MediaTrackGraphImpl* aGraph)
1624 : ControlMessage(nullptr), mGraph(aGraph) {}
1625 void Run() override {
1626 TRACE("MTG::ForceShutdown ControlMessage");
1627 mGraph->mForceShutDownReceived = true;
1629 // The graph owns this message.
1630 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
1633 if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
1634 // If both the track and port counts are zero, the regular shutdown
1635 // sequence will progress shortly to shutdown threads and destroy the graph.
1636 AppendMessage(MakeUnique<Message>(this));
1637 InterruptJS();
1641 NS_IMETHODIMP
1642 MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
1643 MOZ_ASSERT(NS_IsMainThread());
1644 NS_ASSERTION(!mShutdownBlocker,
1645 "MediaTrackGraph took too long to shut down!");
1646 // Sigh, graph took too long to shut down. Stop blocking system
1647 // shutdown and hope all is well.
1648 RemoveShutdownBlocker();
1649 return NS_OK;
1652 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1653 MOZ_ASSERT(NS_IsMainThread());
1654 MOZ_ASSERT(!mShutdownBlocker);
1656 class Blocker : public media::ShutdownBlocker {
1657 const RefPtr<MediaTrackGraphImpl> mGraph;
1659 public:
1660 Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
1661 : media::ShutdownBlocker(aName), mGraph(aGraph) {}
1663 NS_IMETHOD
1664 BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
1665 mGraph->ForceShutDown();
1666 return NS_OK;
1670 nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
1671 if (!barrier) {
1672 // We're already shutting down, we won't be able to add a blocker, bail.
1673 LOG(LogLevel::Error,
1674 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1675 this));
1676 return false;
1679 // Blocker names must be distinct.
1680 nsString blockerName;
1681 blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
1682 mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
1683 nsresult rv = barrier->AddBlocker(mShutdownBlocker,
1684 NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
1685 __LINE__, u"MediaTrackGraph shutdown"_ns);
1686 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
1687 return true;
1690 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1691 if (!mShutdownBlocker) {
1692 return;
1694 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
1695 mShutdownBlocker = nullptr;
1698 NS_IMETHODIMP
1699 MediaTrackGraphImpl::GetName(nsACString& aName) {
1700 aName.AssignLiteral("MediaTrackGraphImpl");
1701 return NS_OK;
1704 namespace {
1706 class MediaTrackGraphShutDownRunnable : public Runnable {
1707 public:
1708 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
1709 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
1710 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1711 // See bug 1535398.
1712 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
1713 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1714 MOZ_ASSERT(NS_IsMainThread());
1715 MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
1716 "We should know the graph thread control loop isn't running!");
1718 LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
1720 // We've asserted the graph isn't running. Use mDriver instead of
1721 // CurrentDriver to avoid thread-safety checks
1722 #if 0 // AudioCallbackDrivers are released asynchronously anyways
1723 // XXX a better test would be have setting mGraphDriverRunning make sure
1724 // any current callback has finished and block future ones -- or just
1725 // handle it all in Shutdown()!
1726 if (mGraph->mDriver->AsAudioCallbackDriver()) {
1727 MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
1729 #endif
1731 for (MediaTrackGraphImpl::PendingResumeOperation& op :
1732 mGraph->mPendingResumeOperations) {
1733 op.Abort();
1736 if (mGraph->mGraphRunner) {
1737 RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
1740 RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
1742 // Release the driver now so that an AudioCallbackDriver will release its
1743 // SharedThreadPool reference. Each SharedThreadPool reference must be
1744 // released before SharedThreadPool::SpinUntilEmpty() runs on
1745 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
1746 // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1747 // which won't otherwise release its reference on the graph until
1748 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1749 mGraph->SetCurrentDriver(nullptr);
1751 // Safe to access these without the monitor since the graph isn't running.
1752 // We may be one of several graphs. Drop ticket to eventually unblock
1753 // shutdown.
1754 if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
1755 MOZ_ASSERT(
1756 false,
1757 "AudioCallbackDriver took too long to shut down and we let shutdown"
1758 " continue - freezing and leaking");
1760 // The timer fired, so we may be deeper in shutdown now. Block any
1761 // further teardown and just leak, for safety.
1762 return NS_OK;
1765 // mGraph's thread is not running so it's OK to do whatever here
1766 for (MediaTrack* track : mGraph->AllTracks()) {
1767 // Clean up all MediaSegments since we cannot release Images too
1768 // late during shutdown. Also notify listeners that they were removed
1769 // so they can clean up any gfx resources.
1770 track->RemoveAllResourcesAndListenersImpl();
1773 #ifdef DEBUG
1775 MonitorAutoLock lock(mGraph->mMonitor);
1776 MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
1778 #endif
1779 mGraph->mPendingUpdateRunnables.Clear();
1781 mGraph->RemoveShutdownBlocker();
1783 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1784 // stage, since completion of that stage requires all tracks to be freed,
1785 // which requires shutdown to proceed.
1787 if (mGraph->IsEmpty()) {
1788 // mGraph is no longer needed, so delete it.
1789 mGraph->Destroy();
1790 } else {
1791 // The graph is not empty. We must be in a forced shutdown.
1792 // Some later AppendMessage will detect that the graph has
1793 // been emptied, and delete it.
1794 NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
1795 mGraph->LifecycleStateRef() =
1796 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
1798 return NS_OK;
1801 private:
1802 RefPtr<MediaTrackGraphImpl> mGraph;
1805 class MediaTrackGraphStableStateRunnable : public Runnable {
1806 public:
1807 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
1808 bool aSourceIsMTG)
1809 : Runnable("MediaTrackGraphStableStateRunnable"),
1810 mGraph(aGraph),
1811 mSourceIsMTG(aSourceIsMTG) {}
1812 NS_IMETHOD Run() override {
1813 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1814 if (mGraph) {
1815 mGraph->RunInStableState(mSourceIsMTG);
1817 return NS_OK;
1820 private:
1821 RefPtr<MediaTrackGraphImpl> mGraph;
1822 bool mSourceIsMTG;
1826 * Control messages forwarded from main thread to graph manager thread
1828 class CreateMessage : public ControlMessage {
1829 public:
1830 explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
1831 void Run() override {
1832 TRACE("MTG::AddTrackGraphThread ControlMessage");
1833 mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
1835 void RunDuringShutdown() override {
1836 // Make sure to run this message during shutdown too, to make sure
1837 // that we balance the number of tracks registered with the graph
1838 // as they're destroyed during shutdown.
1839 Run();
1843 } // namespace
1845 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
1846 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1848 nsTArray<nsCOMPtr<nsIRunnable>> runnables;
1849 // When we're doing a forced shutdown, pending control messages may be
1850 // run on the main thread via RunDuringShutdown. Those messages must
1851 // run without the graph monitor being held. So, we collect them here.
1852 nsTArray<UniquePtr<ControlMessage>> controlMessagesToRunDuringShutdown;
1855 MonitorAutoLock lock(mMonitor);
1856 if (aSourceIsMTG) {
1857 MOZ_ASSERT(mPostedRunInStableStateEvent);
1858 mPostedRunInStableStateEvent = false;
1861 // This should be kept in sync with the LifecycleState enum in
1862 // MediaTrackGraphImpl.h
1863 const char* LifecycleState_str[] = {
1864 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
1865 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1866 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1867 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
1869 if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
1870 LOG(LogLevel::Debug,
1871 ("%p: Running stable state callback. Current state: %s", this,
1872 LifecycleState_str[LifecycleStateRef()]));
1875 runnables = std::move(mUpdateRunnables);
1876 for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
1877 TrackUpdate* update = &mTrackUpdates[i];
1878 if (update->mTrack) {
1879 ApplyTrackUpdate(update);
1882 mTrackUpdates.Clear();
1884 mMainThreadGraphTime = mNextMainThreadGraphTime;
1886 if (mCurrentTaskMessageQueue.IsEmpty()) {
1887 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1888 IsEmpty()) {
1889 // Complete shutdown. First, ensure that this graph is no longer used.
1890 // A new graph graph will be created if one is needed.
1891 // Asynchronously clean up old graph. We don't want to do this
1892 // synchronously because it spins the event loop waiting for threads
1893 // to shut down, and we don't want to do that in a stable state handler.
1894 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1895 LOG(LogLevel::Debug,
1896 ("%p: Sending MediaTrackGraphShutDownRunnable", this));
1897 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1898 mMainThread->Dispatch(event.forget());
1900 } else {
1901 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
1902 MessageBlock* block = mBackMessageQueue.AppendElement();
1903 block->mMessages = std::move(mCurrentTaskMessageQueue);
1904 EnsureNextIteration();
1907 // If this MediaTrackGraph has entered regular (non-forced) shutdown it
1908 // is not able to process any more messages. Those messages being added to
1909 // the graph in the first place is an error.
1910 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
1911 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
1912 mForceShutDownReceived);
1915 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
1916 // Start the driver now. We couldn't start it earlier because the graph
1917 // might exit immediately on finding it has no tracks. The first message
1918 // for a new graph must create a track. Ensure that his message runs on
1919 // the first iteration.
1920 MOZ_ASSERT(MessagesQueued());
1921 SwapMessageQueues();
1923 LOG(LogLevel::Debug,
1924 ("%p: Starting a graph with a %s", this,
1925 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
1926 : "SystemClockDriver"));
1927 LifecycleStateRef() = LIFECYCLE_RUNNING;
1928 mGraphDriverRunning = true;
1929 RefPtr<GraphDriver> driver = CurrentDriver();
1930 driver->Start();
1931 // It's not safe to Shutdown() a thread from StableState, and
1932 // releasing this may shutdown a SystemClockDriver thread.
1933 // Proxy the release to outside of StableState.
1934 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
1935 driver.forget(),
1936 true); // always proxy
1939 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
1940 mForceShutDownReceived) {
1941 // Defer calls to RunDuringShutdown() to happen while mMonitor is not
1942 // held.
1943 for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
1944 MessageBlock& mb = mBackMessageQueue[i];
1945 controlMessagesToRunDuringShutdown.AppendElements(
1946 std::move(mb.mMessages));
1948 mBackMessageQueue.Clear();
1949 MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
1950 // Stop MediaTrackGraph threads.
1951 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1952 nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
1953 mMainThread->Dispatch(event.forget());
1956 mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
1959 // Make sure we get a new current time in the next event loop task
1960 if (!aSourceIsMTG) {
1961 MOZ_ASSERT(mPostedRunInStableState);
1962 mPostedRunInStableState = false;
1965 for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
1966 controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
1969 #ifdef DEBUG
1970 mCanRunMessagesSynchronously =
1971 !mGraphDriverRunning &&
1972 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
1973 #endif
1975 for (uint32_t i = 0; i < runnables.Length(); ++i) {
1976 runnables[i]->Run();
1980 void MediaTrackGraphImpl::EnsureRunInStableState() {
1981 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
1983 if (mPostedRunInStableState) return;
1984 mPostedRunInStableState = true;
1985 nsCOMPtr<nsIRunnable> event =
1986 new MediaTrackGraphStableStateRunnable(this, false);
1987 nsContentUtils::RunInStableState(event.forget());
1990 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
1991 MOZ_ASSERT(OnGraphThread());
1992 mMonitor.AssertCurrentThreadOwns();
1994 if (mPostedRunInStableStateEvent) return;
1995 mPostedRunInStableStateEvent = true;
1996 nsCOMPtr<nsIRunnable> event =
1997 new MediaTrackGraphStableStateRunnable(this, true);
1998 mMainThread->Dispatch(event.forget());
2001 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2002 MOZ_ASSERT(mDriver->OnThread());
2004 MonitorAutoLock lock(mMonitor);
2005 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2006 // graphs that have not started.
2007 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
2008 LOG(LogLevel::Debug,
2009 ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2010 LifecycleStateRef() =
2011 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
2012 EnsureStableStateEventPosted();
2015 void MediaTrackGraphImpl::AppendMessage(UniquePtr<ControlMessage> aMessage) {
2016 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2017 MOZ_RELEASE_ASSERT(!aMessage->GetTrack() ||
2018 !aMessage->GetTrack()->IsDestroyed());
2019 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
2021 if (!mGraphDriverRunning &&
2022 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
2023 // The graph control loop is not running and main thread cleanup has
2024 // happened. From now on we can't append messages to
2025 // mCurrentTaskMessageQueue, because that will never be processed again, so
2026 // just RunDuringShutdown this message. This should only happen during
2027 // forced shutdown, or after a non-realtime graph has finished processing.
2028 #ifdef DEBUG
2029 MOZ_ASSERT(mCanRunMessagesSynchronously);
2030 mCanRunMessagesSynchronously = false;
2031 #endif
2032 aMessage->RunDuringShutdown();
2033 #ifdef DEBUG
2034 mCanRunMessagesSynchronously = true;
2035 #endif
2036 if (IsEmpty() &&
2037 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
2038 Destroy();
2040 return;
2043 mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
2044 EnsureRunInStableState();
2047 void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
2048 mMainThread->Dispatch(std::move(aRunnable));
2051 MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
2052 MediaSegment* aSegment)
2053 : mSampleRate(aSampleRate),
2054 mType(aType),
2055 mSegment(aSegment),
2056 mStartTime(0),
2057 mForgottenTime(0),
2058 mEnded(false),
2059 mNotifiedEnded(false),
2060 mDisabledMode(DisabledTrackMode::ENABLED),
2061 mStartBlocking(GRAPH_TIME_MAX),
2062 mSuspendedCount(0),
2063 mMainThreadCurrentTime(0),
2064 mMainThreadEnded(false),
2065 mEndedNotificationSent(false),
2066 mMainThreadDestroyed(false),
2067 mGraph(nullptr) {
2068 MOZ_COUNT_CTOR(MediaTrack);
2069 MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
2072 MediaTrack::~MediaTrack() {
2073 MOZ_COUNT_DTOR(MediaTrack);
2074 NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
2075 NS_ASSERTION(mMainThreadListeners.IsEmpty(),
2076 "All main thread listeners should have been removed");
2079 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
2080 size_t amount = 0;
2082 // Not owned:
2083 // - mGraph - Not reported here
2084 // - mConsumers - elements
2085 // Future:
2086 // - mLastPlayedVideoFrame
2087 // - mTrackListeners - elements
2089 amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2090 amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
2091 amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
2093 return amount;
2096 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
2097 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
2100 void MediaTrack::IncrementSuspendCount() {
2101 ++mSuspendedCount;
2102 if (mSuspendedCount != 1 || !mGraph) {
2103 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2104 return;
2106 MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning());
2107 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2108 mConsumers[i]->Suspended();
2110 MOZ_ASSERT(mGraph->mTracks.Contains(this));
2111 mGraph->mTracks.RemoveElement(this);
2112 mGraph->mSuspendedTracks.AppendElement(this);
2113 mGraph->SetTrackOrderDirty();
2116 void MediaTrack::DecrementSuspendCount() {
2117 MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
2118 --mSuspendedCount;
2119 if (mSuspendedCount != 0 || !mGraph) {
2120 MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
2121 return;
2123 MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning());
2124 for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
2125 mConsumers[i]->Resumed();
2127 MOZ_ASSERT(mGraph->mSuspendedTracks.Contains(this));
2128 mGraph->mSuspendedTracks.RemoveElement(this);
2129 mGraph->mTracks.AppendElement(this);
2130 mGraph->SetTrackOrderDirty();
2133 void ProcessedMediaTrack::DecrementSuspendCount() {
2134 mCycleMarker = NOT_VISITED;
2135 MediaTrack::DecrementSuspendCount();
2138 MediaTrackGraphImpl* MediaTrack::GraphImpl() { return mGraph; }
2140 const MediaTrackGraphImpl* MediaTrack::GraphImpl() const { return mGraph; }
2142 MediaTrackGraph* MediaTrack::Graph() { return mGraph; }
2144 const MediaTrackGraph* MediaTrack::Graph() const { return mGraph; }
2146 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
2147 MOZ_ASSERT(!mGraph, "Should only be called once");
2148 MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
2149 mGraph = aGraph;
2152 void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
2153 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
2154 SetGraphImpl(graph);
2157 TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
2158 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2159 aTime <= mStartBlocking,
2160 "Incorrectly ignoring blocking!");
2161 return aTime - mStartTime;
2164 GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
2165 NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
2166 aTime + mStartTime <= mStartBlocking,
2167 "Incorrectly ignoring blocking!");
2168 return aTime + mStartTime;
2171 TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
2172 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
2175 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2176 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2178 for (auto& l : mTrackListeners.Clone()) {
2179 l->NotifyRemoved(Graph());
2181 mTrackListeners.Clear();
2183 RemoveAllDirectListenersImpl();
2185 if (mSegment) {
2186 mSegment->Clear();
2190 void MediaTrack::DestroyImpl() {
2191 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2192 mConsumers[i]->Disconnect();
2194 if (mSegment) {
2195 mSegment->Clear();
2197 mGraph = nullptr;
2200 void MediaTrack::Destroy() {
2201 // Keep this track alive until we leave this method
2202 RefPtr<MediaTrack> kungFuDeathGrip = this;
2204 class Message : public ControlMessage {
2205 public:
2206 explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2207 void RunDuringShutdown() override {
2208 TRACE("MediaTrack::Destroy ControlMessage");
2209 mTrack->RemoveAllResourcesAndListenersImpl();
2210 auto graph = mTrack->GraphImpl();
2211 mTrack->DestroyImpl();
2212 graph->RemoveTrackGraphThread(mTrack);
2214 void Run() override {
2215 mTrack->OnGraphThreadDone();
2216 RunDuringShutdown();
2219 // Keep a reference to the graph, since Message might RunDuringShutdown()
2220 // synchronously and make GraphImpl() invalid.
2221 RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
2222 graph->AppendMessage(MakeUnique<Message>(this));
2223 graph->RemoveTrack(this);
2224 // Message::RunDuringShutdown may have removed this track from the graph,
2225 // but our kungFuDeathGrip above will have kept this track alive if
2226 // necessary.
2227 mMainThreadDestroyed = true;
2230 TrackTime MediaTrack::GetEnd() const {
2231 return mSegment ? mSegment->GetDuration() : 0;
2234 void MediaTrack::AddAudioOutput(void* aKey) {
2235 class Message : public ControlMessage {
2236 public:
2237 Message(MediaTrack* aTrack, void* aKey)
2238 : ControlMessage(aTrack), mKey(aKey) {}
2239 void Run() override {
2240 TRACE("MediaTrack::AddAudioOutputImpl ControlMessage");
2241 mTrack->AddAudioOutputImpl(mKey);
2243 void* mKey;
2245 if (mMainThreadDestroyed) {
2246 return;
2248 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2251 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
2252 float aVolume) {
2253 for (auto& tkv : mAudioOutputs) {
2254 if (tkv.mKey == aKey && aTrack == tkv.mTrack) {
2255 tkv.mVolume = aVolume;
2256 return;
2259 MOZ_CRASH("Audio stream key not found when setting the volume.");
2262 void MediaTrack::SetAudioOutputVolumeImpl(void* aKey, float aVolume) {
2263 MOZ_ASSERT(GraphImpl()->OnGraphThread());
2264 GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
2267 void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
2268 class Message : public ControlMessage {
2269 public:
2270 Message(MediaTrack* aTrack, void* aKey, float aVolume)
2271 : ControlMessage(aTrack), mKey(aKey), mVolume(aVolume) {}
2272 void Run() override {
2273 TRACE("MediaTrack::SetAudioOutputVolumeImpl ControlMessage");
2274 mTrack->SetAudioOutputVolumeImpl(mKey, mVolume);
2276 void* mKey;
2277 float mVolume;
2279 if (mMainThreadDestroyed) {
2280 return;
2282 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey, aVolume));
2285 void MediaTrack::AddAudioOutputImpl(void* aKey) {
2286 LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
2287 GraphImpl()->RegisterAudioOutput(this, aKey);
2290 void MediaTrack::RemoveAudioOutputImpl(void* aKey) {
2291 LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
2292 GraphImpl()->UnregisterAudioOutput(this, aKey);
2295 void MediaTrack::RemoveAudioOutput(void* aKey) {
2296 class Message : public ControlMessage {
2297 public:
2298 explicit Message(MediaTrack* aTrack, void* aKey)
2299 : ControlMessage(aTrack), mKey(aKey) {}
2300 void Run() override {
2301 TRACE("MediaTrack::RemoveAudioOutputImpl ControlMessage");
2302 mTrack->RemoveAudioOutputImpl(mKey);
2304 void* mKey;
2306 if (mMainThreadDestroyed) {
2307 return;
2309 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
2312 void MediaTrack::Suspend() {
2313 class Message : public ControlMessage {
2314 public:
2315 explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2316 void Run() override {
2317 TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2318 mTrack->IncrementSuspendCount();
2322 // This can happen if this method has been called asynchronously, and the
2323 // track has been destroyed since then.
2324 if (mMainThreadDestroyed) {
2325 return;
2327 GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2330 void MediaTrack::Resume() {
2331 class Message : public ControlMessage {
2332 public:
2333 explicit Message(MediaTrack* aTrack) : ControlMessage(aTrack) {}
2334 void Run() override {
2335 TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2336 mTrack->DecrementSuspendCount();
2340 // This can happen if this method has been called asynchronously, and the
2341 // track has been destroyed since then.
2342 if (mMainThreadDestroyed) {
2343 return;
2345 GraphImpl()->AppendMessage(MakeUnique<Message>(this));
2348 void MediaTrack::AddListenerImpl(
2349 already_AddRefed<MediaTrackListener> aListener) {
2350 RefPtr<MediaTrackListener> l(aListener);
2351 mTrackListeners.AppendElement(std::move(l));
2353 PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
2354 mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
2355 Graph(), lastPrincipalHandle);
2356 if (mNotifiedEnded) {
2357 mTrackListeners.LastElement()->NotifyEnded(Graph());
2359 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
2360 mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2364 void MediaTrack::AddListener(MediaTrackListener* aListener) {
2365 class Message : public ControlMessage {
2366 public:
2367 Message(MediaTrack* aTrack, MediaTrackListener* aListener)
2368 : ControlMessage(aTrack), mListener(aListener) {}
2369 void Run() override {
2370 TRACE("MediaTrack::AddListenerImpl ControlMessage");
2371 mTrack->AddListenerImpl(mListener.forget());
2373 RefPtr<MediaTrackListener> mListener;
2375 MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
2376 if (mMainThreadDestroyed) {
2377 return;
2379 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2382 void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
2383 for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
2384 if (mTrackListeners[i] == aListener) {
2385 mTrackListeners[i]->NotifyRemoved(Graph());
2386 mTrackListeners.RemoveElementAt(i);
2387 return;
2392 RefPtr<GenericPromise> MediaTrack::RemoveListener(
2393 MediaTrackListener* aListener) {
2394 class Message : public ControlMessage {
2395 public:
2396 Message(MediaTrack* aTrack, MediaTrackListener* aListener)
2397 : ControlMessage(aTrack), mListener(aListener) {}
2398 void Run() override {
2399 TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2400 mTrack->RemoveListenerImpl(mListener);
2401 mRemovedPromise.Resolve(true, __func__);
2403 void RunDuringShutdown() override {
2404 // During shutdown we still want the listener's NotifyRemoved to be
2405 // called, since not doing that might block shutdown of other modules.
2406 Run();
2408 RefPtr<MediaTrackListener> mListener;
2409 MozPromiseHolder<GenericPromise> mRemovedPromise;
2412 UniquePtr<Message> message = MakeUnique<Message>(this, aListener);
2413 RefPtr<GenericPromise> p = message->mRemovedPromise.Ensure(__func__);
2414 if (mMainThreadDestroyed) {
2415 message->mRemovedPromise.Reject(NS_ERROR_FAILURE, __func__);
2416 return p;
2418 GraphImpl()->AppendMessage(std::move(message));
2419 return p;
2422 void MediaTrack::AddDirectListenerImpl(
2423 already_AddRefed<DirectMediaTrackListener> aListener) {
2424 // Base implementation, for tracks that don't support direct track listeners.
2425 RefPtr<DirectMediaTrackListener> listener = aListener;
2426 listener->NotifyDirectListenerInstalled(
2427 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
2430 void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
2431 class Message : public ControlMessage {
2432 public:
2433 Message(MediaTrack* aTrack, DirectMediaTrackListener* aListener)
2434 : ControlMessage(aTrack), mListener(aListener) {}
2435 void Run() override {
2436 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2437 mTrack->AddDirectListenerImpl(mListener.forget());
2439 RefPtr<DirectMediaTrackListener> mListener;
2441 if (mMainThreadDestroyed) {
2442 return;
2444 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2447 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
2448 // Base implementation, the listener was never added so nothing to do.
2451 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
2452 class Message : public ControlMessage {
2453 public:
2454 Message(MediaTrack* aTrack, DirectMediaTrackListener* aListener)
2455 : ControlMessage(aTrack), mListener(aListener) {}
2456 void Run() override {
2457 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2458 mTrack->RemoveDirectListenerImpl(mListener);
2460 void RunDuringShutdown() override {
2461 // During shutdown we still want the listener's
2462 // NotifyDirectListenerUninstalled to be called, since not doing that
2463 // might block shutdown of other modules.
2464 Run();
2466 RefPtr<DirectMediaTrackListener> mListener;
2468 if (mMainThreadDestroyed) {
2469 return;
2471 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aListener));
2474 void MediaTrack::RunAfterPendingUpdates(
2475 already_AddRefed<nsIRunnable> aRunnable) {
2476 MOZ_ASSERT(NS_IsMainThread());
2477 MediaTrackGraphImpl* graph = GraphImpl();
2478 nsCOMPtr<nsIRunnable> runnable(aRunnable);
2480 class Message : public ControlMessage {
2481 public:
2482 Message(MediaTrack* aTrack, already_AddRefed<nsIRunnable> aRunnable)
2483 : ControlMessage(aTrack), mRunnable(aRunnable) {}
2484 void Run() override {
2485 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2486 mTrack->Graph()->DispatchToMainThreadStableState(mRunnable.forget());
2488 void RunDuringShutdown() override {
2489 // Don't run mRunnable now as it may call AppendMessage() which would
2490 // assume that there are no remaining controlMessagesToRunDuringShutdown.
2491 MOZ_ASSERT(NS_IsMainThread());
2492 mTrack->GraphImpl()->Dispatch(mRunnable.forget());
2495 private:
2496 nsCOMPtr<nsIRunnable> mRunnable;
2499 if (mMainThreadDestroyed) {
2500 return;
2502 graph->AppendMessage(MakeUnique<Message>(this, runnable.forget()));
2505 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
2506 MOZ_DIAGNOSTIC_ASSERT(
2507 aMode == DisabledTrackMode::ENABLED ||
2508 mDisabledMode == DisabledTrackMode::ENABLED,
2509 "Changing disabled track mode for a track is not allowed");
2510 DisabledTrackMode oldMode = CombinedDisabledMode();
2511 mDisabledMode = aMode;
2512 NotifyIfDisabledModeChangedFrom(oldMode);
2515 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
2516 class Message : public ControlMessage {
2517 public:
2518 Message(MediaTrack* aTrack, DisabledTrackMode aMode)
2519 : ControlMessage(aTrack), mMode(aMode) {}
2520 void Run() override {
2521 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2522 mTrack->SetDisabledTrackModeImpl(mMode);
2524 DisabledTrackMode mMode;
2526 if (mMainThreadDestroyed) {
2527 return;
2529 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aMode));
2532 void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
2533 MediaSegment* aRawSegment) {
2534 if (mDisabledMode == DisabledTrackMode::ENABLED) {
2535 return;
2537 if (mDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
2538 aSegment->ReplaceWithDisabled();
2539 if (aRawSegment) {
2540 aRawSegment->ReplaceWithDisabled();
2542 } else if (mDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
2543 aSegment->ReplaceWithNull();
2544 if (aRawSegment) {
2545 aRawSegment->ReplaceWithNull();
2547 } else {
2548 MOZ_CRASH("Unsupported mode");
2552 void MediaTrack::AddMainThreadListener(
2553 MainThreadMediaTrackListener* aListener) {
2554 MOZ_ASSERT(NS_IsMainThread());
2555 MOZ_ASSERT(aListener);
2556 MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
2558 mMainThreadListeners.AppendElement(aListener);
2560 // If it is not yet time to send the notification, then exit here.
2561 if (!mEndedNotificationSent) {
2562 return;
2565 class NotifyRunnable final : public Runnable {
2566 public:
2567 explicit NotifyRunnable(MediaTrack* aTrack)
2568 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
2570 NS_IMETHOD Run() override {
2571 TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2572 MOZ_ASSERT(NS_IsMainThread());
2573 mTrack->NotifyMainThreadListeners();
2574 return NS_OK;
2577 private:
2578 ~NotifyRunnable() = default;
2580 RefPtr<MediaTrack> mTrack;
2583 nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
2584 GraphImpl()->Dispatch(runnable.forget());
2587 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
2588 GraphTime aBlockedTime) {
2589 mStartTime += aBlockedTime;
2591 if (!mSegment) {
2592 // No data to be forgotten.
2593 return;
2596 TrackTime time = aCurrentTime - mStartTime;
2597 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2598 // spend too much time pruning segments.
2599 const TrackTime minChunkSize = mSampleRate * 50 / 1000;
2600 if (time < mForgottenTime + minChunkSize) {
2601 return;
2604 mForgottenTime = std::min(GetEnd() - 1, time);
2605 mSegment->ForgetUpTo(mForgottenTime);
2608 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
2609 DisabledTrackMode mode = CombinedDisabledMode();
2610 if (aOldMode == mode) {
2611 return;
2614 for (const auto& listener : mTrackListeners) {
2615 listener->NotifyEnabledStateChanged(
2616 Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
2619 for (const auto& c : mConsumers) {
2620 if (c->GetDestination()) {
2621 c->GetDestination()->OnInputDisabledModeChanged(mode);
2626 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
2627 TrackRate aSampleRate)
2628 : MediaTrack(aSampleRate, aType,
2629 aType == MediaSegment::AUDIO
2630 ? static_cast<MediaSegment*>(new AudioSegment())
2631 : static_cast<MediaSegment*>(new VideoSegment())),
2632 mMutex("mozilla::media::SourceMediaTrack") {
2633 mUpdateTrack = MakeUnique<TrackData>();
2634 mUpdateTrack->mInputRate = aSampleRate;
2635 mUpdateTrack->mResamplerChannelCount = 0;
2636 mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
2637 mUpdateTrack->mEnded = false;
2638 mUpdateTrack->mPullingEnabled = false;
2639 mUpdateTrack->mGraphThreadDone = false;
2642 void SourceMediaTrack::DestroyImpl() {
2643 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2644 for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
2645 // Disconnect before we come under mMutex's lock since it can call back
2646 // through RemoveDirectListenerImpl() and deadlock.
2647 mConsumers[i]->Disconnect();
2650 // Hold mMutex while mGraph is reset so that other threads holding mMutex
2651 // can null-check know that the graph will not destroyed.
2652 MutexAutoLock lock(mMutex);
2653 mUpdateTrack = nullptr;
2654 MediaTrack::DestroyImpl();
2657 void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
2658 class Message : public ControlMessage {
2659 public:
2660 Message(SourceMediaTrack* aTrack, bool aEnabled)
2661 : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
2662 void Run() override {
2663 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2664 MutexAutoLock lock(mTrack->mMutex);
2665 if (!mTrack->mUpdateTrack) {
2666 // We can't enable pulling for a track that has ended. We ignore
2667 // this if we're disabling pulling, since shutdown sequences are
2668 // complex. If there's truly an issue we'll have issues enabling anyway.
2669 MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
2670 return;
2672 MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
2673 "Pulling is not allowed for video");
2674 mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
2676 SourceMediaTrack* mTrack;
2677 bool mEnabled;
2679 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
2682 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
2683 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2684 TrackTime t;
2685 TrackTime current;
2687 if (mEnded) {
2688 return false;
2690 MutexAutoLock lock(mMutex);
2691 if (mUpdateTrack->mEnded) {
2692 return false;
2694 if (!mUpdateTrack->mPullingEnabled) {
2695 return false;
2697 // Compute how much track time we'll need assuming we don't block
2698 // the track at all.
2699 t = GraphTimeToTrackTime(aDesiredUpToTime);
2700 current = GetEnd() + mUpdateTrack->mData->GetDuration();
2702 if (t <= current) {
2703 return false;
2705 LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2706 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
2707 GraphImpl()->MediaTimeToSeconds(current)));
2708 for (auto& l : mTrackListeners) {
2709 l->NotifyPull(Graph(), current, t);
2711 return true;
2715 * This moves chunks from aIn to aOut. For audio this is simple. For video
2716 * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2718 * We also handle "resetters" from captured media elements. This type of source
2719 * pushes future frames into the track, and should it need to remove some, e.g.,
2720 * because of a seek or pause, it tells us by letting time go backwards. Without
2721 * this, tracks would be live for too long after a seek or pause.
2723 static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
2724 MediaSegment* aOut, TrackTime aCurrentTime,
2725 TrackTime aDesiredUpToTime)
2726 MOZ_REQUIRES(aTrack->GetMutex()) {
2727 MOZ_ASSERT(aIn->GetType() == aOut->GetType());
2728 MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
2729 MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
2730 if (aIn->GetType() == MediaSegment::AUDIO) {
2731 AudioSegment* in = static_cast<AudioSegment*>(aIn);
2732 AudioSegment* out = static_cast<AudioSegment*>(aOut);
2733 TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
2734 TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
2736 out->AppendSlice(*in, 0, end);
2737 in->RemoveLeading(end);
2739 aTrack->GetMutex().AssertCurrentThreadOwns();
2740 out->ApplyVolume(aTrack->GetVolumeLocked());
2741 } else {
2742 VideoSegment* in = static_cast<VideoSegment*>(aIn);
2743 VideoSegment* out = static_cast<VideoSegment*>(aOut);
2744 for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
2745 MOZ_ASSERT(!c->mTimeStamp.IsNull());
2746 VideoChunk* last = out->GetLastChunk();
2747 if (!last || last->mTimeStamp.IsNull()) {
2748 // This is the first frame, or the last frame pushed to `out` has been
2749 // all consumed. Just append and we deal with its duration later.
2750 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2751 c->mFrame.GetIntrinsicSize(),
2752 c->mFrame.GetPrincipalHandle(),
2753 c->mFrame.GetForceBlack(), c->mTimeStamp);
2754 if (c->GetDuration() > 0) {
2755 out->ExtendLastFrameBy(c->GetDuration());
2757 continue;
2760 // We now know when this frame starts, aka when the last frame ends.
2762 if (c->mTimeStamp < last->mTimeStamp) {
2763 // Time is going backwards. This is a resetting frame from
2764 // DecodedStream. Clear everything up to currentTime.
2765 out->Clear();
2766 out->AppendNullData(aCurrentTime);
2769 // Append the current frame (will have duration 0).
2770 out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
2771 c->mFrame.GetIntrinsicSize(),
2772 c->mFrame.GetPrincipalHandle(),
2773 c->mFrame.GetForceBlack(), c->mTimeStamp);
2774 if (c->GetDuration() > 0) {
2775 out->ExtendLastFrameBy(c->GetDuration());
2778 if (out->GetDuration() < aDesiredUpToTime) {
2779 out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
2781 in->Clear();
2782 MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
2786 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
2787 GraphTime aDesiredUpToTime) {
2788 MutexAutoLock lock(mMutex);
2790 if (!mUpdateTrack) {
2791 MOZ_ASSERT(mEnded);
2792 return;
2795 TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
2797 ApplyTrackDisabling(mUpdateTrack->mData.get());
2799 if (!mUpdateTrack->mData->IsEmpty()) {
2800 for (const auto& l : mTrackListeners) {
2801 l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
2804 TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
2805 TrackTime endTime = trackDesiredUpToTime;
2806 if (mUpdateTrack->mEnded) {
2807 endTime = std::min(trackDesiredUpToTime,
2808 GetEnd() + mUpdateTrack->mData->GetDuration());
2810 LOG(LogLevel::Verbose,
2811 ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
2812 GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
2813 MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
2814 trackCurrentTime, endTime);
2815 if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
2816 mEnded = true;
2817 mUpdateTrack = nullptr;
2821 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
2822 mMutex.AssertCurrentThreadOwns();
2823 if (aSegment->GetType() != MediaSegment::AUDIO ||
2824 mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
2825 return;
2827 AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
2828 segment->ResampleChunks(mUpdateTrack->mResampler,
2829 &mUpdateTrack->mResamplerChannelCount,
2830 mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
2833 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
2834 GraphTime aCurrentTime, GraphTime aBlockedTime) {
2835 MutexAutoLock lock(mMutex);
2836 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
2839 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
2840 MutexAutoLock lock(mMutex);
2841 if (!mUpdateTrack) {
2842 return;
2844 MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
2845 // Set the new input rate and reset the resampler.
2846 mUpdateTrack->mInputRate = aRate;
2847 mUpdateTrack->mResampler.own(nullptr);
2848 mUpdateTrack->mResamplerChannelCount = 0;
2851 TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
2852 MediaSegment* aRawSegment) {
2853 MutexAutoLock lock(mMutex);
2854 MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
2855 TrackTime appended = 0;
2856 if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
2857 aSegment->Clear();
2858 return appended;
2861 // Data goes into mData, and on the next iteration of the MTG moves
2862 // into the track's segment after NotifyQueuedTrackChanges(). This adds
2863 // 0-10ms of delay before data gets to direct listeners.
2864 // Indirect listeners (via subsequent TrackUnion nodes) are synced to
2865 // playout time, and so can be delayed by buffering.
2867 // Apply track disabling before notifying any consumers directly
2868 // or inserting into the graph
2869 ApplyTrackDisabling(aSegment, aRawSegment);
2871 ResampleAudioToGraphSampleRate(aSegment);
2873 // Must notify first, since AppendFrom() will empty out aSegment
2874 NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
2875 appended = aSegment->GetDuration();
2876 mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead
2878 auto graph = GraphImpl();
2879 MonitorAutoLock lock(graph->GetMonitor());
2880 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
2881 graph->EnsureNextIteration();
2885 return appended;
2888 TrackTime SourceMediaTrack::ClearFutureData() {
2889 MutexAutoLock lock(mMutex);
2890 auto graph = GraphImpl();
2891 if (!mUpdateTrack || !graph) {
2892 return 0;
2895 TrackTime duration = mUpdateTrack->mData->GetDuration();
2896 mUpdateTrack->mData->Clear();
2897 return duration;
2900 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
2901 mMutex.AssertCurrentThreadOwns();
2903 for (const auto& l : mDirectTrackListeners) {
2904 TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of
2905 // the internal buffer
2906 l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
2907 *aSegment);
2911 void SourceMediaTrack::AddDirectListenerImpl(
2912 already_AddRefed<DirectMediaTrackListener> aListener) {
2913 MutexAutoLock lock(mMutex);
2915 RefPtr<DirectMediaTrackListener> listener = aListener;
2916 LOG(LogLevel::Debug,
2917 ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
2918 listener.get(), this));
2920 MOZ_ASSERT(mType == MediaSegment::VIDEO);
2921 for (const auto& l : mDirectTrackListeners) {
2922 if (l == listener) {
2923 listener->NotifyDirectListenerInstalled(
2924 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
2925 return;
2929 mDirectTrackListeners.AppendElement(listener);
2931 LOG(LogLevel::Debug,
2932 ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
2933 listener->NotifyDirectListenerInstalled(
2934 DirectMediaTrackListener::InstallationResult::SUCCESS);
2936 if (mDisabledMode != DisabledTrackMode::ENABLED) {
2937 listener->IncreaseDisabled(mDisabledMode);
2940 if (mEnded) {
2941 return;
2944 // Pass buffered data to the listener
2945 VideoSegment bufferedData;
2946 size_t videoFrames = 0;
2947 VideoSegment& segment = *GetData<VideoSegment>();
2948 for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
2949 iter.Next()) {
2950 if (iter->mTimeStamp.IsNull()) {
2951 // No timestamp means this is only for the graph's internal book-keeping,
2952 // denoting a late start of the track.
2953 continue;
2955 ++videoFrames;
2956 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2957 iter->mFrame.GetIntrinsicSize(),
2958 iter->mFrame.GetPrincipalHandle(),
2959 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2962 VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
2963 for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
2964 iter.Next()) {
2965 ++videoFrames;
2966 MOZ_ASSERT(!iter->mTimeStamp.IsNull());
2967 bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
2968 iter->mFrame.GetIntrinsicSize(),
2969 iter->mFrame.GetPrincipalHandle(),
2970 iter->mFrame.GetForceBlack(), iter->mTimeStamp);
2973 LOG(LogLevel::Info,
2974 ("%p: Notifying direct listener %p of %zu video frames and duration "
2975 "%" PRId64,
2976 GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
2977 listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
2980 void SourceMediaTrack::RemoveDirectListenerImpl(
2981 DirectMediaTrackListener* aListener) {
2982 MutexAutoLock lock(mMutex);
2983 for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
2984 const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
2985 if (l == aListener) {
2986 if (mDisabledMode != DisabledTrackMode::ENABLED) {
2987 aListener->DecreaseDisabled(mDisabledMode);
2989 aListener->NotifyDirectListenerUninstalled();
2990 mDirectTrackListeners.RemoveElementAt(i);
2995 void SourceMediaTrack::End() {
2996 MutexAutoLock lock(mMutex);
2997 if (!mUpdateTrack) {
2998 // Already ended
2999 return;
3001 mUpdateTrack->mEnded = true;
3002 if (auto graph = GraphImpl()) {
3003 MonitorAutoLock lock(graph->GetMonitor());
3004 if (graph->CurrentDriver()) { // graph has not completed forced shutdown
3005 graph->EnsureNextIteration();
3010 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
3012 MutexAutoLock lock(mMutex);
3013 for (const auto& l : mDirectTrackListeners) {
3014 DisabledTrackMode oldMode = mDisabledMode;
3015 bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
3016 if (!oldEnabled && aMode == DisabledTrackMode::ENABLED) {
3017 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3018 "direct listener enabled",
3019 GraphImpl(), this));
3020 l->DecreaseDisabled(oldMode);
3021 } else if (oldEnabled && aMode != DisabledTrackMode::ENABLED) {
3022 LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
3023 "direct listener disabled",
3024 GraphImpl(), this));
3025 l->IncreaseDisabled(aMode);
3029 MediaTrack::SetDisabledTrackModeImpl(aMode);
3032 uint32_t SourceMediaTrack::NumberOfChannels() const {
3033 AudioSegment* audio = GetData<AudioSegment>();
3034 MOZ_DIAGNOSTIC_ASSERT(audio);
3035 if (!audio) {
3036 return 0;
3038 return audio->MaxChannelCount();
3041 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3042 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3043 MutexAutoLock lock(mMutex);
3045 for (auto& l : mDirectTrackListeners.Clone()) {
3046 l->NotifyDirectListenerUninstalled();
3048 mDirectTrackListeners.Clear();
3051 void SourceMediaTrack::SetVolume(float aVolume) {
3052 MutexAutoLock lock(mMutex);
3053 mVolume = aVolume;
3056 float SourceMediaTrack::GetVolumeLocked() {
3057 mMutex.AssertCurrentThreadOwns();
3058 return mVolume;
3061 SourceMediaTrack::~SourceMediaTrack() = default;
3063 void MediaInputPort::Init() {
3064 mGraph->AssertOnGraphThreadOrNotRunning();
3065 LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)",
3066 mSource->GraphImpl(), this, mSource, mDest));
3067 // Only connect the port if it wasn't disconnected on allocation.
3068 if (mSource) {
3069 mSource->AddConsumer(this);
3070 mDest->AddInput(this);
3072 // mPortCount decremented via MediaInputPort::Destroy's message
3073 ++mGraph->mPortCount;
3076 void MediaInputPort::Disconnect() {
3077 mGraph->AssertOnGraphThreadOrNotRunning();
3078 NS_ASSERTION(!mSource == !mDest,
3079 "mSource and mDest must either both be null or both non-null");
3081 if (!mSource) {
3082 return;
3085 mSource->RemoveConsumer(this);
3086 mDest->RemoveInput(this);
3087 mSource = nullptr;
3088 mDest = nullptr;
3090 mGraph->SetTrackOrderDirty();
3093 MediaTrack* MediaInputPort::GetSource() const {
3094 mGraph->AssertOnGraphThreadOrNotRunning();
3095 return mSource;
3098 ProcessedMediaTrack* MediaInputPort::GetDestination() const {
3099 mGraph->AssertOnGraphThreadOrNotRunning();
3100 return mDest;
3103 MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
3104 MediaInputPort const* aPort, GraphTime aTime) {
3105 InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
3106 if (!aPort) {
3107 result.mStart = aTime;
3108 result.mInputIsBlocked = true;
3109 return result;
3111 aPort->mGraph->AssertOnGraphThreadOrNotRunning();
3112 if (aTime >= aPort->mDest->mStartBlocking) {
3113 return result;
3115 result.mStart = aTime;
3116 result.mEnd = aPort->mDest->mStartBlocking;
3117 result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
3118 if (!result.mInputIsBlocked) {
3119 result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
3121 return result;
3124 void MediaInputPort::Suspended() {
3125 mGraph->AssertOnGraphThreadOrNotRunning();
3126 mDest->InputSuspended(this);
3129 void MediaInputPort::Resumed() {
3130 mGraph->AssertOnGraphThreadOrNotRunning();
3131 mDest->InputResumed(this);
3134 void MediaInputPort::Destroy() {
3135 class Message : public ControlMessage {
3136 public:
3137 explicit Message(MediaInputPort* aPort)
3138 : ControlMessage(nullptr), mPort(aPort) {}
3139 void Run() override {
3140 TRACE("MediaInputPort::Destroy ControlMessage");
3141 mPort->Disconnect();
3142 --mPort->GraphImpl()->mPortCount;
3143 mPort->SetGraphImpl(nullptr);
3144 NS_RELEASE(mPort);
3146 void RunDuringShutdown() override { Run(); }
3147 MediaInputPort* mPort;
3149 // Keep a reference to the graph, since Message might RunDuringShutdown()
3150 // synchronously and make GraphImpl() invalid.
3151 RefPtr<MediaTrackGraphImpl> graph = mGraph;
3152 graph->AppendMessage(MakeUnique<Message>(this));
3153 --graph->mMainThreadPortCount;
3156 MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
3157 mGraph->AssertOnGraphThreadOrNotRunning();
3158 return mGraph;
3161 MediaTrackGraph* MediaInputPort::Graph() const {
3162 mGraph->AssertOnGraphThreadOrNotRunning();
3163 return mGraph;
3166 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
3167 MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
3168 DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
3169 MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
3170 mGraph = aGraph;
3173 already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
3174 MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
3175 // This method creates two references to the MediaInputPort: one for
3176 // the main thread, and one for the MediaTrackGraph.
3177 class Message : public ControlMessage {
3178 public:
3179 explicit Message(MediaInputPort* aPort)
3180 : ControlMessage(aPort->mDest), mPort(aPort) {}
3181 void Run() override {
3182 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3183 mPort->Init();
3184 // The graph holds its reference implicitly
3185 mPort->GraphImpl()->SetTrackOrderDirty();
3186 Unused << mPort.forget();
3188 void RunDuringShutdown() override { Run(); }
3189 RefPtr<MediaInputPort> mPort;
3192 MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
3193 RefPtr<MediaInputPort> port;
3194 if (aTrack->IsDestroyed()) {
3195 // Create a port that's disconnected, which is what it'd be after its source
3196 // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3197 // this later is fine.
3198 port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
3199 aOutputNumber);
3200 } else {
3201 MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
3202 port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
3203 aOutputNumber);
3205 ++GraphImpl()->mMainThreadPortCount;
3206 GraphImpl()->AppendMessage(MakeUnique<Message>(port));
3207 return port.forget();
3210 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
3211 class Message : public ControlMessage {
3212 public:
3213 Message(ProcessedMediaTrack* aTrack, bool aAutoend)
3214 : ControlMessage(aTrack), mAutoend(aAutoend) {}
3215 void Run() override {
3216 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3217 static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
3219 bool mAutoend;
3221 if (mMainThreadDestroyed) {
3222 return;
3224 GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
3227 void ProcessedMediaTrack::DestroyImpl() {
3228 for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
3229 mInputs[i]->Disconnect();
3232 for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
3233 mSuspendedInputs[i]->Disconnect();
3236 MediaTrack::DestroyImpl();
3237 // The track order is only important if there are connections, in which
3238 // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3239 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3240 // SetTrackOrderDirty(), for other reasons.
3243 MediaTrackGraphImpl::MediaTrackGraphImpl(
3244 GraphDriverType aDriverRequested, GraphRunType aRunTypeRequested,
3245 TrackRate aSampleRate, uint32_t aChannelCount,
3246 CubebUtils::AudioDeviceID aOutputDeviceID,
3247 nsISerialEventTarget* aMainThread)
3248 : MediaTrackGraph(aSampleRate),
3249 mGraphRunner(aRunTypeRequested == SINGLE_THREAD
3250 ? GraphRunner::Create(this)
3251 : already_AddRefed<GraphRunner>(nullptr)),
3252 mFirstCycleBreaker(0)
3253 // An offline graph is not initially processing.
3255 mEndTime(aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX),
3256 mPortCount(0),
3257 mOutputDeviceID(aOutputDeviceID),
3258 mMonitor("MediaTrackGraphImpl"),
3259 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
3260 mPostedRunInStableStateEvent(false),
3261 mGraphDriverRunning(false),
3262 mPostedRunInStableState(false),
3263 mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER),
3264 mTrackOrderDirty(false),
3265 mMainThread(aMainThread),
3266 mSelfRef(this),
3267 mGlobalVolume(CubebUtils::GetVolumeScale())
3268 #ifdef DEBUG
3270 mCanRunMessagesSynchronously(false)
3271 #endif
3273 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3274 mAudioOutputLatency(0.0),
3275 mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
3276 bool failedToGetShutdownBlocker = false;
3277 if (!IsNonRealtime()) {
3278 failedToGetShutdownBlocker = !AddShutdownBlocker();
3281 if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
3282 failedToGetShutdownBlocker) {
3283 // At least one of the following happened
3284 // - Failed to create thread.
3285 // - Failed to install a shutdown blocker when one is needed.
3286 // Because we have a fail state, jump to last phase of the lifecycle.
3287 mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
3288 RemoveShutdownBlocker(); // No-op if blocker wasn't added.
3289 #ifdef DEBUG
3290 mCanRunMessagesSynchronously = true;
3291 #endif
3292 return;
3294 if (mRealtime) {
3295 if (aDriverRequested == AUDIO_THREAD_DRIVER) {
3296 // Always start with zero input channels, and no particular preferences
3297 // for the input channel.
3298 mDriver = new AudioCallbackDriver(this, nullptr, mSampleRate,
3299 aChannelCount, 0, mOutputDeviceID,
3300 nullptr, AudioInputType::Unknown);
3301 } else {
3302 mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
3304 } else {
3305 mDriver =
3306 new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
3309 mLastMainThreadUpdate = TimeStamp::Now();
3311 RegisterWeakAsyncMemoryReporter(this);
3314 #ifdef DEBUG
3315 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
3316 return aDriver->OnThread() ||
3317 (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
3319 #endif
3321 void MediaTrackGraphImpl::Destroy() {
3322 // First unregister from memory reporting.
3323 UnregisterWeakMemoryReporter(this);
3325 // Clear the self reference which will destroy this instance if all
3326 // associated GraphDrivers are destroyed.
3327 mSelfRef = nullptr;
3330 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3331 // GTests can create a graph without a window.
3332 /* static */
3333 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
3334 uint64_t aWindowID, bool aShouldResistFingerprinting, TrackRate aSampleRate,
3335 CubebUtils::AudioDeviceID aOutputDeviceID) {
3336 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3338 TrackRate sampleRate =
3339 aSampleRate
3340 ? aSampleRate
3341 : CubebUtils::PreferredSampleRate(aShouldResistFingerprinting);
3342 GraphKey key(aWindowID, sampleRate, aOutputDeviceID);
3344 return gGraphs.Get(key);
3347 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3348 // window is a real inner Window, not a WindowProxy.
3349 /* static */
3350 MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
3351 nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
3352 CubebUtils::AudioDeviceID aOutputDeviceID) {
3353 return MediaTrackGraphImpl::GetInstanceIfExists(
3354 aWindow->WindowID(),
3355 aWindow->AsGlobal()->ShouldResistFingerprinting(RFPTarget::Unknown),
3356 aSampleRate, aOutputDeviceID);
3359 /* static */
3360 MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
3361 GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
3362 bool aShouldResistFingerprinting, TrackRate aSampleRate,
3363 CubebUtils::AudioDeviceID aOutputDeviceID,
3364 nsISerialEventTarget* aMainThread) {
3365 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3367 TrackRate sampleRate =
3368 aSampleRate
3369 ? aSampleRate
3370 : CubebUtils::PreferredSampleRate(aShouldResistFingerprinting);
3371 MediaTrackGraphImpl* graph = GetInstanceIfExists(
3372 aWindowID, aShouldResistFingerprinting, sampleRate, aOutputDeviceID);
3374 if (!graph) {
3375 GraphRunType runType = DIRECT_DRIVER;
3376 if (aGraphDriverRequested != OFFLINE_THREAD_DRIVER &&
3377 (StaticPrefs::dom_audioworklet_enabled() ||
3378 Preferences::GetBool("media.audiograph.single_thread.enabled",
3379 false))) {
3380 runType = SINGLE_THREAD;
3383 // In a real time graph, the number of output channels is determined by
3384 // the underlying number of channel of the default audio output device, and
3385 // capped to 8.
3386 uint32_t channelCount =
3387 std::min<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
3388 graph = new MediaTrackGraphImpl(aGraphDriverRequested, runType, sampleRate,
3389 channelCount, aOutputDeviceID, aMainThread);
3390 GraphKey key(aWindowID, sampleRate, aOutputDeviceID);
3391 gGraphs.InsertOrUpdate(key, graph);
3393 LOG(LogLevel::Debug,
3394 ("Starting up MediaTrackGraph %p for window 0x%" PRIx64, graph,
3395 aWindowID));
3398 return graph;
3401 /* static */
3402 MediaTrackGraph* MediaTrackGraph::GetInstance(
3403 GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
3404 TrackRate aSampleRate, CubebUtils::AudioDeviceID aOutputDeviceID) {
3405 return MediaTrackGraphImpl::GetInstance(
3406 aGraphDriverRequested, aWindow->WindowID(),
3407 aWindow->AsGlobal()->ShouldResistFingerprinting(RFPTarget::Unknown),
3408 aSampleRate, aOutputDeviceID,
3409 aWindow->EventTargetFor(TaskCategory::Other));
3412 MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
3413 TrackRate aSampleRate, nsPIDOMWindowInner* aWindow) {
3414 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3416 nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
3417 // aWindow can be null when the document is being unlinked, so this works when
3418 // with a generic main thread if that's the case.
3419 if (aWindow) {
3420 mainThread =
3421 aWindow->AsGlobal()->AbstractMainThreadFor(TaskCategory::Other);
3424 // Offline graphs have 0 output channel count: they write the output to a
3425 // buffer, not an audio output track.
3426 MediaTrackGraphImpl* graph =
3427 new MediaTrackGraphImpl(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, aSampleRate,
3428 0, DEFAULT_OUTPUT_DEVICE, mainThread);
3430 LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
3432 return graph;
3435 void MediaTrackGraph::ForceShutDown() {
3436 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3438 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3440 graph->ForceShutDown();
3443 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIThreadObserver,
3444 nsITimerCallback, nsINamed)
3446 NS_IMETHODIMP
3447 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
3448 nsISupports* aData, bool aAnonymize) {
3449 MOZ_ASSERT(NS_IsMainThread());
3450 if (mMainThreadTrackCount == 0) {
3451 // No tracks to report.
3452 FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
3453 return NS_OK;
3456 class Message final : public ControlMessage {
3457 public:
3458 Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
3459 nsISupports* aHandlerData)
3460 : ControlMessage(nullptr),
3461 mGraph(aGraph),
3462 mHandleReport(aHandleReport),
3463 mHandlerData(aHandlerData) {}
3464 void Run() override {
3465 TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3466 mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
3467 mHandlerData.forget());
3469 void RunDuringShutdown() override {
3470 // Run this message during shutdown too, so that endReports is called.
3471 Run();
3473 MediaTrackGraphImpl* mGraph;
3474 // nsMemoryReporterManager keeps the callback and data alive only if it
3475 // does not time out.
3476 nsCOMPtr<nsIHandleReportCallback> mHandleReport;
3477 nsCOMPtr<nsISupports> mHandlerData;
3480 AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
3482 return NS_OK;
3485 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3486 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3487 already_AddRefed<nsISupports> aHandlerData) {
3488 class FinishCollectRunnable final : public Runnable {
3489 public:
3490 explicit FinishCollectRunnable(
3491 already_AddRefed<nsIHandleReportCallback> aHandleReport,
3492 already_AddRefed<nsISupports> aHandlerData)
3493 : mozilla::Runnable("FinishCollectRunnable"),
3494 mHandleReport(aHandleReport),
3495 mHandlerData(aHandlerData) {}
3497 NS_IMETHOD Run() override {
3498 TRACE("MTG::FinishCollectReports ControlMessage");
3499 MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
3500 std::move(mAudioTrackSizes));
3501 return NS_OK;
3504 nsTArray<AudioNodeSizes> mAudioTrackSizes;
3506 private:
3507 ~FinishCollectRunnable() = default;
3509 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3510 // constructor modifies the ref-count, which cannot be done off main
3511 // thread.
3512 RefPtr<nsIHandleReportCallback> mHandleReport;
3513 RefPtr<nsISupports> mHandlerData;
3516 RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
3517 std::move(aHandleReport), std::move(aHandlerData));
3519 auto audioTrackSizes = &runnable->mAudioTrackSizes;
3521 for (MediaTrack* t : AllTracks()) {
3522 AudioNodeTrack* track = t->AsAudioNodeTrack();
3523 if (track) {
3524 AudioNodeSizes* usage = audioTrackSizes->AppendElement();
3525 track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
3529 mMainThread->Dispatch(runnable.forget());
3532 void MediaTrackGraphImpl::FinishCollectReports(
3533 nsIHandleReportCallback* aHandleReport, nsISupports* aData,
3534 const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
3535 MOZ_ASSERT(NS_IsMainThread());
3537 nsCOMPtr<nsIMemoryReporterManager> manager =
3538 do_GetService("@mozilla.org/memory-reporter-manager;1");
3540 if (!manager) return;
3542 #define REPORT(_path, _amount, _desc) \
3543 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3544 nsLiteralCString(_desc), aData);
3546 for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
3547 const AudioNodeSizes& usage = aAudioTrackSizes[i];
3548 const char* const nodeType =
3549 usage.mNodeType ? usage.mNodeType : "<unknown>";
3551 nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3552 nodeType);
3553 REPORT(enginePath, usage.mEngine,
3554 "Memory used by AudioNode engine objects (Web Audio).");
3556 nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
3557 nodeType);
3558 REPORT(trackPath, usage.mTrack,
3559 "Memory used by AudioNode track objects (Web Audio).");
3562 size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
3563 if (hrtfLoaders) {
3564 REPORT(nsLiteralCString(
3565 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3566 hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
3569 #undef REPORT
3571 manager->EndReport();
3574 SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
3575 SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
3576 AddTrack(track);
3577 return track;
3580 ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
3581 MediaSegment::Type aType) {
3582 ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
3583 AddTrack(track);
3584 return track;
3587 AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
3588 AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
3589 AddTrack(track);
3590 return track;
3593 CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
3594 CrossGraphReceiver* aReceiver) {
3595 CrossGraphTransmitter* track =
3596 new CrossGraphTransmitter(GraphRate(), aReceiver);
3597 AddTrack(track);
3598 return track;
3601 CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
3602 TrackRate aTransmitterRate) {
3603 CrossGraphReceiver* track =
3604 new CrossGraphReceiver(GraphRate(), aTransmitterRate);
3605 AddTrack(track);
3606 return track;
3609 void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
3610 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3611 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3612 if (graph->mRealtime) {
3613 bool found = false;
3614 for (const auto& currentGraph : gGraphs.Values()) {
3615 if (currentGraph == graph) {
3616 found = true;
3617 break;
3620 MOZ_DIAGNOSTIC_ASSERT(found, "Graph must not be shutting down");
3622 #endif
3623 NS_ADDREF(aTrack);
3624 aTrack->SetGraphImpl(graph);
3625 ++graph->mMainThreadTrackCount;
3626 graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
3629 void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
3630 MOZ_ASSERT(NS_IsMainThread());
3631 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
3632 if (--mMainThreadTrackCount == 0) {
3633 LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
3634 "main thread. Graph will shut down.",
3635 this, aTrack));
3636 // Find the graph in the hash table and remove it.
3637 for (auto iter = gGraphs.Iter(); !iter.Done(); iter.Next()) {
3638 if (iter.UserData() == this) {
3639 iter.Remove();
3640 break;
3643 // The graph thread will shut itself down soon, but won't be able to do
3644 // that if JS continues to run.
3645 InterruptJS();
3649 auto MediaTrackGraph::NotifyWhenDeviceStarted(MediaTrack* aTrack)
3650 -> RefPtr<GraphStartedPromise> {
3651 MOZ_ASSERT(NS_IsMainThread());
3652 MozPromiseHolder<GraphStartedPromise> h;
3653 RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
3654 aTrack->GraphImpl()->NotifyWhenGraphStarted(aTrack, std::move(h));
3655 return p;
3658 void MediaTrackGraphImpl::NotifyWhenGraphStarted(
3659 RefPtr<MediaTrack> aTrack,
3660 MozPromiseHolder<GraphStartedPromise>&& aHolder) {
3661 class GraphStartedNotificationControlMessage : public ControlMessage {
3662 RefPtr<MediaTrack> mMediaTrack;
3663 MozPromiseHolder<GraphStartedPromise> mHolder;
3665 public:
3666 GraphStartedNotificationControlMessage(
3667 RefPtr<MediaTrack> aTrack,
3668 MozPromiseHolder<GraphStartedPromise>&& aHolder)
3669 : ControlMessage(nullptr),
3670 mMediaTrack(std::move(aTrack)),
3671 mHolder(std::move(aHolder)) {}
3672 void Run() override {
3673 TRACE("MTG::GraphStartedNotificationControlMessage ControlMessage");
3674 // This runs on the graph thread, so when this runs, and the current
3675 // driver is an AudioCallbackDriver, we know the audio hardware is
3676 // started. If not, we are going to switch soon, keep reposting this
3677 // ControlMessage.
3678 MediaTrackGraphImpl* graphImpl = mMediaTrack->GraphImpl();
3679 if (graphImpl->CurrentDriver()->AsAudioCallbackDriver() &&
3680 graphImpl->CurrentDriver()->ThreadRunning() &&
3681 !graphImpl->CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3682 // Avoid Resolve's locking on the graph thread by doing it on main.
3683 graphImpl->Dispatch(NS_NewRunnableFunction(
3684 "MediaTrackGraphImpl::NotifyWhenGraphStarted::Resolver",
3685 [holder = std::move(mHolder)]() mutable {
3686 holder.Resolve(true, __func__);
3687 }));
3688 } else {
3689 graphImpl->DispatchToMainThreadStableState(
3690 NewRunnableMethod<
3691 StoreCopyPassByRRef<RefPtr<MediaTrack>>,
3692 StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
3693 "MediaTrackGraphImpl::NotifyWhenGraphStarted", graphImpl,
3694 &MediaTrackGraphImpl::NotifyWhenGraphStarted,
3695 std::move(mMediaTrack), std::move(mHolder)));
3698 void RunDuringShutdown() override {
3699 mHolder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
3703 if (aTrack->IsDestroyed()) {
3704 aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
3705 return;
3708 MediaTrackGraphImpl* graph = aTrack->GraphImpl();
3709 graph->AppendMessage(MakeUnique<GraphStartedNotificationControlMessage>(
3710 std::move(aTrack), std::move(aHolder)));
3713 class AudioContextOperationControlMessage : public ControlMessage {
3714 using AudioContextOperationPromise =
3715 MediaTrackGraph::AudioContextOperationPromise;
3717 public:
3718 AudioContextOperationControlMessage(
3719 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3720 AudioContextOperation aOperation,
3721 MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
3722 : ControlMessage(aDestinationTrack),
3723 mTracks(std::move(aTracks)),
3724 mAudioContextOperation(aOperation),
3725 mHolder(std::move(aHolder)) {}
3726 void Run() override {
3727 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3728 kAudioContextOptionsStrings[static_cast<uint8_t>(
3729 mAudioContextOperation)]);
3730 mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
3732 void RunDuringShutdown() override {
3733 MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
3734 "We should be reviving the graph?");
3735 mHolder.Reject(false, __func__);
3738 nsTArray<RefPtr<MediaTrack>> mTracks;
3739 AudioContextOperation mAudioContextOperation;
3740 MozPromiseHolder<AudioContextOperationPromise> mHolder;
3743 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3744 AudioContextOperationControlMessage* aMessage) {
3745 MOZ_ASSERT(OnGraphThread());
3746 // Initialize state to zero. This silences a GCC warning about uninitialized
3747 // values, because although the switch below initializes state for all valid
3748 // enum values, the actual value could be any integer that fits in the enum.
3749 AudioContextState state{0};
3750 switch (aMessage->mAudioContextOperation) {
3751 // Suspend and Close operations may be performed immediately because no
3752 // specific kind of GraphDriver is required. CheckDriver() will schedule
3753 // a change to a SystemCallbackDriver if all tracks are suspended.
3754 case AudioContextOperation::Suspend:
3755 state = AudioContextState::Suspended;
3756 break;
3757 case AudioContextOperation::Close:
3758 state = AudioContextState::Closed;
3759 break;
3760 case AudioContextOperation::Resume:
3761 // Resume operations require an AudioCallbackDriver. CheckDriver() will
3762 // schedule an AudioCallbackDriver if necessary and process pending
3763 // operations if and when an AudioCallbackDriver is running.
3764 mPendingResumeOperations.EmplaceBack(aMessage);
3765 return;
3767 // First resolve any pending Resume promises for the same AudioContext so as
3768 // to resolve its associated promises in the same order as they were
3769 // created. These Resume operations are considered complete and immediately
3770 // canceled by the Suspend or Close.
3771 MediaTrack* destinationTrack = aMessage->GetTrack();
3772 bool shrinking = false;
3773 auto moveDest = mPendingResumeOperations.begin();
3774 for (PendingResumeOperation& op : mPendingResumeOperations) {
3775 if (op.DestinationTrack() == destinationTrack) {
3776 op.Apply(this);
3777 shrinking = true;
3778 continue;
3780 if (shrinking) { // Fill-in gaps in the array.
3781 *moveDest = std::move(op);
3783 ++moveDest;
3785 mPendingResumeOperations.TruncateLength(moveDest -
3786 mPendingResumeOperations.begin());
3788 for (MediaTrack* track : aMessage->mTracks) {
3789 track->IncrementSuspendCount();
3791 // Resolve after main thread state is up to date with completed processing.
3792 DispatchToMainThreadStableState(NS_NewRunnableFunction(
3793 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
3794 [holder = std::move(aMessage->mHolder), state]() mutable {
3795 holder.Resolve(state, __func__);
3796 }));
3799 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
3800 AudioContextOperationControlMessage* aMessage)
3801 : mDestinationTrack(aMessage->GetTrack()),
3802 mTracks(std::move(aMessage->mTracks)),
3803 mHolder(std::move(aMessage->mHolder)) {
3804 MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
3807 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
3808 MediaTrackGraphImpl* aGraph) {
3809 MOZ_ASSERT(aGraph->OnGraphThread());
3810 for (MediaTrack* track : mTracks) {
3811 track->DecrementSuspendCount();
3813 // The graph is provided through the parameter so that it is available even
3814 // when the track is destroyed.
3815 aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
3816 "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
3817 holder.Resolve(AudioContextState::Running, __func__);
3818 }));
3821 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
3822 // The graph is shutting down before the operation completed.
3823 MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
3824 mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
3825 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
3826 mHolder.Reject(false, __func__);
3829 auto MediaTrackGraph::ApplyAudioContextOperation(
3830 MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
3831 AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
3832 MozPromiseHolder<AudioContextOperationPromise> holder;
3833 RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
3834 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3835 graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
3836 aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
3837 return p;
3840 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount() const {
3841 MOZ_ASSERT(OnGraphThread());
3842 // The audio output channel count for a graph is the maximum of the output
3843 // channel count of all the tracks that are in mAudioOutputs, or the max audio
3844 // output channel count the machine can do, whichever is smaller.
3845 uint32_t channelCount = 0;
3846 for (auto& tkv : mAudioOutputs) {
3847 channelCount = std::max(channelCount, tkv.mTrack->NumberOfChannels());
3849 channelCount = std::min(channelCount, mMaxOutputChannelCount);
3850 if (channelCount) {
3851 return channelCount;
3852 } else {
3853 if (CurrentDriver()->AsAudioCallbackDriver()) {
3854 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
3856 return 2;
3860 double MediaTrackGraph::AudioOutputLatency() {
3861 return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
3864 double MediaTrackGraphImpl::AudioOutputLatency() {
3865 MOZ_ASSERT(NS_IsMainThread());
3866 if (mAudioOutputLatency != 0.0) {
3867 return mAudioOutputLatency;
3869 MonitorAutoLock lock(mMonitor);
3870 if (CurrentDriver()->AsAudioCallbackDriver()) {
3871 mAudioOutputLatency = CurrentDriver()
3872 ->AsAudioCallbackDriver()
3873 ->AudioOutputLatency()
3874 .ToSeconds();
3875 } else {
3876 // Failure mode: return 0.0 if running on a normal thread.
3877 mAudioOutputLatency = 0.0;
3880 return mAudioOutputLatency;
3883 bool MediaTrackGraph::IsNonRealtime() const {
3884 return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
3887 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
3888 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
3890 MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
3891 NS_ASSERTION(!graph->mRealtime, "non-realtime only");
3893 class Message : public ControlMessage {
3894 public:
3895 explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
3896 : ControlMessage(nullptr),
3897 mGraph(aGraph),
3898 mTicksToProcess(aTicksToProcess) {}
3899 void Run() override {
3900 TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
3901 MOZ_ASSERT(mGraph->mEndTime == 0,
3902 "StartNonRealtimeProcessing should be called only once");
3903 mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
3904 mGraph->mStateComputedTime + mTicksToProcess);
3906 // The graph owns this message.
3907 MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
3908 uint32_t mTicksToProcess;
3911 graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
3914 void MediaTrackGraphImpl::InterruptJS() {
3915 MonitorAutoLock lock(mMonitor);
3916 mInterruptJSCalled = true;
3917 if (mJSContext) {
3918 JS_RequestInterruptCallback(mJSContext);
3922 static bool InterruptCallback(JSContext* aCx) {
3923 // Interrupt future calls also.
3924 JS_RequestInterruptCallback(aCx);
3925 // Stop execution.
3926 return false;
3929 void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
3930 MOZ_ASSERT(OnGraphThread());
3931 MOZ_ASSERT(aCx);
3933 auto* impl = static_cast<MediaTrackGraphImpl*>(this);
3934 MonitorAutoLock lock(impl->mMonitor);
3935 if (impl->mJSContext) {
3936 MOZ_ASSERT(impl->mJSContext == aCx);
3937 return;
3939 JS_AddInterruptCallback(aCx, InterruptCallback);
3940 impl->mJSContext = aCx;
3941 if (impl->mInterruptJSCalled) {
3942 JS_RequestInterruptCallback(aCx);
3946 void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
3947 MediaTrack* t = aPort->GetSource();
3948 if (!t->IsSuspended()) {
3949 mInputs.AppendElement(aPort);
3950 } else {
3951 mSuspendedInputs.AppendElement(aPort);
3953 GraphImpl()->SetTrackOrderDirty();
3956 void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
3957 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3958 mInputs.RemoveElement(aPort);
3959 mSuspendedInputs.AppendElement(aPort);
3960 GraphImpl()->SetTrackOrderDirty();
3963 void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
3964 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3965 mSuspendedInputs.RemoveElement(aPort);
3966 mInputs.AppendElement(aPort);
3967 GraphImpl()->SetTrackOrderDirty();
3970 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
3971 MOZ_ASSERT(OnGraphThread());
3972 LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
3973 if (GraphDriver* nextDriver = NextDriver()) {
3974 if (nextDriver != CurrentDriver()) {
3975 LOG(LogLevel::Debug,
3976 ("%p: Discarding previous next driver: %p", this, nextDriver));
3979 mNextDriver = aNextDriver;
3982 void MediaTrackGraph::RegisterCaptureTrackForWindow(
3983 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
3984 MOZ_ASSERT(NS_IsMainThread());
3985 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
3986 graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
3989 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
3990 uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
3991 MOZ_ASSERT(NS_IsMainThread());
3992 WindowAndTrack winAndTrack;
3993 winAndTrack.mWindowId = aWindowId;
3994 winAndTrack.mCaptureTrackSink = aCaptureTrack;
3995 mWindowCaptureTracks.AppendElement(winAndTrack);
3998 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
3999 MOZ_ASSERT(NS_IsMainThread());
4000 MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
4001 graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
4004 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
4005 MOZ_ASSERT(NS_IsMainThread());
4006 mWindowCaptureTracks.RemoveElementsBy(
4007 [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
4010 already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
4011 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4012 return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
4013 aMediaTrack);
4016 already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
4017 uint64_t aWindowId, MediaTrack* aMediaTrack) {
4018 MOZ_ASSERT(NS_IsMainThread());
4019 for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
4020 if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
4021 ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
4022 return sink->AllocateInputPort(aMediaTrack);
4025 return nullptr;
4028 void MediaTrackGraph::DispatchToMainThreadStableState(
4029 already_AddRefed<nsIRunnable> aRunnable) {
4030 AssertOnGraphThreadOrNotRunning();
4031 static_cast<MediaTrackGraphImpl*>(this)
4032 ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
4035 Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
4036 MOZ_ASSERT(NS_IsMainThread());
4037 return mMainThreadGraphTime;
4040 GraphTime MediaTrackGraph::ProcessedTime() const {
4041 AssertOnGraphThreadOrNotRunning();
4042 return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
4045 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4046 CubebUtils::AudioDeviceID aID) {
4047 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4048 DeviceInputTrack* t =
4049 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4050 return t ? t->MaxRequestedInputChannels() : 0;
4053 AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
4054 CubebUtils::AudioDeviceID aID) {
4055 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4056 DeviceInputTrack* t =
4057 mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
4058 return t && t->HasVoiceInput() ? AudioInputType::Voice
4059 : AudioInputType::Unknown;
4062 void MediaTrackGraphImpl::SetNewNativeInput() {
4063 MOZ_ASSERT(NS_IsMainThread());
4064 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4066 LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
4068 NonNativeInputTrack* track =
4069 mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
4070 if (!track) {
4071 LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
4072 return;
4075 const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
4076 const PrincipalHandle principal = track->mPrincipalHandle;
4078 LOG(LogLevel::Debug,
4079 ("%p Select device %p as the new native input device", this, deviceId));
4081 struct TrackListener {
4082 DeviceInputConsumerTrack* track;
4083 // Keep its reference so it won't be dropped when after
4084 // DisconnectDeviceInput().
4085 RefPtr<AudioDataListener> listener;
4087 nsTArray<TrackListener> pairs;
4089 for (const auto& t : track->GetConsumerTracks()) {
4090 pairs.AppendElement(
4091 TrackListener{t.get(), t->GetAudioDataListener().get()});
4094 for (TrackListener& pair : pairs) {
4095 pair.track->DisconnectDeviceInput();
4098 for (TrackListener& pair : pairs) {
4099 pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
4100 LOG(LogLevel::Debug,
4101 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4102 pair.track, deviceId));
4105 LOG(LogLevel::Debug,
4106 ("%p Native input device is set to device %p now", this, deviceId));
4108 MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
4111 // nsIThreadObserver methods
4113 NS_IMETHODIMP
4114 MediaTrackGraphImpl::OnDispatchedEvent() {
4115 MonitorAutoLock lock(mMonitor);
4116 EnsureNextIteration();
4117 return NS_OK;
4120 NS_IMETHODIMP
4121 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
4122 return NS_OK;
4125 NS_IMETHODIMP
4126 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
4127 return NS_OK;
4129 } // namespace mozilla