1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "MediaTrackGraphImpl.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "CrossGraphPort.h"
12 #include "VideoSegment.h"
13 #include "nsContentUtils.h"
14 #include "nsGlobalWindowInner.h"
15 #include "nsPrintfCString.h"
16 #include "nsServiceManagerUtils.h"
18 #include "mozilla/Logging.h"
19 #include "mozilla/Attributes.h"
20 #include "ForwardedInputTrack.h"
21 #include "ImageContainer.h"
22 #include "AudioCaptureTrack.h"
23 #include "AudioDeviceInfo.h"
24 #include "AudioNodeTrack.h"
25 #include "AudioNodeExternalInputTrack.h"
26 #if defined(MOZ_WEBRTC)
27 # include "MediaEngineWebRTCAudio.h"
29 #include "MediaTrackListener.h"
30 #include "mozilla/dom/BaseAudioContextBinding.h"
31 #include "mozilla/dom/Document.h"
32 #include "mozilla/dom/WorkletThread.h"
33 #include "mozilla/media/MediaUtils.h"
35 #include "GeckoProfiler.h"
36 #include "VideoFrameContainer.h"
37 #include "mozilla/AbstractThread.h"
38 #include "mozilla/StaticPrefs_dom.h"
39 #include "mozilla/StaticPrefs_media.h"
40 #include "transport/runnable_utils.h"
41 #include "VideoUtils.h"
42 #include "GraphRunner.h"
44 #include "UnderrunHandler.h"
45 #include "mozilla/CycleCollectedJSRuntime.h"
46 #include "mozilla/Preferences.h"
48 #include "webaudio/blink/DenormalDisabler.h"
49 #include "webaudio/blink/HRTFDatabaseLoader.h"
51 using namespace mozilla::layers
;
52 using namespace mozilla::dom
;
53 using namespace mozilla::gfx
;
54 using namespace mozilla::media
;
58 using AudioDeviceID
= CubebUtils::AudioDeviceID
;
59 using IsInShutdown
= MediaTrack::IsInShutdown
;
61 LazyLogModule
gMediaTrackGraphLog("MediaTrackGraph");
65 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
67 NativeInputTrack
* DeviceInputTrackManager::GetNativeInputTrack() {
68 return mNativeInputTrack
.get();
71 DeviceInputTrack
* DeviceInputTrackManager::GetDeviceInputTrack(
72 CubebUtils::AudioDeviceID aID
) {
73 if (mNativeInputTrack
&& mNativeInputTrack
->mDeviceId
== aID
) {
74 return mNativeInputTrack
.get();
76 for (const RefPtr
<NonNativeInputTrack
>& t
: mNonNativeInputTracks
) {
77 if (t
->mDeviceId
== aID
) {
84 NonNativeInputTrack
* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
85 if (mNonNativeInputTracks
.IsEmpty()) {
88 return mNonNativeInputTracks
[0].get();
91 void DeviceInputTrackManager::Add(DeviceInputTrack
* aTrack
) {
92 if (NativeInputTrack
* native
= aTrack
->AsNativeInputTrack()) {
93 MOZ_ASSERT(!mNativeInputTrack
);
94 mNativeInputTrack
= native
;
96 NonNativeInputTrack
* nonNative
= aTrack
->AsNonNativeInputTrack();
97 MOZ_ASSERT(nonNative
);
98 struct DeviceTrackComparator
{
100 bool Equals(const RefPtr
<NonNativeInputTrack
>& aTrack
,
101 CubebUtils::AudioDeviceID aDeviceId
) const {
102 return aTrack
->mDeviceId
== aDeviceId
;
105 MOZ_ASSERT(!mNonNativeInputTracks
.Contains(aTrack
->mDeviceId
,
106 DeviceTrackComparator()));
107 mNonNativeInputTracks
.AppendElement(nonNative
);
111 void DeviceInputTrackManager::Remove(DeviceInputTrack
* aTrack
) {
112 if (aTrack
->AsNativeInputTrack()) {
113 MOZ_ASSERT(mNativeInputTrack
);
114 MOZ_ASSERT(mNativeInputTrack
.get() == aTrack
->AsNativeInputTrack());
115 mNativeInputTrack
= nullptr;
117 NonNativeInputTrack
* nonNative
= aTrack
->AsNonNativeInputTrack();
118 MOZ_ASSERT(nonNative
);
119 DebugOnly
<bool> removed
= mNonNativeInputTracks
.RemoveElement(nonNative
);
125 * A hash table containing the graph instances, one per Window ID,
126 * sample rate, and device ID combination.
129 struct MediaTrackGraphImpl::Lookup final
{
130 HashNumber
Hash() const {
131 return HashGeneric(mWindowID
, mSampleRate
, mOutputDeviceID
);
133 const uint64_t mWindowID
;
134 const TrackRate mSampleRate
;
135 const CubebUtils::AudioDeviceID mOutputDeviceID
;
138 // Implicit to support GraphHashSet.lookup(*graph).
139 MOZ_IMPLICIT
MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const {
140 return {mWindowID
, mSampleRate
, PrimaryOutputDeviceID()};
144 struct GraphHasher
{ // for HashSet
145 using Lookup
= const MediaTrackGraphImpl::Lookup
;
147 static HashNumber
hash(const Lookup
& aLookup
) { return aLookup
.Hash(); }
149 static bool match(const MediaTrackGraphImpl
* aGraph
, const Lookup
& aLookup
) {
150 return aGraph
->mWindowID
== aLookup
.mWindowID
&&
151 aGraph
->GraphRate() == aLookup
.mSampleRate
&&
152 aGraph
->PrimaryOutputDeviceID() == aLookup
.mOutputDeviceID
;
156 // The weak reference to the graph is removed when its last track is removed.
158 HashSet
<MediaTrackGraphImpl
*, GraphHasher
, InfallibleAllocPolicy
>;
159 GraphHashSet
* Graphs() {
160 MOZ_ASSERT(NS_IsMainThread());
161 static GraphHashSet
sGraphs(4); // 4 is minimum HashSet capacity
164 } // anonymous namespace
166 static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode
,
167 MediaSegment
* aSegment
,
168 MediaSegment
* aRawSegment
) {
169 if (aDisabledMode
== DisabledTrackMode::ENABLED
) {
172 if (aDisabledMode
== DisabledTrackMode::SILENCE_BLACK
) {
173 aSegment
->ReplaceWithDisabled();
175 aRawSegment
->ReplaceWithDisabled();
177 } else if (aDisabledMode
== DisabledTrackMode::SILENCE_FREEZE
) {
178 aSegment
->ReplaceWithNull();
180 aRawSegment
->ReplaceWithNull();
183 MOZ_CRASH("Unsupported mode");
187 MediaTrackGraphImpl::~MediaTrackGraphImpl() {
188 MOZ_ASSERT(mTracks
.IsEmpty() && mSuspendedTracks
.IsEmpty(),
189 "All tracks should have been destroyed by messages from the main "
191 LOG(LogLevel::Debug
, ("MediaTrackGraph %p destroyed", this));
192 LOG(LogLevel::Debug
, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
195 void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack
* aTrack
) {
196 MOZ_ASSERT(OnGraphThreadOrNotRunning());
197 aTrack
->mStartTime
= mProcessedTime
;
199 if (aTrack
->IsSuspended()) {
200 mSuspendedTracks
.AppendElement(aTrack
);
202 ("%p: Adding media track %p, in the suspended track array", this,
205 mTracks
.AppendElement(aTrack
);
206 LOG(LogLevel::Debug
, ("%p: Adding media track %p, count %zu", this, aTrack
,
210 SetTrackOrderDirty();
213 void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack
* aTrack
) {
214 MOZ_ASSERT(OnGraphThreadOrNotRunning());
215 // Remove references in mTrackUpdates before we allow aTrack to die.
216 // Pending updates are not needed (since the main thread has already given
217 // up the track) so we will just drop them.
219 MonitorAutoLock
lock(mMonitor
);
220 for (uint32_t i
= 0; i
< mTrackUpdates
.Length(); ++i
) {
221 if (mTrackUpdates
[i
].mTrack
== aTrack
) {
222 mTrackUpdates
[i
].mTrack
= nullptr;
227 // Ensure that mFirstCycleBreaker is updated when necessary.
228 SetTrackOrderDirty();
230 UnregisterAllAudioOutputs(aTrack
);
232 if (aTrack
->IsSuspended()) {
233 mSuspendedTracks
.RemoveElement(aTrack
);
235 mTracks
.RemoveElement(aTrack
);
238 LOG(LogLevel::Debug
, ("%p: Removed media track %p, count %zu", this, aTrack
,
241 NS_RELEASE(aTrack
); // probably destroying it
244 TrackTime
MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
245 const MediaTrack
* aTrack
, GraphTime aTime
) const {
247 aTime
<= mStateComputedTime
,
248 "Don't ask about times where we haven't made blocking decisions yet");
249 return std::max
<TrackTime
>(
250 0, std::min(aTime
, aTrack
->mStartBlocking
) - aTrack
->mStartTime
);
253 GraphTime
MediaTrackGraphImpl::IterationEnd() const {
254 MOZ_ASSERT(OnGraphThread());
255 return mIterationEndTime
;
258 void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
259 GraphTime aPrevCurrentTime
) {
260 MOZ_ASSERT(OnGraphThread());
261 for (MediaTrack
* track
: AllTracks()) {
262 // Shouldn't have already notified of ended *and* have output!
263 MOZ_ASSERT_IF(track
->mStartBlocking
> aPrevCurrentTime
,
264 !track
->mNotifiedEnded
);
266 // Calculate blocked time and fire Blocked/Unblocked events
267 GraphTime blockedTime
= mStateComputedTime
- track
->mStartBlocking
;
268 NS_ASSERTION(blockedTime
>= 0, "Error in blocking time");
269 track
->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime
,
271 LOG(LogLevel::Verbose
,
272 ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track
,
273 MediaTimeToSeconds(track
->mStartTime
),
274 MediaTimeToSeconds(blockedTime
)));
275 track
->mStartBlocking
= mStateComputedTime
;
277 TrackTime trackCurrentTime
=
278 track
->GraphTimeToTrackTime(mStateComputedTime
);
280 MOZ_ASSERT(track
->GetEnd() <= trackCurrentTime
);
281 if (!track
->mNotifiedEnded
) {
282 // Playout of this track ended and listeners have not been notified.
283 track
->mNotifiedEnded
= true;
284 SetTrackOrderDirty();
285 for (const auto& listener
: track
->mTrackListeners
) {
286 listener
->NotifyOutput(this, track
->GetEnd());
287 listener
->NotifyEnded(this);
291 for (const auto& listener
: track
->mTrackListeners
) {
292 listener
->NotifyOutput(this, trackCurrentTime
);
298 template <typename C
, typename Chunk
>
299 void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack
* aTrack
,
303 MOZ_ASSERT(OnGraphThreadOrNotRunning());
306 TrackTime offset
= 0;
307 for (typename
C::ConstChunkIterator
chunk(aSegment
); !chunk
.IsEnded();
309 if (offset
>= aEnd
) {
312 offset
+= chunk
->GetDuration();
313 if (chunk
->IsNull() || offset
< aStart
) {
316 const PrincipalHandle
& principalHandle
= chunk
->GetPrincipalHandle();
317 if (principalHandle
!= aSegment
.GetLastPrincipalHandle()) {
318 aSegment
.SetLastPrincipalHandle(principalHandle
);
320 ("%p: MediaTrack %p, principalHandle "
321 "changed in %sChunk with duration %lld",
323 aSegment
.GetType() == MediaSegment::AUDIO
? "Audio" : "Video",
324 (long long)chunk
->GetDuration()));
325 for (const auto& listener
: aTrack
->mTrackListeners
) {
326 listener
->NotifyPrincipalHandleChanged(this, principalHandle
);
332 void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime
) {
333 MOZ_ASSERT(OnGraphThreadOrNotRunning());
334 for (MediaTrack
* track
: AllTracks()) {
335 TrackTime iterationStart
= track
->GraphTimeToTrackTime(aPrevCurrentTime
);
336 TrackTime iterationEnd
= track
->GraphTimeToTrackTime(mProcessedTime
);
337 if (!track
->mSegment
) {
340 if (track
->mType
== MediaSegment::AUDIO
) {
341 ProcessChunkMetadataForInterval
<AudioSegment
, AudioChunk
>(
342 track
, *track
->GetData
<AudioSegment
>(), iterationStart
, iterationEnd
);
343 } else if (track
->mType
== MediaSegment::VIDEO
) {
344 ProcessChunkMetadataForInterval
<VideoSegment
, VideoChunk
>(
345 track
, *track
->GetData
<VideoSegment
>(), iterationStart
, iterationEnd
);
347 MOZ_CRASH("Unknown track type");
352 GraphTime
MediaTrackGraphImpl::WillUnderrun(MediaTrack
* aTrack
,
353 GraphTime aEndBlockingDecisions
) {
354 // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
355 // underrun currently, since we'll always be able to produce data for them
356 // unless they block on some other track.
357 if (aTrack
->mEnded
|| aTrack
->AsProcessedTrack()) {
358 return aEndBlockingDecisions
;
360 // This track isn't ended or suspended. We don't need to call
361 // TrackTimeToGraphTime since an underrun is the only thing that can block
363 GraphTime bufferEnd
= aTrack
->GetEnd() + aTrack
->mStartTime
;
365 if (bufferEnd
< mProcessedTime
) {
366 LOG(LogLevel::Error
, ("%p: MediaTrack %p underrun, "
367 "bufferEnd %f < mProcessedTime %f (%" PRId64
368 " < %" PRId64
"), TrackTime %" PRId64
,
369 this, aTrack
, MediaTimeToSeconds(bufferEnd
),
370 MediaTimeToSeconds(mProcessedTime
), bufferEnd
,
371 mProcessedTime
, aTrack
->GetEnd()));
372 NS_ASSERTION(bufferEnd
>= mProcessedTime
, "Buffer underran");
375 return std::min(bufferEnd
, aEndBlockingDecisions
);
379 // Value of mCycleMarker for unvisited tracks in cycle detection.
380 const uint32_t NOT_VISITED
= UINT32_MAX
;
381 // Value of mCycleMarker for ordered tracks in muted cycles.
382 const uint32_t IN_MUTED_CYCLE
= 1;
385 bool MediaTrackGraphImpl::AudioTrackPresent() {
386 MOZ_ASSERT(OnGraphThreadOrNotRunning());
388 bool audioTrackPresent
= false;
389 for (MediaTrack
* track
: mTracks
) {
390 if (track
->AsAudioNodeTrack()) {
391 audioTrackPresent
= true;
395 if (track
->mType
== MediaSegment::AUDIO
&& !track
->mNotifiedEnded
) {
396 audioTrackPresent
= true;
401 // We may not have audio input device when we only have AudioNodeTracks. But
402 // if audioTrackPresent is false, we must have no input device.
403 MOZ_DIAGNOSTIC_ASSERT_IF(
405 !mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack());
407 return audioTrackPresent
;
410 void MediaTrackGraphImpl::CheckDriver() {
411 MOZ_ASSERT(OnGraphThread());
412 // An offline graph has only one driver.
413 // Otherwise, if a switch is already pending, let that happen.
414 if (!mRealtime
|| Switching()) {
418 AudioCallbackDriver
* audioCallbackDriver
=
419 CurrentDriver()->AsAudioCallbackDriver();
420 if (audioCallbackDriver
&& !audioCallbackDriver
->OnFallback()) {
421 for (PendingResumeOperation
& op
: mPendingResumeOperations
) {
424 mPendingResumeOperations
.Clear();
427 // Note that this looks for any audio tracks, input or output, and switches
428 // to a SystemClockDriver if there are none active or no resume operations
429 // to make any active.
430 bool needAudioCallbackDriver
=
431 !mPendingResumeOperations
.IsEmpty() || AudioTrackPresent();
432 if (!needAudioCallbackDriver
) {
433 if (audioCallbackDriver
&& audioCallbackDriver
->IsStarted()) {
434 SwitchAtNextIteration(
435 new SystemClockDriver(this, CurrentDriver(), mSampleRate
));
440 NativeInputTrack
* native
=
441 mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack();
442 CubebUtils::AudioDeviceID inputDevice
= native
? native
->mDeviceId
: nullptr;
443 uint32_t inputChannelCount
=
444 native
? AudioInputChannelCount(native
->mDeviceId
) : 0;
445 AudioInputType inputPreference
=
446 native
? AudioInputDevicePreference(native
->mDeviceId
)
447 : AudioInputType::Unknown
;
449 uint32_t primaryOutputChannelCount
= PrimaryOutputChannelCount();
450 if (!audioCallbackDriver
) {
451 if (primaryOutputChannelCount
> 0) {
452 AudioCallbackDriver
* driver
= new AudioCallbackDriver(
453 this, CurrentDriver(), mSampleRate
, primaryOutputChannelCount
,
454 inputChannelCount
, PrimaryOutputDeviceID(), inputDevice
,
456 SwitchAtNextIteration(driver
);
461 // Check if this graph should switch to a different number of output channels.
462 // Generally, a driver switch is explicitly made by an event (e.g., setting
463 // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
464 // directly playing back via another HTMLMediaElement, the number of channels
465 // of the media determines how many channels to output, and it can change
467 if (primaryOutputChannelCount
!= audioCallbackDriver
->OutputChannelCount()) {
468 AudioCallbackDriver
* driver
= new AudioCallbackDriver(
469 this, CurrentDriver(), mSampleRate
, primaryOutputChannelCount
,
470 inputChannelCount
, PrimaryOutputDeviceID(), inputDevice
,
472 SwitchAtNextIteration(driver
);
476 void MediaTrackGraphImpl::UpdateTrackOrder() {
477 if (!mTrackOrderDirty
) {
481 mTrackOrderDirty
= false;
483 // The algorithm for finding cycles is based on Tim Leslie's iterative
484 // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
485 // connected components (SCC) algorithm. There are variations (a) to
486 // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
487 // re-run the algorithm over SCCs with breaks at DelayNodes.
489 // [1] http://www.timl.id.au/?p=327
491 // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
492 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
494 // There are two stacks. One for the depth-first search (DFS),
495 mozilla::LinkedList
<MediaTrack
> dfsStack
;
496 // and another for tracks popped from the DFS stack, but still being
497 // considered as part of SCCs involving tracks on the stack.
498 mozilla::LinkedList
<MediaTrack
> sccStack
;
500 // An index into mTracks for the next track found with no unsatisfied
501 // upstream dependencies.
502 uint32_t orderedTrackCount
= 0;
504 for (uint32_t i
= 0; i
< mTracks
.Length(); ++i
) {
505 MediaTrack
* t
= mTracks
[i
];
506 ProcessedMediaTrack
* pt
= t
->AsProcessedTrack();
508 // The dfsStack initially contains a list of all processed tracks in
510 dfsStack
.insertBack(t
);
511 pt
->mCycleMarker
= NOT_VISITED
;
513 // SourceMediaTracks have no inputs and so can be ordered now.
514 mTracks
[orderedTrackCount
] = t
;
519 // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
520 // counter to label mCycleMarker on the next visited track in the DFS
521 // uniquely in the set of visited tracks that are still being considered.
523 // In this implementation, the counter descends so that the values are
524 // strictly greater than the values that mCycleMarker takes when the track
525 // has been ordered (0 or IN_MUTED_CYCLE).
527 // Each new track labelled, as the DFS searches upstream, receives a value
528 // less than those used for all other tracks being considered.
529 uint32_t nextStackMarker
= NOT_VISITED
- 1;
530 // Reset list of DelayNodes in cycles stored at the tail of mTracks.
531 mFirstCycleBreaker
= mTracks
.Length();
533 // Rearrange dfsStack order as required to DFS upstream and pop tracks
534 // in processing order to place in mTracks.
535 while (auto pt
= static_cast<ProcessedMediaTrack
*>(dfsStack
.getFirst())) {
536 const auto& inputs
= pt
->mInputs
;
537 MOZ_ASSERT(pt
->AsProcessedTrack());
538 if (pt
->mCycleMarker
== NOT_VISITED
) {
539 // Record the position on the visited stack, so that any searches
540 // finding this track again know how much of the stack is in the cycle.
541 pt
->mCycleMarker
= nextStackMarker
;
543 // Not-visited input tracks should be processed first.
544 // SourceMediaTracks have already been ordered.
545 for (uint32_t i
= inputs
.Length(); i
--;) {
546 if (inputs
[i
]->GetSource()->IsSuspended()) {
549 auto input
= inputs
[i
]->GetSource()->AsProcessedTrack();
550 if (input
&& input
->mCycleMarker
== NOT_VISITED
) {
551 // It can be that this track has an input which is from a suspended
553 if (input
->isInList()) {
555 dfsStack
.insertFront(input
);
562 // Returning from DFS. Pop from dfsStack.
565 // cycleStackMarker keeps track of the highest marker value on any
566 // upstream track, if any, found receiving input, directly or indirectly,
567 // from the visited stack (and so from |ps|, making a cycle). In a
568 // variation from Tarjan's SCC algorithm, this does not include |ps|
569 // unless it is part of the cycle.
570 uint32_t cycleStackMarker
= 0;
571 for (uint32_t i
= inputs
.Length(); i
--;) {
572 if (inputs
[i
]->GetSource()->IsSuspended()) {
575 auto input
= inputs
[i
]->GetSource()->AsProcessedTrack();
577 cycleStackMarker
= std::max(cycleStackMarker
, input
->mCycleMarker
);
581 if (cycleStackMarker
<= IN_MUTED_CYCLE
) {
582 // All inputs have been ordered and their stack markers have been removed.
583 // This track is not part of a cycle. It can be processed next.
584 pt
->mCycleMarker
= 0;
585 mTracks
[orderedTrackCount
] = pt
;
590 // A cycle has been found. Record this track for ordering when all
591 // tracks in this SCC have been popped from the DFS stack.
592 sccStack
.insertFront(pt
);
594 if (cycleStackMarker
> pt
->mCycleMarker
) {
595 // Cycles have been found that involve tracks that remain on the stack.
596 // Leave mCycleMarker indicating the most downstream (last) track on
597 // the stack known to be part of this SCC. In this way, any searches on
598 // other paths that find |ps| will know (without having to traverse from
599 // this track again) that they are part of this SCC (i.e. part of an
600 // intersecting cycle).
601 pt
->mCycleMarker
= cycleStackMarker
;
605 // |pit| is the root of an SCC involving no other tracks on dfsStack, the
606 // complete SCC has been recorded, and tracks in this SCC are part of at
608 MOZ_ASSERT(cycleStackMarker
== pt
->mCycleMarker
);
609 // If there are DelayNodes in this SCC, then they may break the cycles.
610 bool haveDelayNode
= false;
611 auto next
= sccStack
.getFirst();
612 // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
613 // (There may be other tracks later in sccStack from other incompletely
614 // searched SCCs, involving tracks still on dfsStack.)
616 // DelayNodes in cycles must behave differently from those not in cycles,
617 // so all DelayNodes in the SCC must be identified.
618 while (next
&& static_cast<ProcessedMediaTrack
*>(next
)->mCycleMarker
<=
620 auto nt
= next
->AsAudioNodeTrack();
621 // Get next before perhaps removing from list below.
622 next
= next
->getNext();
623 if (nt
&& nt
->Engine()->AsDelayNodeEngine()) {
624 haveDelayNode
= true;
625 // DelayNodes break cycles by producing their output in a
626 // preprocessing phase; they do not need to be ordered before their
627 // consumers. Order them at the tail of mTracks so that they can be
628 // handled specially. Do so now, so that DFS ignores them.
630 nt
->mCycleMarker
= 0;
631 --mFirstCycleBreaker
;
632 mTracks
[mFirstCycleBreaker
] = nt
;
635 auto after_scc
= next
;
636 while ((next
= sccStack
.getFirst()) != after_scc
) {
638 auto removed
= static_cast<ProcessedMediaTrack
*>(next
);
640 // Return tracks to the DFS stack again (to order and detect cycles
641 // without delayNodes). Any of these tracks that are still inputs
642 // for tracks on the visited stack must be returned to the front of
643 // the stack to be ordered before their dependents. We know that none
644 // of these tracks need input from tracks on the visited stack, so
645 // they can all be searched and ordered before the current stack head
647 removed
->mCycleMarker
= NOT_VISITED
;
648 dfsStack
.insertFront(removed
);
650 // Tracks in cycles without any DelayNodes must be muted, and so do
651 // not need input and can be ordered now. They must be ordered before
652 // their consumers so that their muted output is available.
653 removed
->mCycleMarker
= IN_MUTED_CYCLE
;
654 mTracks
[orderedTrackCount
] = removed
;
660 MOZ_ASSERT(orderedTrackCount
== mFirstCycleBreaker
);
663 TrackTime
MediaTrackGraphImpl::PlayAudio(const TrackAndVolume
& aOutput
,
664 GraphTime aPlayedTime
,
665 uint32_t aOutputChannelCount
) {
666 MOZ_ASSERT(OnGraphThread());
667 MOZ_ASSERT(mRealtime
, "Should only attempt to play audio in realtime mode");
669 TrackTime ticksWritten
= 0;
672 MediaTrack
* track
= aOutput
.mTrack
;
673 AudioSegment
* audio
= track
->GetData
<AudioSegment
>();
676 TrackTime offset
= track
->GraphTimeToTrackTime(aPlayedTime
);
678 // We don't update Track->mTracksStartTime here to account for time spent
679 // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
680 // the blocked period has completed. But we do need to make sure we play
681 // from the right offsets in the track buffer, even if we've already
682 // written silence for some amount of blocked time after the current time.
683 GraphTime t
= aPlayedTime
;
684 while (t
< mStateComputedTime
) {
685 bool blocked
= t
>= track
->mStartBlocking
;
686 GraphTime end
= blocked
? mStateComputedTime
: track
->mStartBlocking
;
687 NS_ASSERTION(end
<= mStateComputedTime
, "mStartBlocking is wrong!");
689 // Check how many ticks of sound we can provide if we are blocked some
690 // time in the middle of this cycle.
691 TrackTime toWrite
= end
- t
;
694 output
.InsertNullDataAtStart(toWrite
);
695 ticksWritten
+= toWrite
;
696 LOG(LogLevel::Verbose
,
697 ("%p: MediaTrack %p writing %" PRId64
" blocking-silence samples for "
698 "%f to %f (%" PRId64
" to %" PRId64
")",
699 this, track
, toWrite
, MediaTimeToSeconds(t
), MediaTimeToSeconds(end
),
700 offset
, offset
+ toWrite
));
702 TrackTime endTicksNeeded
= offset
+ toWrite
;
703 TrackTime endTicksAvailable
= audio
->GetDuration();
705 if (endTicksNeeded
<= endTicksAvailable
) {
706 LOG(LogLevel::Verbose
,
707 ("%p: MediaTrack %p writing %" PRId64
" samples for %f to %f "
708 "(samples %" PRId64
" to %" PRId64
")",
709 this, track
, toWrite
, MediaTimeToSeconds(t
),
710 MediaTimeToSeconds(end
), offset
, endTicksNeeded
));
711 output
.AppendSlice(*audio
, offset
, endTicksNeeded
);
712 ticksWritten
+= toWrite
;
713 offset
= endTicksNeeded
;
715 // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
716 // ended."); If we are at the end of the track, maybe write the
717 // remaining samples, and pad with/output silence.
718 if (endTicksNeeded
> endTicksAvailable
&& offset
< endTicksAvailable
) {
719 output
.AppendSlice(*audio
, offset
, endTicksAvailable
);
721 LOG(LogLevel::Verbose
,
722 ("%p: MediaTrack %p writing %" PRId64
" samples for %f to %f "
723 "(samples %" PRId64
" to %" PRId64
")",
724 this, track
, toWrite
, MediaTimeToSeconds(t
),
725 MediaTimeToSeconds(end
), offset
, endTicksNeeded
));
726 uint32_t available
= endTicksAvailable
- offset
;
727 ticksWritten
+= available
;
728 toWrite
-= available
;
729 offset
= endTicksAvailable
;
731 output
.AppendNullData(toWrite
);
732 LOG(LogLevel::Verbose
,
733 ("%p MediaTrack %p writing %" PRId64
" padding slsamples for %f to "
734 "%f (samples %" PRId64
" to %" PRId64
")",
735 this, track
, toWrite
, MediaTimeToSeconds(t
),
736 MediaTimeToSeconds(end
), offset
, endTicksNeeded
));
737 ticksWritten
+= toWrite
;
739 output
.ApplyVolume(mGlobalVolume
* aOutput
.mVolume
);
743 output
.Mix(mMixer
, aOutputChannelCount
, mSampleRate
);
748 DeviceInputTrack
* MediaTrackGraph::GetDeviceInputTrackMainThread(
749 CubebUtils::AudioDeviceID aID
) {
750 MOZ_ASSERT(NS_IsMainThread());
751 auto* impl
= static_cast<MediaTrackGraphImpl
*>(this);
752 return impl
->mDeviceInputTrackManagerMainThread
.GetDeviceInputTrack(aID
);
755 NativeInputTrack
* MediaTrackGraph::GetNativeInputTrackMainThread() {
756 MOZ_ASSERT(NS_IsMainThread());
757 auto* impl
= static_cast<MediaTrackGraphImpl
*>(this);
758 return impl
->mDeviceInputTrackManagerMainThread
.GetNativeInputTrack();
761 void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack
* aTrack
) {
762 MOZ_ASSERT(OnGraphThread());
764 ("%p OpenAudioInputImpl: device %p", this, aTrack
->mDeviceId
));
766 mDeviceInputTrackManagerGraphThread
.Add(aTrack
);
768 if (aTrack
->AsNativeInputTrack()) {
769 // Switch Drivers since we're adding input (to input-only or full-duplex)
770 AudioCallbackDriver
* driver
= new AudioCallbackDriver(
771 this, CurrentDriver(), mSampleRate
, PrimaryOutputChannelCount(),
772 AudioInputChannelCount(aTrack
->mDeviceId
), PrimaryOutputDeviceID(),
773 aTrack
->mDeviceId
, AudioInputDevicePreference(aTrack
->mDeviceId
));
775 ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
777 SwitchAtNextIteration(driver
);
779 NonNativeInputTrack
* nonNative
= aTrack
->AsNonNativeInputTrack();
780 MOZ_ASSERT(nonNative
);
781 // Start non-native input right away.
782 nonNative
->StartAudio(MakeRefPtr
<AudioInputSource
>(
783 MakeRefPtr
<AudioInputSourceListener
>(nonNative
),
784 nonNative
->GenerateSourceId(), nonNative
->mDeviceId
,
785 AudioInputChannelCount(nonNative
->mDeviceId
),
786 AudioInputDevicePreference(nonNative
->mDeviceId
) ==
787 AudioInputType::Voice
,
788 nonNative
->mPrincipalHandle
, nonNative
->mSampleRate
, GraphRate()));
792 void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack
* aTrack
) {
793 MOZ_ASSERT(NS_IsMainThread());
796 LOG(LogLevel::Debug
, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
797 aTrack
, aTrack
->mDeviceId
));
799 class Message
: public ControlMessage
{
801 Message(MediaTrackGraphImpl
* aGraph
, DeviceInputTrack
* aInputTrack
)
802 : ControlMessage(nullptr), mGraph(aGraph
), mInputTrack(aInputTrack
) {}
803 void Run() override
{
804 TRACE("MTG::OpenAudioInputImpl ControlMessage");
805 mGraph
->OpenAudioInputImpl(mInputTrack
);
807 MediaTrackGraphImpl
* mGraph
;
808 DeviceInputTrack
* mInputTrack
;
811 mDeviceInputTrackManagerMainThread
.Add(aTrack
);
813 this->AppendMessage(MakeUnique
<Message
>(this, aTrack
));
816 void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack
* aTrack
) {
817 MOZ_ASSERT(OnGraphThread());
820 ("%p CloseAudioInputImpl: device %p", this, aTrack
->mDeviceId
));
822 if (NonNativeInputTrack
* nonNative
= aTrack
->AsNonNativeInputTrack()) {
823 nonNative
->StopAudio();
824 mDeviceInputTrackManagerGraphThread
.Remove(aTrack
);
828 MOZ_ASSERT(aTrack
->AsNativeInputTrack());
830 mDeviceInputTrackManagerGraphThread
.Remove(aTrack
);
832 // Switch Drivers since we're adding or removing an input (to nothing/system
834 bool audioTrackPresent
= AudioTrackPresent();
837 if (audioTrackPresent
) {
838 // We still have audio output
840 ("%p: CloseInput: output present (AudioCallback)", this));
842 driver
= new AudioCallbackDriver(
843 this, CurrentDriver(), mSampleRate
, PrimaryOutputChannelCount(),
844 AudioInputChannelCount(aTrack
->mDeviceId
), PrimaryOutputDeviceID(),
845 nullptr, AudioInputDevicePreference(aTrack
->mDeviceId
));
846 SwitchAtNextIteration(driver
);
847 } else if (CurrentDriver()->AsAudioCallbackDriver()) {
849 ("%p: CloseInput: no output present (SystemClockCallback)", this));
851 driver
= new SystemClockDriver(this, CurrentDriver(), mSampleRate
);
852 SwitchAtNextIteration(driver
);
853 } // else SystemClockDriver->SystemClockDriver, no switch
856 void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack
* aTrack
) {
857 MOZ_ASSERT(OnGraphThreadOrNotRunning());
858 mOutputDevices
.RemoveElementsBy([&](OutputDeviceEntry
& aDeviceRef
) {
859 aDeviceRef
.mTrackOutputs
.RemoveElement(aTrack
);
860 // mReceiver is null for the primary output device, which is retained for
861 // AudioCallbackDriver output even when no tracks have audio outputs.
862 return aDeviceRef
.mTrackOutputs
.IsEmpty() && aDeviceRef
.mReceiver
;
866 void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack
* aTrack
) {
867 MOZ_ASSERT(NS_IsMainThread());
870 LOG(LogLevel::Debug
, ("%p CloseInput: DeviceInputTrack %p for device %p",
871 this, aTrack
, aTrack
->mDeviceId
));
873 class Message
: public ControlMessage
{
875 Message(MediaTrackGraphImpl
* aGraph
, DeviceInputTrack
* aInputTrack
)
876 : ControlMessage(nullptr), mGraph(aGraph
), mInputTrack(aInputTrack
) {}
877 void Run() override
{
878 TRACE("MTG::CloseAudioInputImpl ControlMessage");
879 mGraph
->CloseAudioInputImpl(mInputTrack
);
881 MediaTrackGraphImpl
* mGraph
;
882 DeviceInputTrack
* mInputTrack
;
885 // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
886 // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
888 mDeviceInputTrackManagerMainThread
.Remove(aTrack
);
890 this->AppendMessage(MakeUnique
<Message
>(this, aTrack
));
892 if (aTrack
->AsNativeInputTrack()) {
894 ("%p Native input device %p is closed!", this, aTrack
->mDeviceId
));
899 // All AudioInput listeners get the same speaker data (at least for now).
900 void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk
& aChunk
) {
901 if (!mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack()) {
905 #if defined(MOZ_WEBRTC)
906 for (const auto& track
: mTracks
) {
907 if (const auto& t
= track
->AsAudioProcessingTrack()) {
908 t
->NotifyOutputData(this, aChunk
);
914 void MediaTrackGraphImpl::NotifyInputStopped() {
915 NativeInputTrack
* native
=
916 mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack();
920 native
->NotifyInputStopped(this);
923 void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue
* aBuffer
,
924 size_t aFrames
, TrackRate aRate
,
926 uint32_t aAlreadyBuffered
) {
927 // Either we have an audio input device, or we just removed the audio input
928 // this iteration, and we're switching back to an output-only driver next
930 NativeInputTrack
* native
=
931 mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack();
932 MOZ_ASSERT(native
|| Switching());
936 native
->NotifyInputData(this, aBuffer
, aFrames
, aRate
, aChannels
,
940 void MediaTrackGraphImpl::DeviceChangedImpl() {
941 MOZ_ASSERT(OnGraphThread());
942 NativeInputTrack
* native
=
943 mDeviceInputTrackManagerGraphThread
.GetNativeInputTrack();
947 native
->DeviceChanged(this);
950 void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount
) {
951 MOZ_ASSERT(OnGraphThread());
952 mMaxOutputChannelCount
= aMaxChannelCount
;
955 void MediaTrackGraphImpl::DeviceChanged() {
956 // This is safe to be called from any thread: this message comes from an
957 // underlying platform API, and we don't have much guarantees. If it is not
958 // called from the main thread (and it probably will rarely be), it will post
959 // itself to the main thread, and the actual device change message will be ran
960 // and acted upon on the graph thread.
961 if (!NS_IsMainThread()) {
962 RefPtr
<nsIRunnable
> runnable
= WrapRunnable(
963 RefPtr
<MediaTrackGraphImpl
>(this), &MediaTrackGraphImpl::DeviceChanged
);
964 mMainThread
->Dispatch(runnable
.forget());
968 class Message
: public ControlMessage
{
970 explicit Message(MediaTrackGraph
* aGraph
)
971 : ControlMessage(nullptr),
972 mGraphImpl(static_cast<MediaTrackGraphImpl
*>(aGraph
)) {}
973 void Run() override
{
974 TRACE("MTG::DeviceChangeImpl ControlMessage");
975 mGraphImpl
->DeviceChangedImpl();
977 // We know that this is valid, because the graph can't shutdown if it has
979 MediaTrackGraphImpl
* mGraphImpl
;
982 if (mMainThreadTrackCount
== 0 && mMainThreadPortCount
== 0) {
983 // This is a special case where the origin of this event cannot control the
984 // lifetime of the graph, because the graph is controling the lifetime of
985 // the AudioCallbackDriver where the event originated.
986 // We know the graph is soon going away, so there's no need to notify about
987 // this device change.
991 // Reset the latency, it will get fetched again next time it's queried.
992 MOZ_ASSERT(NS_IsMainThread());
993 mAudioOutputLatency
= 0.0;
995 // Dispatch to the bg thread to do the (potentially expensive) query of the
996 // maximum channel count, and then dispatch back to the main thread, then to
997 // the graph, with the new info.
998 RefPtr
<MediaTrackGraphImpl
> self
= this;
999 NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1000 "MaxChannelCountUpdateOnBgThread", [self
{std::move(self
)}]() {
1001 uint32_t maxChannelCount
= CubebUtils::MaxNumberOfChannels();
1002 self
->Dispatch(NS_NewRunnableFunction(
1003 "MaxChannelCountUpdateToMainThread",
1004 [self
{self
}, maxChannelCount
]() {
1005 class MessageToGraph
: public ControlMessage
{
1007 explicit MessageToGraph(MediaTrackGraph
* aGraph
,
1008 uint32_t aMaxChannelCount
)
1009 : ControlMessage(nullptr),
1010 mGraphImpl(static_cast<MediaTrackGraphImpl
*>(aGraph
)),
1011 mMaxChannelCount(aMaxChannelCount
) {}
1012 void Run() override
{
1013 TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
1014 mGraphImpl
->SetMaxOutputChannelCount(mMaxChannelCount
);
1016 MediaTrackGraphImpl
* mGraphImpl
;
1017 uint32_t mMaxChannelCount
;
1019 self
->AppendMessage(
1020 MakeUnique
<MessageToGraph
>(self
, maxChannelCount
));
1024 AppendMessage(MakeUnique
<Message
>(this));
1027 static const char* GetAudioInputTypeString(const AudioInputType
& aType
) {
1028 return aType
== AudioInputType::Voice
? "Voice" : "Unknown";
1031 void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID
) {
1032 MOZ_ASSERT(OnGraphThread());
1033 auto* impl
= static_cast<MediaTrackGraphImpl
*>(this);
1034 impl
->ReevaluateInputDevice(aID
);
1037 void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID
) {
1038 MOZ_ASSERT(OnGraphThread());
1039 LOG(LogLevel::Debug
, ("%p: ReevaluateInputDevice: device %p", this, aID
));
1041 DeviceInputTrack
* track
=
1042 mDeviceInputTrackManagerGraphThread
.GetDeviceInputTrack(aID
);
1044 LOG(LogLevel::Debug
,
1045 ("%p: No DeviceInputTrack for this device. Ignore", this));
1049 bool needToSwitch
= false;
1051 if (NonNativeInputTrack
* nonNative
= track
->AsNonNativeInputTrack()) {
1052 if (nonNative
->NumberOfChannels() != AudioInputChannelCount(aID
)) {
1053 LOG(LogLevel::Debug
,
1054 ("%p: %u-channel non-native input device %p (track %p) is "
1055 "re-configured to %d-channel",
1056 this, nonNative
->NumberOfChannels(), aID
, track
,
1057 AudioInputChannelCount(aID
)));
1058 needToSwitch
= true;
1060 if (nonNative
->DevicePreference() != AudioInputDevicePreference(aID
)) {
1061 LOG(LogLevel::Debug
,
1062 ("%p: %s-type non-native input device %p (track %p) is re-configured "
1064 this, GetAudioInputTypeString(nonNative
->DevicePreference()), aID
,
1065 track
, GetAudioInputTypeString(AudioInputDevicePreference(aID
))));
1066 needToSwitch
= true;
1070 nonNative
->StopAudio();
1071 nonNative
->StartAudio(MakeRefPtr
<AudioInputSource
>(
1072 MakeRefPtr
<AudioInputSourceListener
>(nonNative
),
1073 nonNative
->GenerateSourceId(), aID
, AudioInputChannelCount(aID
),
1074 AudioInputDevicePreference(aID
) == AudioInputType::Voice
,
1075 nonNative
->mPrincipalHandle
, nonNative
->mSampleRate
, GraphRate()));
1081 MOZ_ASSERT(track
->AsNativeInputTrack());
1083 if (AudioCallbackDriver
* audioCallbackDriver
=
1084 CurrentDriver()->AsAudioCallbackDriver()) {
1085 if (audioCallbackDriver
->InputChannelCount() !=
1086 AudioInputChannelCount(aID
)) {
1087 LOG(LogLevel::Debug
,
1088 ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
1089 "re-configured to %d-channel",
1090 this, audioCallbackDriver
->InputChannelCount(), audioCallbackDriver
,
1091 AudioInputChannelCount(aID
)));
1092 needToSwitch
= true;
1094 if (audioCallbackDriver
->InputDevicePreference() !=
1095 AudioInputDevicePreference(aID
)) {
1096 LOG(LogLevel::Debug
,
1097 ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
1098 "re-configured to %s-type",
1100 GetAudioInputTypeString(
1101 audioCallbackDriver
->InputDevicePreference()),
1102 audioCallbackDriver
,
1103 GetAudioInputTypeString(AudioInputDevicePreference(aID
))));
1104 needToSwitch
= true;
1106 } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
1107 // We're already in the process of switching to a audio callback driver,
1108 // which will happen at the next iteration.
1109 // However, maybe it's not the correct number of channels. Re-query the
1110 // correct channel amount at this time.
1111 needToSwitch
= true;
1115 AudioCallbackDriver
* newDriver
= new AudioCallbackDriver(
1116 this, CurrentDriver(), mSampleRate
, PrimaryOutputChannelCount(),
1117 AudioInputChannelCount(aID
), PrimaryOutputDeviceID(), aID
,
1118 AudioInputDevicePreference(aID
));
1119 SwitchAtNextIteration(newDriver
);
1123 bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
1124 // either we're on the right thread (and calling CurrentDriver() is safe),
1125 // or we're going to fail the assert anyway, so don't cross-check
1126 // via CurrentDriver().
1127 return mGraphDriverRunning
? OnGraphThread() : NS_IsMainThread();
1130 bool MediaTrackGraphImpl::OnGraphThread() const {
1131 // we're on the right thread (and calling mDriver is safe),
1132 MOZ_ASSERT(mDriver
);
1133 if (mGraphRunner
&& mGraphRunner
->OnThread()) {
1136 return mDriver
->OnThread();
1139 bool MediaTrackGraphImpl::Destroyed() const {
1140 MOZ_ASSERT(NS_IsMainThread());
1144 bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
1145 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1150 TimeStamp now
= TimeStamp::Now();
1151 // For offline graphs, update now if it has been long enough since the last
1152 // update, or if it has reached the end.
1153 if ((now
- mLastMainThreadUpdate
).ToMilliseconds() >
1154 CurrentDriver()->IterationDuration() ||
1155 mStateComputedTime
>= mEndTime
) {
1156 mLastMainThreadUpdate
= now
;
1162 void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate
) {
1163 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1164 mMonitor
.AssertCurrentThreadOwns();
1166 // We don't want to frequently update the main thread about timing update
1167 // when we are not running in realtime.
1168 if (aFinalUpdate
|| ShouldUpdateMainThread()) {
1169 // Strip updates that will be obsoleted below, so as to keep the length of
1170 // mTrackUpdates sane.
1171 size_t keptUpdateCount
= 0;
1172 for (size_t i
= 0; i
< mTrackUpdates
.Length(); ++i
) {
1173 MediaTrack
* track
= mTrackUpdates
[i
].mTrack
;
1174 // RemoveTrackGraphThread() clears mTrack in updates for
1175 // tracks that are removed from the graph.
1176 MOZ_ASSERT(!track
|| track
->GraphImpl() == this);
1177 if (!track
|| track
->MainThreadNeedsUpdates()) {
1178 // Discard this update as it has either been cleared when the track
1179 // was destroyed or there will be a newer update below.
1182 if (keptUpdateCount
!= i
) {
1183 mTrackUpdates
[keptUpdateCount
] = std::move(mTrackUpdates
[i
]);
1184 MOZ_ASSERT(!mTrackUpdates
[i
].mTrack
);
1188 mTrackUpdates
.TruncateLength(keptUpdateCount
);
1190 mTrackUpdates
.SetCapacity(mTrackUpdates
.Length() + mTracks
.Length() +
1191 mSuspendedTracks
.Length());
1192 for (MediaTrack
* track
: AllTracks()) {
1193 if (!track
->MainThreadNeedsUpdates()) {
1196 TrackUpdate
* update
= mTrackUpdates
.AppendElement();
1197 update
->mTrack
= track
;
1198 // No blocking to worry about here, since we've passed
1199 // UpdateCurrentTimeForTracks.
1200 update
->mNextMainThreadCurrentTime
=
1201 track
->GraphTimeToTrackTime(mProcessedTime
);
1202 update
->mNextMainThreadEnded
= track
->mNotifiedEnded
;
1204 mNextMainThreadGraphTime
= mProcessedTime
;
1205 if (!mPendingUpdateRunnables
.IsEmpty()) {
1206 mUpdateRunnables
.AppendElements(std::move(mPendingUpdateRunnables
));
1210 // If this is the final update, then a stable state event will soon be
1211 // posted just before this thread finishes, and so there is no need to also
1213 if (!aFinalUpdate
&&
1214 // Don't send the message to the main thread if it's not going to have
1216 !(mUpdateRunnables
.IsEmpty() && mTrackUpdates
.IsEmpty())) {
1217 EnsureStableStateEventPosted();
1221 GraphTime
MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime
) {
1222 if (aTime
% WEBAUDIO_BLOCK_SIZE
== 0) {
1225 return RoundUpToNextAudioBlock(aTime
);
1228 GraphTime
MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime
) {
1229 uint64_t block
= aTime
>> WEBAUDIO_BLOCK_SIZE_BITS
;
1230 uint64_t nextBlock
= block
+ 1;
1231 GraphTime nextTime
= nextBlock
<< WEBAUDIO_BLOCK_SIZE_BITS
;
1235 void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
1236 uint32_t aTrackIndex
, TrackRate aSampleRate
) {
1237 MOZ_ASSERT(OnGraphThread());
1238 MOZ_ASSERT(aTrackIndex
<= mFirstCycleBreaker
,
1239 "Cycle breaker is not AudioNodeTrack?");
1241 while (mProcessedTime
< mStateComputedTime
) {
1242 // Microtask checkpoints are in between render quanta.
1245 GraphTime next
= RoundUpToNextAudioBlock(mProcessedTime
);
1246 for (uint32_t i
= mFirstCycleBreaker
; i
< mTracks
.Length(); ++i
) {
1247 auto nt
= static_cast<AudioNodeTrack
*>(mTracks
[i
]);
1248 MOZ_ASSERT(nt
->AsAudioNodeTrack());
1249 nt
->ProduceOutputBeforeInput(mProcessedTime
);
1251 for (uint32_t i
= aTrackIndex
; i
< mTracks
.Length(); ++i
) {
1252 ProcessedMediaTrack
* pt
= mTracks
[i
]->AsProcessedTrack();
1255 mProcessedTime
, next
,
1256 (next
== mStateComputedTime
) ? ProcessedMediaTrack::ALLOW_END
: 0);
1259 mProcessedTime
= next
;
1261 NS_ASSERTION(mProcessedTime
== mStateComputedTime
,
1262 "Something went wrong with rounding to block boundaries");
1265 void MediaTrackGraphImpl::RunMessageAfterProcessing(
1266 UniquePtr
<ControlMessageInterface
> aMessage
) {
1267 MOZ_ASSERT(OnGraphThread());
1269 if (mFrontMessageQueue
.IsEmpty()) {
1270 mFrontMessageQueue
.AppendElement();
1273 // Only one block is used for messages from the graph thread.
1274 MOZ_ASSERT(mFrontMessageQueue
.Length() == 1);
1275 mFrontMessageQueue
[0].mMessages
.AppendElement(std::move(aMessage
));
1278 void MediaTrackGraphImpl::RunMessagesInQueue() {
1279 TRACE("MTG::RunMessagesInQueue");
1280 MOZ_ASSERT(OnGraphThread());
1281 // Calculate independent action times for each batch of messages (each
1282 // batch corresponding to an event loop task). This isolates the performance
1283 // of different scripts to some extent.
1284 for (uint32_t i
= 0; i
< mFrontMessageQueue
.Length(); ++i
) {
1285 nsTArray
<UniquePtr
<ControlMessageInterface
>>& messages
=
1286 mFrontMessageQueue
[i
].mMessages
;
1288 for (uint32_t j
= 0; j
< messages
.Length(); ++j
) {
1289 TRACE("ControlMessage::Run");
1293 mFrontMessageQueue
.Clear();
1296 void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions
) {
1297 TRACE("MTG::UpdateGraph");
1298 MOZ_ASSERT(OnGraphThread());
1299 MOZ_ASSERT(aEndBlockingDecisions
>= mProcessedTime
);
1300 // The next state computed time can be the same as the previous: it
1301 // means the driver would have been blocking indefinitly, but the graph has
1302 // been woken up right after having been to sleep.
1303 MOZ_ASSERT(aEndBlockingDecisions
>= mStateComputedTime
);
1308 // Always do another iteration if there are tracks waiting to resume.
1309 bool ensureNextIteration
= !mPendingResumeOperations
.IsEmpty();
1311 for (MediaTrack
* track
: mTracks
) {
1312 if (SourceMediaTrack
* is
= track
->AsSourceTrack()) {
1313 ensureNextIteration
|= is
->PullNewData(aEndBlockingDecisions
);
1314 is
->ExtractPendingInput(mStateComputedTime
, aEndBlockingDecisions
);
1316 if (track
->mEnded
) {
1317 // The track's not suspended, and since it's ended, underruns won't
1318 // stop it playing out. So there's no blocking other than what we impose
1320 GraphTime endTime
= track
->GetEnd() + track
->mStartTime
;
1321 if (endTime
<= mStateComputedTime
) {
1322 LOG(LogLevel::Verbose
,
1323 ("%p: MediaTrack %p is blocked due to being ended", this, track
));
1324 track
->mStartBlocking
= mStateComputedTime
;
1326 LOG(LogLevel::Verbose
,
1327 ("%p: MediaTrack %p has ended, but is not blocked yet (current "
1328 "time %f, end at %f)",
1329 this, track
, MediaTimeToSeconds(mStateComputedTime
),
1330 MediaTimeToSeconds(endTime
)));
1331 // Data can't be added to a ended track, so underruns are irrelevant.
1332 MOZ_ASSERT(endTime
<= aEndBlockingDecisions
);
1333 track
->mStartBlocking
= endTime
;
1336 track
->mStartBlocking
= WillUnderrun(track
, aEndBlockingDecisions
);
1338 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
1339 if (SourceMediaTrack
* s
= track
->AsSourceTrack()) {
1344 MutexAutoLock
lock(s
->mMutex
);
1345 if (!s
->mUpdateTrack
->mPullingEnabled
) {
1346 // The invariant that data must be provided is only enforced when
1351 if (track
->GetEnd() <
1352 track
->GraphTimeToTrackTime(aEndBlockingDecisions
)) {
1353 LOG(LogLevel::Error
,
1354 ("%p: SourceMediaTrack %p (%s) is live and pulled, "
1356 "enough data. TrackListeners=%zu. Track-end=%f, "
1359 (track
->mType
== MediaSegment::AUDIO
? "audio" : "video"),
1360 track
->mTrackListeners
.Length(),
1361 MediaTimeToSeconds(track
->GetEnd()),
1363 track
->GraphTimeToTrackTime(aEndBlockingDecisions
))));
1364 MOZ_DIAGNOSTIC_ASSERT(false,
1365 "A non-ended SourceMediaTrack wasn't fed "
1366 "enough data by NotifyPull");
1369 #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
1373 for (MediaTrack
* track
: mSuspendedTracks
) {
1374 track
->mStartBlocking
= mStateComputedTime
;
1377 // If the loop is woken up so soon that IterationEnd() barely advances or
1378 // if an offline graph is not currently rendering, we end up having
1379 // aEndBlockingDecisions == mStateComputedTime.
1380 // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
1381 // empty, Process() will not find any unblocked track and so will not
1382 // ensure another iteration. If the graph should be rendering, then ensure
1383 // another iteration to render.
1384 if (ensureNextIteration
|| (aEndBlockingDecisions
== mStateComputedTime
&&
1385 mStateComputedTime
< mEndTime
)) {
1386 EnsureNextIteration();
1390 void MediaTrackGraphImpl::Process(MixerCallbackReceiver
* aMixerReceiver
) {
1391 TRACE("MTG::Process");
1392 MOZ_ASSERT(OnGraphThread());
1393 // Play track contents.
1394 bool allBlockedForever
= true;
1395 // True when we've done ProcessInput for all processed tracks.
1396 bool doneAllProducing
= false;
1397 const GraphTime oldProcessedTime
= mProcessedTime
;
1399 // Figure out what each track wants to do
1400 for (uint32_t i
= 0; i
< mTracks
.Length(); ++i
) {
1401 MediaTrack
* track
= mTracks
[i
];
1402 if (!doneAllProducing
) {
1403 ProcessedMediaTrack
* pt
= track
->AsProcessedTrack();
1405 AudioNodeTrack
* n
= track
->AsAudioNodeTrack();
1408 // Verify that the sampling rate for all of the following tracks is
1410 for (uint32_t j
= i
+ 1; j
< mTracks
.Length(); ++j
) {
1411 AudioNodeTrack
* nextTrack
= mTracks
[j
]->AsAudioNodeTrack();
1413 MOZ_ASSERT(n
->mSampleRate
== nextTrack
->mSampleRate
,
1414 "All AudioNodeTracks in the graph must have the same "
1419 // Since an AudioNodeTrack is present, go ahead and
1420 // produce audio block by block for all the rest of the tracks.
1421 ProduceDataForTracksBlockByBlock(i
, n
->mSampleRate
);
1422 doneAllProducing
= true;
1424 pt
->ProcessInput(mProcessedTime
, mStateComputedTime
,
1425 ProcessedMediaTrack::ALLOW_END
);
1426 // Assert that a live track produced enough data
1427 MOZ_ASSERT_IF(!track
->mEnded
,
1428 track
->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
1429 track
, mStateComputedTime
));
1433 if (track
->mStartBlocking
> oldProcessedTime
) {
1434 allBlockedForever
= false;
1437 mProcessedTime
= mStateComputedTime
;
1439 for (const auto& outputDeviceEntry
: mOutputDevices
) {
1440 uint32_t outputChannelCount
;
1441 if (!outputDeviceEntry
.mReceiver
) { // primary output
1442 if (!aMixerReceiver
) {
1443 // Running off a system clock driver. No need to mix output.
1446 MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(),
1447 "Driver must be AudioCallbackDriver if aMixerReceiver");
1448 // Use the number of channel the driver expects: this is the number of
1449 // channel that can be output by the underlying system level audio stream.
1450 outputChannelCount
=
1451 CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
1453 outputChannelCount
= AudioOutputChannelCount(outputDeviceEntry
);
1455 MOZ_ASSERT(mRealtime
,
1456 "If there's an output device, this graph must be realtime");
1457 mMixer
.StartMixing();
1458 // This is the number of frames that are written to the output buffer, for
1460 TrackTime ticksPlayed
= 0;
1461 for (const auto& t
: outputDeviceEntry
.mTrackOutputs
) {
1462 TrackTime ticksPlayedForThisTrack
=
1463 PlayAudio(t
, oldProcessedTime
, outputChannelCount
);
1464 if (ticksPlayed
== 0) {
1465 ticksPlayed
= ticksPlayedForThisTrack
;
1468 !ticksPlayedForThisTrack
|| ticksPlayedForThisTrack
== ticksPlayed
,
1469 "Each track should have the same number of frames.");
1473 if (ticksPlayed
== 0) {
1474 // Nothing was played, so the mixer doesn't know how many frames were
1475 // processed. We still tell it so AudioCallbackDriver knows how much has
1476 // been processed. (bug 1406027)
1477 mMixer
.Mix(nullptr, outputChannelCount
,
1478 mStateComputedTime
- oldProcessedTime
, mSampleRate
);
1480 AudioChunk
* outputChunk
= mMixer
.MixedChunk();
1481 if (!outputDeviceEntry
.mReceiver
) { // primary output
1482 // Callback any observers for the AEC speaker data. Note that one
1483 // (maybe) of these will be full-duplex, the others will get their input
1484 // data off separate cubeb callbacks.
1485 NotifyOutputData(*outputChunk
);
1487 aMixerReceiver
->MixerCallback(outputChunk
, mSampleRate
);
1489 outputDeviceEntry
.mReceiver
->EnqueueAudio(*outputChunk
);
1493 if (!allBlockedForever
) {
1494 EnsureNextIteration();
1498 bool MediaTrackGraphImpl::UpdateMainThreadState() {
1499 MOZ_ASSERT(OnGraphThread());
1500 if (mForceShutDownReceived
) {
1501 for (MediaTrack
* track
: AllTracks()) {
1502 track
->OnGraphThreadDone();
1506 MonitorAutoLock
lock(mMonitor
);
1508 mForceShutDownReceived
|| (IsEmpty() && mBackMessageQueue
.IsEmpty());
1509 PrepareUpdatesToMainThreadState(finalUpdate
);
1511 SwapMessageQueues();
1514 // The JSContext will not be used again.
1515 // Clear main thread access while under monitor.
1516 mJSContext
= nullptr;
1518 dom::WorkletThread::DeleteCycleCollectedJSContext();
1519 // Enter shutdown mode when this iteration is completed.
1520 // No need to Destroy tracks here. The main-thread owner of each
1521 // track is responsible for calling Destroy on them.
1525 auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime
,
1526 GraphTime aIterationEnd
,
1527 MixerCallbackReceiver
* aMixerReceiver
)
1528 -> IterationResult
{
1530 return mGraphRunner
->OneIteration(aStateTime
, aIterationEnd
,
1534 return OneIterationImpl(aStateTime
, aIterationEnd
, aMixerReceiver
);
1537 auto MediaTrackGraphImpl::OneIterationImpl(
1538 GraphTime aStateTime
, GraphTime aIterationEnd
,
1539 MixerCallbackReceiver
* aMixerReceiver
) -> IterationResult
{
1540 TRACE("MTG::OneIterationImpl");
1542 mIterationEndTime
= aIterationEnd
;
1544 if (SoftRealTimeLimitReached()) {
1545 TRACE("MTG::Demoting real-time thread!");
1546 DemoteThreadFromRealTime();
1549 // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
1550 // thread, and so the monitor need not be held to check mLifecycleState.
1551 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
1552 // graphs that have not started.
1554 // While changes occur on mainthread, this assert confirms that
1555 // this code shouldn't run if mainthread might be changing the state (to
1556 // > LIFECYCLE_RUNNING)
1558 // Ignore mutex warning: static during execution of the graph
1559 MOZ_PUSH_IGNORE_THREAD_SAFETY
1560 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState
<= LIFECYCLE_RUNNING
);
1561 MOZ_POP_THREAD_SAFETY
1563 MOZ_ASSERT(OnGraphThread());
1565 WebCore::DenormalDisabler disabler
;
1567 // Process graph message from the main thread for this iteration.
1568 RunMessagesInQueue();
1570 // Process MessagePort events.
1571 // These require a single thread, which has an nsThread with an event queue.
1572 if (mGraphRunner
|| !mRealtime
) {
1573 TRACE("MTG::MessagePort events");
1574 NS_ProcessPendingEvents(nullptr);
1577 GraphTime stateTime
= std::min(aStateTime
, GraphTime(mEndTime
));
1578 UpdateGraph(stateTime
);
1580 mStateComputedTime
= stateTime
;
1582 GraphTime oldProcessedTime
= mProcessedTime
;
1583 Process(aMixerReceiver
);
1584 MOZ_ASSERT(mProcessedTime
== stateTime
);
1586 UpdateCurrentTimeForTracks(oldProcessedTime
);
1588 ProcessChunkMetadata(oldProcessedTime
);
1590 // Process graph messages queued from RunMessageAfterProcessing() on this
1591 // thread during the iteration.
1592 RunMessagesInQueue();
1594 if (!UpdateMainThreadState()) {
1596 // We'll never get to do this switch. Clear mNextDriver to break the
1597 // ref-cycle graph->nextDriver->currentDriver->graph.
1598 SwitchAtNextIteration(nullptr);
1600 return IterationResult::CreateStop(
1601 NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
1602 &MediaTrackGraphImpl::SignalMainThreadCleanup
));
1606 RefPtr
<GraphDriver
> nextDriver
= std::move(mNextDriver
);
1607 return IterationResult::CreateSwitchDriver(
1608 nextDriver
, NewRunnableMethod
<StoreRefPtrPassByPtr
<GraphDriver
>>(
1609 "MediaTrackGraphImpl::SetCurrentDriver", this,
1610 &MediaTrackGraphImpl::SetCurrentDriver
, nextDriver
));
1613 return IterationResult::CreateStillProcessing();
1616 void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate
* aUpdate
) {
1617 MOZ_ASSERT(NS_IsMainThread());
1618 mMonitor
.AssertCurrentThreadOwns();
1620 MediaTrack
* track
= aUpdate
->mTrack
;
1622 track
->mMainThreadCurrentTime
= aUpdate
->mNextMainThreadCurrentTime
;
1623 track
->mMainThreadEnded
= aUpdate
->mNextMainThreadEnded
;
1625 if (track
->ShouldNotifyTrackEnded()) {
1626 track
->NotifyMainThreadListeners();
1630 void MediaTrackGraphImpl::ForceShutDown() {
1631 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1632 LOG(LogLevel::Debug
, ("%p: MediaTrackGraph::ForceShutdown", this));
1634 if (mShutdownBlocker
) {
1635 // Avoid waiting forever for a graph to shut down
1636 // synchronously. Reports are that some 3rd-party audio drivers
1637 // occasionally hang in shutdown (both for us and Chrome).
1638 NS_NewTimerWithCallback(
1639 getter_AddRefs(mShutdownTimer
), this,
1640 MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT
,
1641 nsITimer::TYPE_ONE_SHOT
);
1644 class Message final
: public ControlMessage
{
1646 explicit Message(MediaTrackGraphImpl
* aGraph
)
1647 : ControlMessage(nullptr), mGraph(aGraph
) {}
1648 void Run() override
{
1649 TRACE("MTG::ForceShutdown ControlMessage");
1650 mGraph
->mForceShutDownReceived
= true;
1652 // The graph owns this message.
1653 MediaTrackGraphImpl
* MOZ_NON_OWNING_REF mGraph
;
1656 if (mMainThreadTrackCount
> 0 || mMainThreadPortCount
> 0) {
1657 // If both the track and port counts are zero, the regular shutdown
1658 // sequence will progress shortly to shutdown threads and destroy the graph.
1659 AppendMessage(MakeUnique
<Message
>(this));
1665 MediaTrackGraphImpl::Notify(nsITimer
* aTimer
) {
1666 MOZ_ASSERT(NS_IsMainThread());
1667 NS_ASSERTION(!mShutdownBlocker
,
1668 "MediaTrackGraph took too long to shut down!");
1669 // Sigh, graph took too long to shut down. Stop blocking system
1670 // shutdown and hope all is well.
1671 RemoveShutdownBlocker();
1675 static nsCString
GetDocumentTitle(uint64_t aWindowID
) {
1676 MOZ_ASSERT(NS_IsMainThread());
1678 auto* win
= nsGlobalWindowInner::GetInnerWindowWithId(aWindowID
);
1682 Document
* doc
= win
->GetExtantDoc();
1686 nsAutoString titleUTF16
;
1687 doc
->GetTitle(titleUTF16
);
1688 CopyUTF16toUTF8(titleUTF16
, title
);
1693 MediaTrackGraphImpl::Observe(nsISupports
* aSubject
, const char* aTopic
,
1694 const char16_t
* aData
) {
1695 MOZ_ASSERT(NS_IsMainThread());
1696 MOZ_ASSERT(strcmp(aTopic
, "document-title-changed") == 0);
1697 nsCString streamName
= GetDocumentTitle(mWindowID
);
1698 LOG(LogLevel::Debug
, ("%p: document title: %s", this, streamName
.get()));
1699 if (streamName
.IsEmpty()) {
1702 QueueControlMessageWithNoShutdown(
1703 [self
= RefPtr
{this}, this, streamName
= std::move(streamName
)] {
1704 CurrentDriver()->SetStreamName(streamName
);
1709 bool MediaTrackGraphImpl::AddShutdownBlocker() {
1710 MOZ_ASSERT(NS_IsMainThread());
1711 MOZ_ASSERT(!mShutdownBlocker
);
1713 class Blocker
: public media::ShutdownBlocker
{
1714 const RefPtr
<MediaTrackGraphImpl
> mGraph
;
1717 Blocker(MediaTrackGraphImpl
* aGraph
, const nsString
& aName
)
1718 : media::ShutdownBlocker(aName
), mGraph(aGraph
) {}
1721 BlockShutdown(nsIAsyncShutdownClient
* aProfileBeforeChange
) override
{
1722 mGraph
->ForceShutDown();
1727 nsCOMPtr
<nsIAsyncShutdownClient
> barrier
= media::GetShutdownBarrier();
1729 // We're already shutting down, we won't be able to add a blocker, bail.
1730 LOG(LogLevel::Error
,
1731 ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
1736 // Blocker names must be distinct.
1737 nsString blockerName
;
1738 blockerName
.AppendPrintf("MediaTrackGraph %p shutdown", this);
1739 mShutdownBlocker
= MakeAndAddRef
<Blocker
>(this, blockerName
);
1740 nsresult rv
= barrier
->AddBlocker(mShutdownBlocker
,
1741 NS_LITERAL_STRING_FROM_CSTRING(__FILE__
),
1742 __LINE__
, u
"MediaTrackGraph shutdown"_ns
);
1743 MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv
));
1747 void MediaTrackGraphImpl::RemoveShutdownBlocker() {
1748 if (!mShutdownBlocker
) {
1751 media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker
);
1752 mShutdownBlocker
= nullptr;
1756 MediaTrackGraphImpl::GetName(nsACString
& aName
) {
1757 aName
.AssignLiteral("MediaTrackGraphImpl");
1763 class MediaTrackGraphShutDownRunnable
: public Runnable
{
1765 explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl
* aGraph
)
1766 : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph
) {}
1767 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
1769 MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD
Run() override
{
1770 TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
1771 MOZ_ASSERT(NS_IsMainThread());
1772 MOZ_ASSERT(!mGraph
->mGraphDriverRunning
&& mGraph
->mDriver
,
1773 "We should know the graph thread control loop isn't running!");
1775 LOG(LogLevel::Debug
, ("%p: Shutting down graph", mGraph
.get()));
1777 // We've asserted the graph isn't running. Use mDriver instead of
1778 // CurrentDriver to avoid thread-safety checks
1779 #if 0 // AudioCallbackDrivers are released asynchronously anyways
1780 // XXX a better test would be have setting mGraphDriverRunning make sure
1781 // any current callback has finished and block future ones -- or just
1782 // handle it all in Shutdown()!
1783 if (mGraph
->mDriver
->AsAudioCallbackDriver()) {
1784 MOZ_ASSERT(!mGraph
->mDriver
->AsAudioCallbackDriver()->InCallback());
1788 for (MediaTrackGraphImpl::PendingResumeOperation
& op
:
1789 mGraph
->mPendingResumeOperations
) {
1793 if (mGraph
->mGraphRunner
) {
1794 RefPtr
<GraphRunner
>(mGraph
->mGraphRunner
)->Shutdown();
1797 RefPtr
<GraphDriver
>(mGraph
->mDriver
)->Shutdown();
1799 // Release the driver now so that an AudioCallbackDriver will release its
1800 // SharedThreadPool reference. Each SharedThreadPool reference must be
1801 // released before SharedThreadPool::SpinUntilEmpty() runs on
1802 // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
1803 // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
1804 // which won't otherwise release its reference on the graph until
1805 // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
1806 mGraph
->SetCurrentDriver(nullptr);
1808 // Safe to access these without the monitor since the graph isn't running.
1809 // We may be one of several graphs. Drop ticket to eventually unblock
1811 if (mGraph
->mShutdownTimer
&& !mGraph
->mShutdownBlocker
) {
1814 "AudioCallbackDriver took too long to shut down and we let shutdown"
1815 " continue - freezing and leaking");
1817 // The timer fired, so we may be deeper in shutdown now. Block any
1818 // further teardown and just leak, for safety.
1822 // mGraph's thread is not running so it's OK to do whatever here
1823 for (MediaTrack
* track
: mGraph
->AllTracks()) {
1824 // Clean up all MediaSegments since we cannot release Images too
1825 // late during shutdown. Also notify listeners that they were removed
1826 // so they can clean up any gfx resources.
1827 track
->RemoveAllResourcesAndListenersImpl();
1832 MonitorAutoLock
lock(mGraph
->mMonitor
);
1833 MOZ_ASSERT(mGraph
->mUpdateRunnables
.IsEmpty());
1836 mGraph
->mPendingUpdateRunnables
.Clear();
1838 mGraph
->RemoveShutdownBlocker();
1840 // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
1841 // stage, since completion of that stage requires all tracks to be freed,
1842 // which requires shutdown to proceed.
1844 if (mGraph
->IsEmpty()) {
1845 // mGraph is no longer needed, so delete it.
1848 // The graph is not empty. We must be in a forced shutdown.
1849 // Some later AppendMessage will detect that the graph has
1850 // been emptied, and delete it.
1851 NS_ASSERTION(mGraph
->mForceShutDownReceived
, "Not in forced shutdown?");
1852 mGraph
->LifecycleStateRef() =
1853 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
;
1859 RefPtr
<MediaTrackGraphImpl
> mGraph
;
1862 class MediaTrackGraphStableStateRunnable
: public Runnable
{
1864 explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl
* aGraph
,
1866 : Runnable("MediaTrackGraphStableStateRunnable"),
1868 mSourceIsMTG(aSourceIsMTG
) {}
1869 NS_IMETHOD
Run() override
{
1870 TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
1872 mGraph
->RunInStableState(mSourceIsMTG
);
1878 RefPtr
<MediaTrackGraphImpl
> mGraph
;
1883 * Control messages forwarded from main thread to graph manager thread
1885 class CreateMessage
: public ControlMessage
{
1887 explicit CreateMessage(MediaTrack
* aTrack
) : ControlMessage(aTrack
) {}
1888 void Run() override
{
1889 TRACE("MTG::AddTrackGraphThread ControlMessage");
1890 mTrack
->GraphImpl()->AddTrackGraphThread(mTrack
);
1892 void RunDuringShutdown() override
{
1893 // Make sure to run this message during shutdown too, to make sure
1894 // that we balance the number of tracks registered with the graph
1895 // as they're destroyed during shutdown.
1902 void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG
) {
1903 MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
1905 nsTArray
<nsCOMPtr
<nsIRunnable
>> runnables
;
1906 // When we're doing a forced shutdown, pending control messages may be
1907 // run on the main thread via RunDuringShutdown. Those messages must
1908 // run without the graph monitor being held. So, we collect them here.
1909 nsTArray
<UniquePtr
<ControlMessageInterface
>>
1910 controlMessagesToRunDuringShutdown
;
1913 MonitorAutoLock
lock(mMonitor
);
1915 MOZ_ASSERT(mPostedRunInStableStateEvent
);
1916 mPostedRunInStableStateEvent
= false;
1919 // This should be kept in sync with the LifecycleState enum in
1920 // MediaTrackGraphImpl.h
1921 const char* LifecycleState_str
[] = {
1922 "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
1923 "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
1924 "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
1925 "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
1927 if (LifecycleStateRef() != LIFECYCLE_RUNNING
) {
1928 LOG(LogLevel::Debug
,
1929 ("%p: Running stable state callback. Current state: %s", this,
1930 LifecycleState_str
[LifecycleStateRef()]));
1933 runnables
= std::move(mUpdateRunnables
);
1934 for (uint32_t i
= 0; i
< mTrackUpdates
.Length(); ++i
) {
1935 TrackUpdate
* update
= &mTrackUpdates
[i
];
1936 if (update
->mTrack
) {
1937 ApplyTrackUpdate(update
);
1940 mTrackUpdates
.Clear();
1942 mMainThreadGraphTime
= mNextMainThreadGraphTime
;
1944 if (mCurrentTaskMessageQueue
.IsEmpty()) {
1945 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
&&
1947 // Complete shutdown. First, ensure that this graph is no longer used.
1948 // A new graph graph will be created if one is needed.
1949 // Asynchronously clean up old graph. We don't want to do this
1950 // synchronously because it spins the event loop waiting for threads
1951 // to shut down, and we don't want to do that in a stable state handler.
1952 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN
;
1953 LOG(LogLevel::Debug
,
1954 ("%p: Sending MediaTrackGraphShutDownRunnable", this));
1955 nsCOMPtr
<nsIRunnable
> event
= new MediaTrackGraphShutDownRunnable(this);
1956 mMainThread
->Dispatch(event
.forget());
1959 if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
) {
1960 MessageBlock
* block
= mBackMessageQueue
.AppendElement();
1961 block
->mMessages
= std::move(mCurrentTaskMessageQueue
);
1962 EnsureNextIteration();
1965 // If this MediaTrackGraph has entered regular (non-forced) shutdown it
1966 // is not able to process any more messages. Those messages being added to
1967 // the graph in the first place is an error.
1968 MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
1969 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
||
1970 mForceShutDownReceived
);
1973 if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED
) {
1974 // Start the driver now. We couldn't start it earlier because the graph
1975 // might exit immediately on finding it has no tracks. The first message
1976 // for a new graph must create a track. Ensure that his message runs on
1977 // the first iteration.
1978 MOZ_ASSERT(MessagesQueued());
1979 SwapMessageQueues();
1981 LOG(LogLevel::Debug
,
1982 ("%p: Starting a graph with a %s", this,
1983 CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
1984 : "SystemClockDriver"));
1985 LifecycleStateRef() = LIFECYCLE_RUNNING
;
1986 mGraphDriverRunning
= true;
1987 RefPtr
<GraphDriver
> driver
= CurrentDriver();
1989 // It's not safe to Shutdown() a thread from StableState, and
1990 // releasing this may shutdown a SystemClockDriver thread.
1991 // Proxy the release to outside of StableState.
1992 NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
1994 true); // always proxy
1997 if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
&&
1998 mForceShutDownReceived
) {
1999 // Defer calls to RunDuringShutdown() to happen while mMonitor is not
2001 for (uint32_t i
= 0; i
< mBackMessageQueue
.Length(); ++i
) {
2002 MessageBlock
& mb
= mBackMessageQueue
[i
];
2003 controlMessagesToRunDuringShutdown
.AppendElements(
2004 std::move(mb
.mMessages
));
2006 mBackMessageQueue
.Clear();
2007 MOZ_ASSERT(mCurrentTaskMessageQueue
.IsEmpty());
2008 // Stop MediaTrackGraph threads.
2009 LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN
;
2010 nsCOMPtr
<nsIRunnable
> event
= new MediaTrackGraphShutDownRunnable(this);
2011 mMainThread
->Dispatch(event
.forget());
2014 mGraphDriverRunning
= LifecycleStateRef() == LIFECYCLE_RUNNING
;
2017 // Make sure we get a new current time in the next event loop task
2018 if (!aSourceIsMTG
) {
2019 MOZ_ASSERT(mPostedRunInStableState
);
2020 mPostedRunInStableState
= false;
2023 for (uint32_t i
= 0; i
< controlMessagesToRunDuringShutdown
.Length(); ++i
) {
2024 controlMessagesToRunDuringShutdown
[i
]->RunDuringShutdown();
2028 mCanRunMessagesSynchronously
=
2029 !mGraphDriverRunning
&&
2030 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN
;
2033 for (uint32_t i
= 0; i
< runnables
.Length(); ++i
) {
2034 runnables
[i
]->Run();
2038 void MediaTrackGraphImpl::EnsureRunInStableState() {
2039 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2041 if (mPostedRunInStableState
) return;
2042 mPostedRunInStableState
= true;
2043 nsCOMPtr
<nsIRunnable
> event
=
2044 new MediaTrackGraphStableStateRunnable(this, false);
2045 nsContentUtils::RunInStableState(event
.forget());
2048 void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
2049 MOZ_ASSERT(OnGraphThread());
2050 mMonitor
.AssertCurrentThreadOwns();
2052 if (mPostedRunInStableStateEvent
) return;
2053 mPostedRunInStableStateEvent
= true;
2054 nsCOMPtr
<nsIRunnable
> event
=
2055 new MediaTrackGraphStableStateRunnable(this, true);
2056 mMainThread
->Dispatch(event
.forget());
2059 void MediaTrackGraphImpl::SignalMainThreadCleanup() {
2060 MOZ_ASSERT(mDriver
->OnThread());
2062 MonitorAutoLock
lock(mMonitor
);
2063 // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
2064 // graphs that have not started.
2065 MOZ_DIAGNOSTIC_ASSERT(mLifecycleState
<= LIFECYCLE_RUNNING
);
2066 LOG(LogLevel::Debug
,
2067 ("%p: MediaTrackGraph waiting for main thread cleanup", this));
2068 LifecycleStateRef() =
2069 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
;
2070 EnsureStableStateEventPosted();
2073 void MediaTrackGraphImpl::AppendMessage(
2074 UniquePtr
<ControlMessageInterface
> aMessage
) {
2075 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
2076 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount
> 0 || mMainThreadPortCount
> 0);
2078 if (!mGraphDriverRunning
&&
2079 LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
) {
2080 // The graph control loop is not running and main thread cleanup has
2081 // happened. From now on we can't append messages to
2082 // mCurrentTaskMessageQueue, because that will never be processed again, so
2083 // just RunDuringShutdown this message. This should only happen during
2084 // forced shutdown, or after a non-realtime graph has finished processing.
2086 MOZ_ASSERT(mCanRunMessagesSynchronously
);
2087 mCanRunMessagesSynchronously
= false;
2089 aMessage
->RunDuringShutdown();
2091 mCanRunMessagesSynchronously
= true;
2094 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
) {
2100 mCurrentTaskMessageQueue
.AppendElement(std::move(aMessage
));
2101 EnsureRunInStableState();
2104 void MediaTrackGraphImpl::Dispatch(already_AddRefed
<nsIRunnable
>&& aRunnable
) {
2105 mMainThread
->Dispatch(std::move(aRunnable
));
2108 MediaTrack::MediaTrack(TrackRate aSampleRate
, MediaSegment::Type aType
,
2109 MediaSegment
* aSegment
)
2110 : mSampleRate(aSampleRate
),
2116 mNotifiedEnded(false),
2117 mDisabledMode(DisabledTrackMode::ENABLED
),
2118 mStartBlocking(GRAPH_TIME_MAX
),
2120 mMainThreadCurrentTime(0),
2121 mMainThreadEnded(false),
2122 mEndedNotificationSent(false),
2123 mMainThreadDestroyed(false),
2125 MOZ_COUNT_CTOR(MediaTrack
);
2126 MOZ_ASSERT_IF(mSegment
, mSegment
->GetType() == aType
);
2129 MediaTrack::~MediaTrack() {
2130 MOZ_COUNT_DTOR(MediaTrack
);
2131 NS_ASSERTION(mMainThreadDestroyed
, "Should have been destroyed already");
2132 NS_ASSERTION(mMainThreadListeners
.IsEmpty(),
2133 "All main thread listeners should have been removed");
2136 size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const {
2140 // - mGraph - Not reported here
2141 // - mConsumers - elements
2143 // - mLastPlayedVideoFrame
2144 // - mTrackListeners - elements
2146 amount
+= mTrackListeners
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
2147 amount
+= mMainThreadListeners
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
2148 amount
+= mConsumers
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
2153 size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf
) const {
2154 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf
);
2157 void MediaTrack::IncrementSuspendCount() {
2159 if (mSuspendedCount
!= 1 || !mGraph
) {
2160 MOZ_ASSERT(mGraph
|| mConsumers
.IsEmpty());
2163 AssertOnGraphThreadOrNotRunning();
2164 auto* graph
= GraphImpl();
2165 for (uint32_t i
= 0; i
< mConsumers
.Length(); ++i
) {
2166 mConsumers
[i
]->Suspended();
2168 MOZ_ASSERT(graph
->mTracks
.Contains(this));
2169 graph
->mTracks
.RemoveElement(this);
2170 graph
->mSuspendedTracks
.AppendElement(this);
2171 graph
->SetTrackOrderDirty();
2174 void MediaTrack::DecrementSuspendCount() {
2175 MOZ_ASSERT(mSuspendedCount
> 0, "Suspend count underrun");
2177 if (mSuspendedCount
!= 0 || !mGraph
) {
2178 MOZ_ASSERT(mGraph
|| mConsumers
.IsEmpty());
2181 AssertOnGraphThreadOrNotRunning();
2182 auto* graph
= GraphImpl();
2183 for (uint32_t i
= 0; i
< mConsumers
.Length(); ++i
) {
2184 mConsumers
[i
]->Resumed();
2186 MOZ_ASSERT(graph
->mSuspendedTracks
.Contains(this));
2187 graph
->mSuspendedTracks
.RemoveElement(this);
2188 graph
->mTracks
.AppendElement(this);
2189 graph
->SetTrackOrderDirty();
2192 void ProcessedMediaTrack::DecrementSuspendCount() {
2193 mCycleMarker
= NOT_VISITED
;
2194 MediaTrack::DecrementSuspendCount();
2197 MediaTrackGraphImpl
* MediaTrack::GraphImpl() {
2198 return static_cast<MediaTrackGraphImpl
*>(mGraph
);
2201 const MediaTrackGraphImpl
* MediaTrack::GraphImpl() const {
2202 return static_cast<MediaTrackGraphImpl
*>(mGraph
);
2205 void MediaTrack::SetGraphImpl(MediaTrackGraphImpl
* aGraph
) {
2206 MOZ_ASSERT(!mGraph
, "Should only be called once");
2207 MOZ_ASSERT(mSampleRate
== aGraph
->GraphRate());
2211 void MediaTrack::SetGraphImpl(MediaTrackGraph
* aGraph
) {
2212 MediaTrackGraphImpl
* graph
= static_cast<MediaTrackGraphImpl
*>(aGraph
);
2213 SetGraphImpl(graph
);
2216 TrackTime
MediaTrack::GraphTimeToTrackTime(GraphTime aTime
) const {
2217 NS_ASSERTION(mStartBlocking
== GraphImpl()->mStateComputedTime
||
2218 aTime
<= mStartBlocking
,
2219 "Incorrectly ignoring blocking!");
2220 return aTime
- mStartTime
;
2223 GraphTime
MediaTrack::TrackTimeToGraphTime(TrackTime aTime
) const {
2224 NS_ASSERTION(mStartBlocking
== GraphImpl()->mStateComputedTime
||
2225 aTime
+ mStartTime
<= mStartBlocking
,
2226 "Incorrectly ignoring blocking!");
2227 return aTime
+ mStartTime
;
2230 TrackTime
MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime
) const {
2231 return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime
);
2234 void MediaTrack::RemoveAllResourcesAndListenersImpl() {
2235 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2237 for (auto& l
: mTrackListeners
.Clone()) {
2238 l
->NotifyRemoved(Graph());
2240 mTrackListeners
.Clear();
2242 RemoveAllDirectListenersImpl();
2249 void MediaTrack::DestroyImpl() {
2250 for (int32_t i
= mConsumers
.Length() - 1; i
>= 0; --i
) {
2251 mConsumers
[i
]->Disconnect();
2259 void MediaTrack::Destroy() {
2260 // Keep this track alive until we leave this method
2261 RefPtr
<MediaTrack
> kungFuDeathGrip
= this;
2262 // Keep a reference to the graph, since Message might RunDuringShutdown()
2263 // synchronously and make GraphImpl() invalid.
2264 RefPtr
<MediaTrackGraphImpl
> graph
= GraphImpl();
2266 QueueControlOrShutdownMessage(
2267 [self
= RefPtr
{this}, this](IsInShutdown aInShutdown
) {
2268 if (aInShutdown
== IsInShutdown::No
) {
2269 OnGraphThreadDone();
2271 TRACE("MediaTrack::Destroy ControlMessage");
2272 RemoveAllResourcesAndListenersImpl();
2273 auto* graph
= GraphImpl();
2275 graph
->RemoveTrackGraphThread(this);
2277 graph
->RemoveTrack(this);
2278 // Message::RunDuringShutdown may have removed this track from the graph,
2279 // but our kungFuDeathGrip above will have kept this track alive if
2281 mMainThreadDestroyed
= true;
2284 TrackTime
MediaTrack::GetEnd() const {
2285 return mSegment
? mSegment
->GetDuration() : 0;
2288 void MediaTrack::AddAudioOutput(void* aKey
, const AudioDeviceInfo
* aSink
) {
2289 MOZ_ASSERT(NS_IsMainThread());
2290 AudioDeviceID deviceID
= nullptr;
2291 TrackRate preferredSampleRate
= 0;
2293 deviceID
= aSink
->DeviceID();
2294 preferredSampleRate
= static_cast<TrackRate
>(aSink
->DefaultRate());
2296 AddAudioOutput(aKey
, deviceID
, preferredSampleRate
);
2299 void MediaTrack::AddAudioOutput(void* aKey
, CubebUtils::AudioDeviceID aDeviceID
,
2300 TrackRate aPreferredSampleRate
) {
2301 MOZ_ASSERT(NS_IsMainThread());
2302 if (mMainThreadDestroyed
) {
2305 LOG(LogLevel::Info
, ("MediaTrack %p adding AudioOutput", this));
2306 GraphImpl()->RegisterAudioOutput(this, aKey
, aDeviceID
, aPreferredSampleRate
);
2309 void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack
* aTrack
, void* aKey
,
2311 MOZ_ASSERT(NS_IsMainThread());
2312 for (auto& params
: mAudioOutputParams
) {
2313 if (params
.mKey
== aKey
&& aTrack
== params
.mTrack
) {
2314 params
.mVolume
= aVolume
;
2315 UpdateAudioOutput(aTrack
, params
.mDeviceID
);
2319 MOZ_CRASH("Audio output key not found when setting the volume.");
2322 void MediaTrack::SetAudioOutputVolume(void* aKey
, float aVolume
) {
2323 if (mMainThreadDestroyed
) {
2326 GraphImpl()->SetAudioOutputVolume(this, aKey
, aVolume
);
2329 void MediaTrack::RemoveAudioOutput(void* aKey
) {
2330 MOZ_ASSERT(NS_IsMainThread());
2331 if (mMainThreadDestroyed
) {
2334 LOG(LogLevel::Info
, ("MediaTrack %p removing AudioOutput", this));
2335 GraphImpl()->UnregisterAudioOutput(this, aKey
);
2338 void MediaTrackGraphImpl::RegisterAudioOutput(
2339 MediaTrack
* aTrack
, void* aKey
, CubebUtils::AudioDeviceID aDeviceID
,
2340 TrackRate aPreferredSampleRate
) {
2341 MOZ_ASSERT(NS_IsMainThread());
2342 MOZ_ASSERT(!mAudioOutputParams
.Contains(TrackAndKey
{aTrack
, aKey
}));
2344 IncrementOutputDeviceRefCnt(aDeviceID
, aPreferredSampleRate
);
2346 mAudioOutputParams
.EmplaceBack(
2347 TrackKeyDeviceAndVolume
{aTrack
, aKey
, aDeviceID
, 1.f
});
2349 UpdateAudioOutput(aTrack
, aDeviceID
);
2352 void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack
* aTrack
,
2354 MOZ_ASSERT(NS_IsMainThread());
2356 size_t index
= mAudioOutputParams
.IndexOf(TrackAndKey
{aTrack
, aKey
});
2357 MOZ_ASSERT(index
!= mAudioOutputParams
.NoIndex
);
2358 AudioDeviceID deviceID
= mAudioOutputParams
[index
].mDeviceID
;
2359 mAudioOutputParams
.UnorderedRemoveElementAt(index
);
2361 UpdateAudioOutput(aTrack
, deviceID
);
2363 DecrementOutputDeviceRefCnt(deviceID
);
2366 void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack
* aTrack
,
2367 AudioDeviceID aDeviceID
) {
2368 MOZ_ASSERT(NS_IsMainThread());
2369 MOZ_ASSERT(!aTrack
->IsDestroyed());
2373 for (const auto& params
: mAudioOutputParams
) {
2374 if (params
.mTrack
== aTrack
&& params
.mDeviceID
== aDeviceID
) {
2375 volume
+= params
.mVolume
;
2380 QueueControlMessageWithNoShutdown(
2381 // track has a strong reference to this.
2382 [track
= RefPtr
{aTrack
}, aDeviceID
, volume
, found
] {
2383 TRACE("MediaTrack::UpdateAudioOutput ControlMessage");
2384 MediaTrackGraphImpl
* graph
= track
->GraphImpl();
2385 auto& outputDevicesRef
= graph
->mOutputDevices
;
2386 size_t deviceIndex
= outputDevicesRef
.IndexOf(aDeviceID
);
2387 MOZ_ASSERT(deviceIndex
!= outputDevicesRef
.NoIndex
);
2388 auto& deviceOutputsRef
= outputDevicesRef
[deviceIndex
].mTrackOutputs
;
2390 for (auto& outputRef
: deviceOutputsRef
) {
2391 if (outputRef
.mTrack
== track
) {
2392 outputRef
.mVolume
= volume
;
2396 deviceOutputsRef
.EmplaceBack(TrackAndVolume
{track
, volume
});
2398 DebugOnly
<bool> removed
= deviceOutputsRef
.RemoveElement(track
);
2399 MOZ_ASSERT(removed
);
2400 // mOutputDevices[0] is retained for AudioCallbackDriver output even
2401 // when no tracks have audio outputs.
2402 if (deviceIndex
!= 0 && deviceOutputsRef
.IsEmpty()) {
2403 // The device is no longer in use.
2404 outputDevicesRef
.UnorderedRemoveElementAt(deviceIndex
);
2410 void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt(
2411 AudioDeviceID aDeviceID
, TrackRate aPreferredSampleRate
) {
2412 MOZ_ASSERT(NS_IsMainThread());
2414 for (auto& elementRef
: mOutputDeviceRefCnts
) {
2415 if (elementRef
.mDeviceID
== aDeviceID
) {
2416 ++elementRef
.mRefCnt
;
2420 MOZ_ASSERT(aDeviceID
!= mPrimaryOutputDeviceID
,
2421 "mOutputDeviceRefCnts should always have the primary device");
2422 // Need to add an output device.
2423 // Output via another graph for this device.
2424 // This sample rate is not exposed to content.
2425 TrackRate sampleRate
=
2426 aPreferredSampleRate
!= 0
2427 ? aPreferredSampleRate
2428 : static_cast<TrackRate
>(CubebUtils::PreferredSampleRate(
2429 /*aShouldResistFingerprinting*/ false));
2430 MediaTrackGraph
* newGraph
= MediaTrackGraphImpl::GetInstance(
2431 MediaTrackGraph::AUDIO_THREAD_DRIVER
, mWindowID
, sampleRate
, aDeviceID
,
2432 GetMainThreadSerialEventTarget());
2433 // CreateCrossGraphReceiver wants the sample rate of this graph.
2434 RefPtr receiver
= newGraph
->CreateCrossGraphReceiver(mSampleRate
);
2435 receiver
->AddAudioOutput(nullptr, aDeviceID
, sampleRate
);
2436 mOutputDeviceRefCnts
.EmplaceBack(
2437 DeviceReceiverAndCount
{aDeviceID
, receiver
, 1});
2439 QueueControlMessageWithNoShutdown([self
= RefPtr
{this}, this, aDeviceID
,
2440 receiver
= std::move(receiver
)]() mutable {
2441 TRACE("MediaTrackGraph add output device ControlMessage");
2442 MOZ_ASSERT(!mOutputDevices
.Contains(aDeviceID
));
2443 mOutputDevices
.EmplaceBack(
2444 OutputDeviceEntry
{aDeviceID
, std::move(receiver
)});
2448 void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID
) {
2449 MOZ_ASSERT(NS_IsMainThread());
2451 size_t index
= mOutputDeviceRefCnts
.IndexOf(aDeviceID
);
2452 MOZ_ASSERT(index
!= mOutputDeviceRefCnts
.NoIndex
);
2453 // mOutputDeviceRefCnts[0] is retained for consistency with
2454 // mOutputDevices[0], which is retained for AudioCallbackDriver output even
2455 // when no tracks have audio outputs.
2456 if (--mOutputDeviceRefCnts
[index
].mRefCnt
== 0 && index
!= 0) {
2457 mOutputDeviceRefCnts
[index
].mReceiver
->Destroy();
2458 mOutputDeviceRefCnts
.UnorderedRemoveElementAt(index
);
2462 void MediaTrack::Suspend() {
2463 // This can happen if this method has been called asynchronously, and the
2464 // track has been destroyed since then.
2465 if (mMainThreadDestroyed
) {
2468 QueueControlMessageWithNoShutdown([self
= RefPtr
{this}, this] {
2469 TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
2470 IncrementSuspendCount();
2474 void MediaTrack::Resume() {
2475 // This can happen if this method has been called asynchronously, and the
2476 // track has been destroyed since then.
2477 if (mMainThreadDestroyed
) {
2480 QueueControlMessageWithNoShutdown([self
= RefPtr
{this}, this] {
2481 TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
2482 DecrementSuspendCount();
2486 void MediaTrack::AddListenerImpl(
2487 already_AddRefed
<MediaTrackListener
> aListener
) {
2488 RefPtr
<MediaTrackListener
> l(aListener
);
2489 mTrackListeners
.AppendElement(std::move(l
));
2491 PrincipalHandle lastPrincipalHandle
= mSegment
->GetLastPrincipalHandle();
2492 mTrackListeners
.LastElement()->NotifyPrincipalHandleChanged(
2493 Graph(), lastPrincipalHandle
);
2494 if (mNotifiedEnded
) {
2495 mTrackListeners
.LastElement()->NotifyEnded(Graph());
2497 if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK
) {
2498 mTrackListeners
.LastElement()->NotifyEnabledStateChanged(Graph(), false);
2502 void MediaTrack::AddListener(MediaTrackListener
* aListener
) {
2503 MOZ_ASSERT(mSegment
, "Segment-less tracks do not support listeners");
2504 if (mMainThreadDestroyed
) {
2507 QueueControlMessageWithNoShutdown(
2508 [self
= RefPtr
{this}, this, listener
= RefPtr
{aListener
}]() mutable {
2509 TRACE("MediaTrack::AddListenerImpl ControlMessage");
2510 AddListenerImpl(listener
.forget());
2514 void MediaTrack::RemoveListenerImpl(MediaTrackListener
* aListener
) {
2515 for (size_t i
= 0; i
< mTrackListeners
.Length(); ++i
) {
2516 if (mTrackListeners
[i
] == aListener
) {
2517 mTrackListeners
[i
]->NotifyRemoved(Graph());
2518 mTrackListeners
.RemoveElementAt(i
);
2524 RefPtr
<GenericPromise
> MediaTrack::RemoveListener(
2525 MediaTrackListener
* aListener
) {
2526 MozPromiseHolder
<GenericPromise
> promiseHolder
;
2527 RefPtr
<GenericPromise
> p
= promiseHolder
.Ensure(__func__
);
2528 if (mMainThreadDestroyed
) {
2529 promiseHolder
.Reject(NS_ERROR_FAILURE
, __func__
);
2532 QueueControlOrShutdownMessage(
2533 [self
= RefPtr
{this}, this, listener
= RefPtr
{aListener
},
2534 promiseHolder
= std::move(promiseHolder
)](IsInShutdown
) mutable {
2535 TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
2536 // During shutdown we still want the listener's NotifyRemoved to be
2537 // called, since not doing that might block shutdown of other modules.
2538 RemoveListenerImpl(listener
);
2539 promiseHolder
.Resolve(true, __func__
);
2544 void MediaTrack::AddDirectListenerImpl(
2545 already_AddRefed
<DirectMediaTrackListener
> aListener
) {
2546 AssertOnGraphThread();
2547 // Base implementation, for tracks that don't support direct track listeners.
2548 RefPtr
<DirectMediaTrackListener
> listener
= aListener
;
2549 listener
->NotifyDirectListenerInstalled(
2550 DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED
);
2553 void MediaTrack::AddDirectListener(DirectMediaTrackListener
* aListener
) {
2554 if (mMainThreadDestroyed
) {
2557 QueueControlMessageWithNoShutdown(
2558 [self
= RefPtr
{this}, this, listener
= RefPtr
{aListener
}]() mutable {
2559 TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
2560 AddDirectListenerImpl(listener
.forget());
2564 void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener
* aListener
) {
2565 // Base implementation, the listener was never added so nothing to do.
2568 void MediaTrack::RemoveDirectListener(DirectMediaTrackListener
* aListener
) {
2569 if (mMainThreadDestroyed
) {
2572 QueueControlOrShutdownMessage(
2573 [self
= RefPtr
{this}, this, listener
= RefPtr
{aListener
}](IsInShutdown
) {
2574 TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
2575 // During shutdown we still want the listener's
2576 // NotifyDirectListenerUninstalled to be called, since not doing that
2577 // might block shutdown of other modules.
2578 RemoveDirectListenerImpl(listener
);
2582 void MediaTrack::RunAfterPendingUpdates(
2583 already_AddRefed
<nsIRunnable
> aRunnable
) {
2584 MOZ_ASSERT(NS_IsMainThread());
2585 if (mMainThreadDestroyed
) {
2588 QueueControlOrShutdownMessage(
2589 [self
= RefPtr
{this}, this,
2590 runnable
= nsCOMPtr
{aRunnable
}](IsInShutdown aInShutdown
) mutable {
2591 TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
2592 if (aInShutdown
== IsInShutdown::No
) {
2593 Graph()->DispatchToMainThreadStableState(runnable
.forget());
2595 // Don't run mRunnable now as it may call AppendMessage() which would
2596 // assume that there are no remaining
2597 // controlMessagesToRunDuringShutdown.
2598 MOZ_ASSERT(NS_IsMainThread());
2599 GraphImpl()->Dispatch(runnable
.forget());
2604 void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode
) {
2605 AssertOnGraphThread();
2606 MOZ_DIAGNOSTIC_ASSERT(
2607 aMode
== DisabledTrackMode::ENABLED
||
2608 mDisabledMode
== DisabledTrackMode::ENABLED
,
2609 "Changing disabled track mode for a track is not allowed");
2610 DisabledTrackMode oldMode
= CombinedDisabledMode();
2611 mDisabledMode
= aMode
;
2612 NotifyIfDisabledModeChangedFrom(oldMode
);
2615 void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode
) {
2616 if (mMainThreadDestroyed
) {
2619 QueueControlMessageWithNoShutdown([self
= RefPtr
{this}, this, aMode
]() {
2620 TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
2621 SetDisabledTrackModeImpl(aMode
);
2625 void MediaTrack::ApplyTrackDisabling(MediaSegment
* aSegment
,
2626 MediaSegment
* aRawSegment
) {
2627 AssertOnGraphThread();
2628 mozilla::ApplyTrackDisabling(mDisabledMode
, aSegment
, aRawSegment
);
2631 void MediaTrack::AddMainThreadListener(
2632 MainThreadMediaTrackListener
* aListener
) {
2633 MOZ_ASSERT(NS_IsMainThread());
2634 MOZ_ASSERT(aListener
);
2635 MOZ_ASSERT(!mMainThreadListeners
.Contains(aListener
));
2637 mMainThreadListeners
.AppendElement(aListener
);
2639 // If it is not yet time to send the notification, then exit here.
2640 if (!mEndedNotificationSent
) {
2644 class NotifyRunnable final
: public Runnable
{
2646 explicit NotifyRunnable(MediaTrack
* aTrack
)
2647 : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack
) {}
2649 NS_IMETHOD
Run() override
{
2650 TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
2651 MOZ_ASSERT(NS_IsMainThread());
2652 mTrack
->NotifyMainThreadListeners();
2657 ~NotifyRunnable() = default;
2659 RefPtr
<MediaTrack
> mTrack
;
2662 nsCOMPtr
<nsIRunnable
> runnable
= new NotifyRunnable(this);
2663 GraphImpl()->Dispatch(runnable
.forget());
2666 void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime
,
2667 GraphTime aBlockedTime
) {
2668 mStartTime
+= aBlockedTime
;
2671 // No data to be forgotten.
2675 TrackTime time
= aCurrentTime
- mStartTime
;
2676 // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
2677 // spend too much time pruning segments.
2678 const TrackTime minChunkSize
= mSampleRate
* 50 / 1000;
2679 if (time
< mForgottenTime
+ minChunkSize
) {
2683 mForgottenTime
= std::min(GetEnd() - 1, time
);
2684 mSegment
->ForgetUpTo(mForgottenTime
);
2687 void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode
) {
2688 DisabledTrackMode mode
= CombinedDisabledMode();
2689 if (aOldMode
== mode
) {
2693 for (const auto& listener
: mTrackListeners
) {
2694 listener
->NotifyEnabledStateChanged(
2695 Graph(), mode
!= DisabledTrackMode::SILENCE_BLACK
);
2698 for (const auto& c
: mConsumers
) {
2699 if (c
->GetDestination()) {
2700 c
->GetDestination()->OnInputDisabledModeChanged(mode
);
2705 void MediaTrack::QueueMessage(UniquePtr
<ControlMessageInterface
> aMessage
) {
2706 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
2707 MOZ_RELEASE_ASSERT(!IsDestroyed());
2708 GraphImpl()->AppendMessage(std::move(aMessage
));
2711 void MediaTrack::RunMessageAfterProcessing(
2712 UniquePtr
<ControlMessageInterface
> aMessage
) {
2713 AssertOnGraphThread();
2714 GraphImpl()->RunMessageAfterProcessing(std::move(aMessage
));
2717 SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType
,
2718 TrackRate aSampleRate
)
2719 : MediaTrack(aSampleRate
, aType
,
2720 aType
== MediaSegment::AUDIO
2721 ? static_cast<MediaSegment
*>(new AudioSegment())
2722 : static_cast<MediaSegment
*>(new VideoSegment())),
2723 mMutex("mozilla::media::SourceMediaTrack") {
2724 mUpdateTrack
= MakeUnique
<TrackData
>();
2725 mUpdateTrack
->mInputRate
= aSampleRate
;
2726 mUpdateTrack
->mResamplerChannelCount
= 0;
2727 mUpdateTrack
->mData
= UniquePtr
<MediaSegment
>(mSegment
->CreateEmptyClone());
2728 mUpdateTrack
->mEnded
= false;
2729 mUpdateTrack
->mPullingEnabled
= false;
2730 mUpdateTrack
->mGraphThreadDone
= false;
2733 void SourceMediaTrack::DestroyImpl() {
2734 GraphImpl()->AssertOnGraphThreadOrNotRunning();
2735 for (int32_t i
= mConsumers
.Length() - 1; i
>= 0; --i
) {
2736 // Disconnect before we come under mMutex's lock since it can call back
2737 // through RemoveDirectListenerImpl() and deadlock.
2738 mConsumers
[i
]->Disconnect();
2741 // Hold mMutex while mGraph is reset so that other threads holding mMutex
2742 // can null-check know that the graph will not destroyed.
2743 MutexAutoLock
lock(mMutex
);
2744 mUpdateTrack
= nullptr;
2745 MediaTrack::DestroyImpl();
2748 void SourceMediaTrack::SetPullingEnabled(bool aEnabled
) {
2749 class Message
: public ControlMessage
{
2751 Message(SourceMediaTrack
* aTrack
, bool aEnabled
)
2752 : ControlMessage(nullptr), mTrack(aTrack
), mEnabled(aEnabled
) {}
2753 void Run() override
{
2754 TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
2755 MutexAutoLock
lock(mTrack
->mMutex
);
2756 if (!mTrack
->mUpdateTrack
) {
2757 // We can't enable pulling for a track that has ended. We ignore
2758 // this if we're disabling pulling, since shutdown sequences are
2759 // complex. If there's truly an issue we'll have issues enabling anyway.
2760 MOZ_ASSERT_IF(mEnabled
, mTrack
->mEnded
);
2763 MOZ_ASSERT(mTrack
->mType
== MediaSegment::AUDIO
,
2764 "Pulling is not allowed for video");
2765 mTrack
->mUpdateTrack
->mPullingEnabled
= mEnabled
;
2767 SourceMediaTrack
* mTrack
;
2770 GraphImpl()->AppendMessage(MakeUnique
<Message
>(this, aEnabled
));
2773 bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime
) {
2774 TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
2781 MutexAutoLock
lock(mMutex
);
2782 if (mUpdateTrack
->mEnded
) {
2785 if (!mUpdateTrack
->mPullingEnabled
) {
2788 // Compute how much track time we'll need assuming we don't block
2789 // the track at all.
2790 t
= GraphTimeToTrackTime(aDesiredUpToTime
);
2791 current
= GetEnd() + mUpdateTrack
->mData
->GetDuration();
2796 LOG(LogLevel::Verbose
, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
2797 GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t
),
2798 GraphImpl()->MediaTimeToSeconds(current
)));
2799 for (auto& l
: mTrackListeners
) {
2800 l
->NotifyPull(Graph(), current
, t
);
2806 * This moves chunks from aIn to aOut. For audio this is simple. For video
2807 * we carry durations over if present, or extend up to aDesiredUpToTime if not.
2809 * We also handle "resetters" from captured media elements. This type of source
2810 * pushes future frames into the track, and should it need to remove some, e.g.,
2811 * because of a seek or pause, it tells us by letting time go backwards. Without
2812 * this, tracks would be live for too long after a seek or pause.
2814 static void MoveToSegment(SourceMediaTrack
* aTrack
, MediaSegment
* aIn
,
2815 MediaSegment
* aOut
, TrackTime aCurrentTime
,
2816 TrackTime aDesiredUpToTime
)
2817 MOZ_REQUIRES(aTrack
->GetMutex()) {
2818 MOZ_ASSERT(aIn
->GetType() == aOut
->GetType());
2819 MOZ_ASSERT(aOut
->GetDuration() >= aCurrentTime
);
2820 MOZ_ASSERT(aDesiredUpToTime
>= aCurrentTime
);
2821 if (aIn
->GetType() == MediaSegment::AUDIO
) {
2822 AudioSegment
* in
= static_cast<AudioSegment
*>(aIn
);
2823 AudioSegment
* out
= static_cast<AudioSegment
*>(aOut
);
2824 TrackTime desiredDurationToMove
= aDesiredUpToTime
- aCurrentTime
;
2825 TrackTime end
= std::min(in
->GetDuration(), desiredDurationToMove
);
2827 out
->AppendSlice(*in
, 0, end
);
2828 in
->RemoveLeading(end
);
2830 aTrack
->GetMutex().AssertCurrentThreadOwns();
2831 out
->ApplyVolume(aTrack
->GetVolumeLocked());
2833 VideoSegment
* in
= static_cast<VideoSegment
*>(aIn
);
2834 VideoSegment
* out
= static_cast<VideoSegment
*>(aOut
);
2835 for (VideoSegment::ConstChunkIterator
c(*in
); !c
.IsEnded(); c
.Next()) {
2836 MOZ_ASSERT(!c
->mTimeStamp
.IsNull());
2837 VideoChunk
* last
= out
->GetLastChunk();
2838 if (!last
|| last
->mTimeStamp
.IsNull()) {
2839 // This is the first frame, or the last frame pushed to `out` has been
2840 // all consumed. Just append and we deal with its duration later.
2841 out
->AppendFrame(do_AddRef(c
->mFrame
.GetImage()),
2842 c
->mFrame
.GetIntrinsicSize(),
2843 c
->mFrame
.GetPrincipalHandle(),
2844 c
->mFrame
.GetForceBlack(), c
->mTimeStamp
);
2845 if (c
->GetDuration() > 0) {
2846 out
->ExtendLastFrameBy(c
->GetDuration());
2851 // We now know when this frame starts, aka when the last frame ends.
2853 if (c
->mTimeStamp
< last
->mTimeStamp
) {
2854 // Time is going backwards. This is a resetting frame from
2855 // DecodedStream. Clear everything up to currentTime.
2857 out
->AppendNullData(aCurrentTime
);
2860 // Append the current frame (will have duration 0).
2861 out
->AppendFrame(do_AddRef(c
->mFrame
.GetImage()),
2862 c
->mFrame
.GetIntrinsicSize(),
2863 c
->mFrame
.GetPrincipalHandle(),
2864 c
->mFrame
.GetForceBlack(), c
->mTimeStamp
);
2865 if (c
->GetDuration() > 0) {
2866 out
->ExtendLastFrameBy(c
->GetDuration());
2869 if (out
->GetDuration() < aDesiredUpToTime
) {
2870 out
->ExtendLastFrameBy(aDesiredUpToTime
- out
->GetDuration());
2873 MOZ_ASSERT(aIn
->GetDuration() == 0, "aIn must be consumed");
2877 void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime
,
2878 GraphTime aDesiredUpToTime
) {
2879 MutexAutoLock
lock(mMutex
);
2881 if (!mUpdateTrack
) {
2886 TrackTime trackCurrentTime
= GraphTimeToTrackTime(aCurrentTime
);
2888 ApplyTrackDisabling(mUpdateTrack
->mData
.get());
2890 if (!mUpdateTrack
->mData
->IsEmpty()) {
2891 for (const auto& l
: mTrackListeners
) {
2892 l
->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack
->mData
);
2895 TrackTime trackDesiredUpToTime
= GraphTimeToTrackTime(aDesiredUpToTime
);
2896 TrackTime endTime
= trackDesiredUpToTime
;
2897 if (mUpdateTrack
->mEnded
) {
2898 endTime
= std::min(trackDesiredUpToTime
,
2899 GetEnd() + mUpdateTrack
->mData
->GetDuration());
2901 LOG(LogLevel::Verbose
,
2902 ("%p: SourceMediaTrack %p advancing end from %" PRId64
" to %" PRId64
,
2903 GraphImpl(), this, int64_t(trackCurrentTime
), int64_t(endTime
)));
2904 MoveToSegment(this, mUpdateTrack
->mData
.get(), mSegment
.get(),
2905 trackCurrentTime
, endTime
);
2906 if (mUpdateTrack
->mEnded
&& GetEnd() < trackDesiredUpToTime
) {
2908 mUpdateTrack
= nullptr;
2912 void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment
* aSegment
) {
2913 mMutex
.AssertCurrentThreadOwns();
2914 if (aSegment
->GetType() != MediaSegment::AUDIO
||
2915 mUpdateTrack
->mInputRate
== GraphImpl()->GraphRate()) {
2918 AudioSegment
* segment
= static_cast<AudioSegment
*>(aSegment
);
2919 segment
->ResampleChunks(mUpdateTrack
->mResampler
,
2920 &mUpdateTrack
->mResamplerChannelCount
,
2921 mUpdateTrack
->mInputRate
, GraphImpl()->GraphRate());
2924 void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
2925 GraphTime aCurrentTime
, GraphTime aBlockedTime
) {
2926 MutexAutoLock
lock(mMutex
);
2927 MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime
, aBlockedTime
);
2930 void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate
) {
2931 MutexAutoLock
lock(mMutex
);
2932 if (!mUpdateTrack
) {
2935 MOZ_DIAGNOSTIC_ASSERT(mSegment
->GetType() == MediaSegment::AUDIO
);
2936 // Set the new input rate and reset the resampler.
2937 mUpdateTrack
->mInputRate
= aRate
;
2938 mUpdateTrack
->mResampler
.own(nullptr);
2939 mUpdateTrack
->mResamplerChannelCount
= 0;
2942 TrackTime
SourceMediaTrack::AppendData(MediaSegment
* aSegment
,
2943 MediaSegment
* aRawSegment
) {
2944 MutexAutoLock
lock(mMutex
);
2945 MOZ_DIAGNOSTIC_ASSERT(aSegment
->GetType() == mType
);
2946 TrackTime appended
= 0;
2947 if (!mUpdateTrack
|| mUpdateTrack
->mEnded
|| mUpdateTrack
->mGraphThreadDone
) {
2952 // Data goes into mData, and on the next iteration of the MTG moves
2953 // into the track's segment after NotifyQueuedTrackChanges(). This adds
2954 // 0-10ms of delay before data gets to direct listeners.
2955 // Indirect listeners (via subsequent TrackUnion nodes) are synced to
2956 // playout time, and so can be delayed by buffering.
2958 // Apply track disabling before notifying any consumers directly
2959 // or inserting into the graph
2960 mozilla::ApplyTrackDisabling(mDirectDisabledMode
, aSegment
, aRawSegment
);
2962 ResampleAudioToGraphSampleRate(aSegment
);
2964 // Must notify first, since AppendFrom() will empty out aSegment
2965 NotifyDirectConsumers(aRawSegment
? aRawSegment
: aSegment
);
2966 appended
= aSegment
->GetDuration();
2967 mUpdateTrack
->mData
->AppendFrom(aSegment
); // note: aSegment is now dead
2969 auto graph
= GraphImpl();
2970 MonitorAutoLock
lock(graph
->GetMonitor());
2971 if (graph
->CurrentDriver()) { // graph has not completed forced shutdown
2972 graph
->EnsureNextIteration();
2979 TrackTime
SourceMediaTrack::ClearFutureData() {
2980 MutexAutoLock
lock(mMutex
);
2981 auto graph
= GraphImpl();
2982 if (!mUpdateTrack
|| !graph
) {
2986 TrackTime duration
= mUpdateTrack
->mData
->GetDuration();
2987 mUpdateTrack
->mData
->Clear();
2991 void SourceMediaTrack::NotifyDirectConsumers(MediaSegment
* aSegment
) {
2992 mMutex
.AssertCurrentThreadOwns();
2994 for (const auto& l
: mDirectTrackListeners
) {
2995 TrackTime offset
= 0; // FIX! need a separate TrackTime.... or the end of
2996 // the internal buffer
2997 l
->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset
,
3002 void SourceMediaTrack::AddDirectListenerImpl(
3003 already_AddRefed
<DirectMediaTrackListener
> aListener
) {
3004 AssertOnGraphThread();
3005 MutexAutoLock
lock(mMutex
);
3007 RefPtr
<DirectMediaTrackListener
> listener
= aListener
;
3008 LOG(LogLevel::Debug
,
3009 ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
3010 listener
.get(), this));
3012 MOZ_ASSERT(mType
== MediaSegment::VIDEO
);
3013 for (const auto& l
: mDirectTrackListeners
) {
3014 if (l
== listener
) {
3015 listener
->NotifyDirectListenerInstalled(
3016 DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS
);
3021 mDirectTrackListeners
.AppendElement(listener
);
3023 LOG(LogLevel::Debug
,
3024 ("%p: Added direct track listener %p", GraphImpl(), listener
.get()));
3025 listener
->NotifyDirectListenerInstalled(
3026 DirectMediaTrackListener::InstallationResult::SUCCESS
);
3028 if (mDisabledMode
!= DisabledTrackMode::ENABLED
) {
3029 listener
->IncreaseDisabled(mDisabledMode
);
3036 // Pass buffered data to the listener
3037 VideoSegment bufferedData
;
3038 size_t videoFrames
= 0;
3039 VideoSegment
& segment
= *GetData
<VideoSegment
>();
3040 for (VideoSegment::ConstChunkIterator
iter(segment
); !iter
.IsEnded();
3042 if (iter
->mTimeStamp
.IsNull()) {
3043 // No timestamp means this is only for the graph's internal book-keeping,
3044 // denoting a late start of the track.
3048 bufferedData
.AppendFrame(do_AddRef(iter
->mFrame
.GetImage()),
3049 iter
->mFrame
.GetIntrinsicSize(),
3050 iter
->mFrame
.GetPrincipalHandle(),
3051 iter
->mFrame
.GetForceBlack(), iter
->mTimeStamp
);
3054 VideoSegment
& video
= static_cast<VideoSegment
&>(*mUpdateTrack
->mData
);
3055 for (VideoSegment::ConstChunkIterator
iter(video
); !iter
.IsEnded();
3058 MOZ_ASSERT(!iter
->mTimeStamp
.IsNull());
3059 bufferedData
.AppendFrame(do_AddRef(iter
->mFrame
.GetImage()),
3060 iter
->mFrame
.GetIntrinsicSize(),
3061 iter
->mFrame
.GetPrincipalHandle(),
3062 iter
->mFrame
.GetForceBlack(), iter
->mTimeStamp
);
3066 ("%p: Notifying direct listener %p of %zu video frames and duration "
3068 GraphImpl(), listener
.get(), videoFrames
, bufferedData
.GetDuration()));
3069 listener
->NotifyRealtimeTrackData(Graph(), 0, bufferedData
);
3072 void SourceMediaTrack::RemoveDirectListenerImpl(
3073 DirectMediaTrackListener
* aListener
) {
3074 mGraph
->AssertOnGraphThreadOrNotRunning();
3075 MutexAutoLock
lock(mMutex
);
3076 for (int32_t i
= mDirectTrackListeners
.Length() - 1; i
>= 0; --i
) {
3077 const RefPtr
<DirectMediaTrackListener
>& l
= mDirectTrackListeners
[i
];
3078 if (l
== aListener
) {
3079 if (mDisabledMode
!= DisabledTrackMode::ENABLED
) {
3080 aListener
->DecreaseDisabled(mDisabledMode
);
3082 aListener
->NotifyDirectListenerUninstalled();
3083 mDirectTrackListeners
.RemoveElementAt(i
);
3088 void SourceMediaTrack::End() {
3089 MutexAutoLock
lock(mMutex
);
3090 if (!mUpdateTrack
) {
3094 mUpdateTrack
->mEnded
= true;
3095 if (auto graph
= GraphImpl()) {
3096 MonitorAutoLock
lock(graph
->GetMonitor());
3097 if (graph
->CurrentDriver()) { // graph has not completed forced shutdown
3098 graph
->EnsureNextIteration();
3103 void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode
) {
3104 AssertOnGraphThread();
3106 MutexAutoLock
lock(mMutex
);
3107 const DisabledTrackMode oldMode
= mDirectDisabledMode
;
3108 const bool oldEnabled
= oldMode
== DisabledTrackMode::ENABLED
;
3109 const bool enabled
= aMode
== DisabledTrackMode::ENABLED
;
3110 mDirectDisabledMode
= aMode
;
3111 for (const auto& l
: mDirectTrackListeners
) {
3112 if (!oldEnabled
&& enabled
) {
3113 LOG(LogLevel::Debug
, ("%p: SourceMediaTrack %p setting "
3114 "direct listener enabled",
3115 GraphImpl(), this));
3116 l
->DecreaseDisabled(oldMode
);
3117 } else if (oldEnabled
&& !enabled
) {
3118 LOG(LogLevel::Debug
, ("%p: SourceMediaTrack %p setting "
3119 "direct listener disabled",
3120 GraphImpl(), this));
3121 l
->IncreaseDisabled(aMode
);
3125 MediaTrack::SetDisabledTrackModeImpl(aMode
);
3128 uint32_t SourceMediaTrack::NumberOfChannels() const {
3129 AudioSegment
* audio
= GetData
<AudioSegment
>();
3130 MOZ_DIAGNOSTIC_ASSERT(audio
);
3134 return audio
->MaxChannelCount();
3137 void SourceMediaTrack::RemoveAllDirectListenersImpl() {
3138 GraphImpl()->AssertOnGraphThreadOrNotRunning();
3139 MutexAutoLock
lock(mMutex
);
3141 for (auto& l
: mDirectTrackListeners
.Clone()) {
3142 l
->NotifyDirectListenerUninstalled();
3144 mDirectTrackListeners
.Clear();
3147 void SourceMediaTrack::SetVolume(float aVolume
) {
3148 MutexAutoLock
lock(mMutex
);
3152 float SourceMediaTrack::GetVolumeLocked() {
3153 mMutex
.AssertCurrentThreadOwns();
3157 SourceMediaTrack::~SourceMediaTrack() = default;
3159 void MediaInputPort::Init() {
3160 mGraph
->AssertOnGraphThreadOrNotRunning();
3161 LOG(LogLevel::Debug
, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph
,
3162 this, mSource
, mDest
));
3163 // Only connect the port if it wasn't disconnected on allocation.
3165 mSource
->AddConsumer(this);
3166 mDest
->AddInput(this);
3168 // mPortCount decremented via MediaInputPort::Destroy's message
3169 ++mGraph
->mPortCount
;
3172 void MediaInputPort::Disconnect() {
3173 mGraph
->AssertOnGraphThreadOrNotRunning();
3174 NS_ASSERTION(!mSource
== !mDest
,
3175 "mSource and mDest must either both be null or both non-null");
3181 mSource
->RemoveConsumer(this);
3182 mDest
->RemoveInput(this);
3186 mGraph
->SetTrackOrderDirty();
3189 MediaTrack
* MediaInputPort::GetSource() const {
3190 mGraph
->AssertOnGraphThreadOrNotRunning();
3194 ProcessedMediaTrack
* MediaInputPort::GetDestination() const {
3195 mGraph
->AssertOnGraphThreadOrNotRunning();
3199 MediaInputPort::InputInterval
MediaInputPort::GetNextInputInterval(
3200 MediaInputPort
const* aPort
, GraphTime aTime
) {
3201 InputInterval result
= {GRAPH_TIME_MAX
, GRAPH_TIME_MAX
, false};
3203 result
.mStart
= aTime
;
3204 result
.mInputIsBlocked
= true;
3207 aPort
->mGraph
->AssertOnGraphThreadOrNotRunning();
3208 if (aTime
>= aPort
->mDest
->mStartBlocking
) {
3211 result
.mStart
= aTime
;
3212 result
.mEnd
= aPort
->mDest
->mStartBlocking
;
3213 result
.mInputIsBlocked
= aTime
>= aPort
->mSource
->mStartBlocking
;
3214 if (!result
.mInputIsBlocked
) {
3215 result
.mEnd
= std::min(result
.mEnd
, aPort
->mSource
->mStartBlocking
);
3220 void MediaInputPort::Suspended() {
3221 mGraph
->AssertOnGraphThreadOrNotRunning();
3222 mDest
->InputSuspended(this);
3225 void MediaInputPort::Resumed() {
3226 mGraph
->AssertOnGraphThreadOrNotRunning();
3227 mDest
->InputResumed(this);
3230 void MediaInputPort::Destroy() {
3231 class Message
: public ControlMessage
{
3233 explicit Message(MediaInputPort
* aPort
)
3234 : ControlMessage(nullptr), mPort(aPort
) {}
3235 void Run() override
{
3236 TRACE("MediaInputPort::Destroy ControlMessage");
3237 mPort
->Disconnect();
3238 --mPort
->GraphImpl()->mPortCount
;
3239 mPort
->SetGraphImpl(nullptr);
3242 void RunDuringShutdown() override
{ Run(); }
3243 MediaInputPort
* mPort
;
3245 // Keep a reference to the graph, since Message might RunDuringShutdown()
3246 // synchronously and make GraphImpl() invalid.
3247 RefPtr
<MediaTrackGraphImpl
> graph
= mGraph
;
3248 graph
->AppendMessage(MakeUnique
<Message
>(this));
3249 --graph
->mMainThreadPortCount
;
3252 MediaTrackGraphImpl
* MediaInputPort::GraphImpl() const {
3253 mGraph
->AssertOnGraphThreadOrNotRunning();
3257 MediaTrackGraph
* MediaInputPort::Graph() const {
3258 mGraph
->AssertOnGraphThreadOrNotRunning();
3262 void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl
* aGraph
) {
3263 MOZ_ASSERT(!mGraph
|| !aGraph
, "Should only be set once");
3264 DebugOnly
<MediaTrackGraphImpl
*> graph
= mGraph
? mGraph
: aGraph
;
3265 MOZ_ASSERT(graph
->OnGraphThreadOrNotRunning());
3269 already_AddRefed
<MediaInputPort
> ProcessedMediaTrack::AllocateInputPort(
3270 MediaTrack
* aTrack
, uint16_t aInputNumber
, uint16_t aOutputNumber
) {
3271 // This method creates two references to the MediaInputPort: one for
3272 // the main thread, and one for the MediaTrackGraph.
3273 class Message
: public ControlMessage
{
3275 explicit Message(MediaInputPort
* aPort
)
3276 : ControlMessage(aPort
->mDest
), mPort(aPort
) {}
3277 void Run() override
{
3278 TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
3280 // The graph holds its reference implicitly
3281 mPort
->GraphImpl()->SetTrackOrderDirty();
3282 Unused
<< mPort
.forget();
3284 void RunDuringShutdown() override
{ Run(); }
3285 RefPtr
<MediaInputPort
> mPort
;
3288 MOZ_DIAGNOSTIC_ASSERT(aTrack
->mType
== mType
);
3289 RefPtr
<MediaInputPort
> port
;
3290 if (aTrack
->IsDestroyed()) {
3291 // Create a port that's disconnected, which is what it'd be after its source
3292 // track is Destroy()ed normally. Disconnect() is idempotent so destroying
3293 // this later is fine.
3294 port
= new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber
,
3297 MOZ_ASSERT(aTrack
->GraphImpl() == GraphImpl());
3298 port
= new MediaInputPort(GraphImpl(), aTrack
, this, aInputNumber
,
3301 ++GraphImpl()->mMainThreadPortCount
;
3302 GraphImpl()->AppendMessage(MakeUnique
<Message
>(port
));
3303 return port
.forget();
3306 void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend
) {
3307 class Message
: public ControlMessage
{
3309 Message(ProcessedMediaTrack
* aTrack
, bool aAutoend
)
3310 : ControlMessage(aTrack
), mAutoend(aAutoend
) {}
3311 void Run() override
{
3312 TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
3313 static_cast<ProcessedMediaTrack
*>(mTrack
)->SetAutoendImpl(mAutoend
);
3317 if (mMainThreadDestroyed
) {
3320 GraphImpl()->AppendMessage(MakeUnique
<Message
>(this, aAutoend
));
3323 void ProcessedMediaTrack::DestroyImpl() {
3324 for (int32_t i
= mInputs
.Length() - 1; i
>= 0; --i
) {
3325 mInputs
[i
]->Disconnect();
3328 for (int32_t i
= mSuspendedInputs
.Length() - 1; i
>= 0; --i
) {
3329 mSuspendedInputs
[i
]->Disconnect();
3332 MediaTrack::DestroyImpl();
3333 // The track order is only important if there are connections, in which
3334 // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
3335 // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
3336 // SetTrackOrderDirty(), for other reasons.
3339 MediaTrackGraphImpl::MediaTrackGraphImpl(
3340 GraphDriverType aDriverRequested
, GraphRunType aRunTypeRequested
,
3341 uint64_t aWindowID
, TrackRate aSampleRate
, uint32_t aChannelCount
,
3342 AudioDeviceID aPrimaryOutputDeviceID
, nsISerialEventTarget
* aMainThread
)
3343 : MediaTrackGraph(aSampleRate
, aPrimaryOutputDeviceID
),
3344 mWindowID(aWindowID
),
3345 mGraphRunner(aRunTypeRequested
== SINGLE_THREAD
3346 ? GraphRunner::Create(this)
3347 : already_AddRefed
<GraphRunner
>(nullptr)),
3348 mFirstCycleBreaker(0)
3349 // An offline graph is not initially processing.
3351 mEndTime(aDriverRequested
== OFFLINE_THREAD_DRIVER
? 0 : GRAPH_TIME_MAX
),
3353 mMonitor("MediaTrackGraphImpl"),
3354 mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED
),
3355 mPostedRunInStableStateEvent(false),
3356 mGraphDriverRunning(false),
3357 mPostedRunInStableState(false),
3358 mRealtime(aDriverRequested
!= OFFLINE_THREAD_DRIVER
),
3359 mTrackOrderDirty(false),
3360 mMainThread(aMainThread
),
3362 mGlobalVolume(CubebUtils::GetVolumeScale())
3365 mCanRunMessagesSynchronously(false)
3368 mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
3369 mAudioOutputLatency(0.0),
3370 mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
3371 // The primary output device always exists because an AudioCallbackDriver
3372 // may exist, and want to be fed data, even when no tracks have audio
3374 mOutputDeviceRefCnts
.EmplaceBack(
3375 DeviceReceiverAndCount
{aPrimaryOutputDeviceID
, nullptr, 0});
3376 mOutputDevices
.EmplaceBack(OutputDeviceEntry
{aPrimaryOutputDeviceID
});
3378 bool failedToGetShutdownBlocker
= false;
3379 if (!IsNonRealtime()) {
3380 failedToGetShutdownBlocker
= !AddShutdownBlocker();
3383 if ((aRunTypeRequested
== SINGLE_THREAD
&& !mGraphRunner
) ||
3384 failedToGetShutdownBlocker
) {
3385 // At least one of the following happened
3386 // - Failed to create thread.
3387 // - Failed to install a shutdown blocker when one is needed.
3388 // Because we have a fail state, jump to last phase of the lifecycle.
3389 mLifecycleState
= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
;
3390 RemoveShutdownBlocker(); // No-op if blocker wasn't added.
3392 mCanRunMessagesSynchronously
= true;
3397 if (aDriverRequested
== AUDIO_THREAD_DRIVER
) {
3398 // Always start with zero input channels, and no particular preferences
3399 // for the input channel.
3400 mDriver
= new AudioCallbackDriver(
3401 this, nullptr, mSampleRate
, aChannelCount
, 0, PrimaryOutputDeviceID(),
3402 nullptr, AudioInputType::Unknown
);
3404 mDriver
= new SystemClockDriver(this, nullptr, mSampleRate
);
3406 nsCString streamName
= GetDocumentTitle(aWindowID
);
3407 LOG(LogLevel::Debug
, ("%p: document title: %s", this, streamName
.get()));
3408 mDriver
->SetStreamName(streamName
);
3411 new OfflineClockDriver(this, mSampleRate
, MEDIA_GRAPH_TARGET_PERIOD_MS
);
3414 mLastMainThreadUpdate
= TimeStamp::Now();
3416 RegisterWeakAsyncMemoryReporter(this);
3420 bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver
* aDriver
) const {
3421 return aDriver
->OnThread() ||
3422 (mGraphRunner
&& mGraphRunner
->InDriverIteration(aDriver
));
3426 void MediaTrackGraphImpl::Destroy() {
3427 // First unregister from memory reporting.
3428 UnregisterWeakMemoryReporter(this);
3430 // Clear the self reference which will destroy this instance if all
3431 // associated GraphDrivers are destroyed.
3435 // Internal method has a Window ID parameter so that TestAudioTrackGraph
3436 // GTests can create a graph without a window.
3438 MediaTrackGraphImpl
* MediaTrackGraphImpl::GetInstanceIfExists(
3439 uint64_t aWindowID
, TrackRate aSampleRate
,
3440 AudioDeviceID aPrimaryOutputDeviceID
) {
3441 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3442 MOZ_ASSERT(aSampleRate
> 0);
3444 GraphHashSet::Ptr p
=
3445 Graphs()->lookup({aWindowID
, aSampleRate
, aPrimaryOutputDeviceID
});
3446 return p
? *p
: nullptr;
3449 // Public method has an nsPIDOMWindowInner* parameter to ensure that the
3450 // window is a real inner Window, not a WindowProxy.
3452 MediaTrackGraph
* MediaTrackGraph::GetInstanceIfExists(
3453 nsPIDOMWindowInner
* aWindow
, TrackRate aSampleRate
,
3454 AudioDeviceID aPrimaryOutputDeviceID
) {
3455 TrackRate sampleRate
=
3456 aSampleRate
? aSampleRate
3457 : CubebUtils::PreferredSampleRate(
3458 aWindow
->AsGlobal()->ShouldResistFingerprinting(
3459 RFPTarget::AudioSampleRate
));
3460 return MediaTrackGraphImpl::GetInstanceIfExists(
3461 aWindow
->WindowID(), sampleRate
, aPrimaryOutputDeviceID
);
3465 MediaTrackGraphImpl
* MediaTrackGraphImpl::GetInstance(
3466 GraphDriverType aGraphDriverRequested
, uint64_t aWindowID
,
3467 TrackRate aSampleRate
, AudioDeviceID aPrimaryOutputDeviceID
,
3468 nsISerialEventTarget
* aMainThread
) {
3469 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3470 MOZ_ASSERT(aSampleRate
> 0);
3471 MOZ_ASSERT(aGraphDriverRequested
!= OFFLINE_THREAD_DRIVER
,
3472 "Use CreateNonRealtimeInstance() for offline graphs");
3474 GraphHashSet
* graphs
= Graphs();
3475 GraphHashSet::AddPtr addPtr
=
3476 graphs
->lookupForAdd({aWindowID
, aSampleRate
, aPrimaryOutputDeviceID
});
3477 if (addPtr
) { // graph already exists
3481 GraphRunType runType
= DIRECT_DRIVER
;
3482 if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) {
3483 runType
= SINGLE_THREAD
;
3486 // In a real time graph, the number of output channels is determined by
3487 // the underlying number of channel of the default audio output device, and
3489 uint32_t channelCount
=
3490 std::min
<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
3491 MediaTrackGraphImpl
* graph
= new MediaTrackGraphImpl(
3492 aGraphDriverRequested
, runType
, aWindowID
, aSampleRate
, channelCount
,
3493 aPrimaryOutputDeviceID
, aMainThread
);
3494 MOZ_ALWAYS_TRUE(graphs
->add(addPtr
, graph
));
3496 nsCOMPtr
<nsIObserverService
> observerService
=
3497 mozilla::services::GetObserverService();
3498 if (observerService
) {
3499 observerService
->AddObserver(graph
, "document-title-changed", false);
3502 LOG(LogLevel::Debug
, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64
,
3509 MediaTrackGraph
* MediaTrackGraph::GetInstance(
3510 GraphDriverType aGraphDriverRequested
, nsPIDOMWindowInner
* aWindow
,
3511 TrackRate aSampleRate
, AudioDeviceID aPrimaryOutputDeviceID
) {
3512 TrackRate sampleRate
=
3513 aSampleRate
? aSampleRate
3514 : CubebUtils::PreferredSampleRate(
3515 aWindow
->AsGlobal()->ShouldResistFingerprinting(
3516 RFPTarget::AudioSampleRate
));
3517 return MediaTrackGraphImpl::GetInstance(
3518 aGraphDriverRequested
, aWindow
->WindowID(), sampleRate
,
3519 aPrimaryOutputDeviceID
, GetMainThreadSerialEventTarget());
3522 MediaTrackGraph
* MediaTrackGraph::CreateNonRealtimeInstance(
3523 TrackRate aSampleRate
) {
3524 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3526 nsISerialEventTarget
* mainThread
= GetMainThreadSerialEventTarget();
3527 // Offline graphs have 0 output channel count: they write the output to a
3528 // buffer, not an audio output track.
3529 MediaTrackGraphImpl
* graph
= new MediaTrackGraphImpl(
3530 OFFLINE_THREAD_DRIVER
, DIRECT_DRIVER
, 0, aSampleRate
, 0,
3531 DEFAULT_OUTPUT_DEVICE
, mainThread
);
3533 LOG(LogLevel::Debug
, ("Starting up Offline MediaTrackGraph %p", graph
));
3538 void MediaTrackGraph::ForceShutDown() {
3539 MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
3541 MediaTrackGraphImpl
* graph
= static_cast<MediaTrackGraphImpl
*>(this);
3543 graph
->ForceShutDown();
3546 NS_IMPL_ISUPPORTS(MediaTrackGraphImpl
, nsIMemoryReporter
, nsIObserver
,
3547 nsIThreadObserver
, nsITimerCallback
, nsINamed
)
3550 MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback
* aHandleReport
,
3551 nsISupports
* aData
, bool aAnonymize
) {
3552 MOZ_ASSERT(NS_IsMainThread());
3553 if (mMainThreadTrackCount
== 0) {
3554 // No tracks to report.
3555 FinishCollectReports(aHandleReport
, aData
, nsTArray
<AudioNodeSizes
>());
3559 class Message final
: public ControlMessage
{
3561 Message(MediaTrackGraphImpl
* aGraph
, nsIHandleReportCallback
* aHandleReport
,
3562 nsISupports
* aHandlerData
)
3563 : ControlMessage(nullptr),
3565 mHandleReport(aHandleReport
),
3566 mHandlerData(aHandlerData
) {}
3567 void Run() override
{
3568 TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
3569 mGraph
->CollectSizesForMemoryReport(mHandleReport
.forget(),
3570 mHandlerData
.forget());
3572 void RunDuringShutdown() override
{
3573 // Run this message during shutdown too, so that endReports is called.
3576 MediaTrackGraphImpl
* mGraph
;
3577 // nsMemoryReporterManager keeps the callback and data alive only if it
3578 // does not time out.
3579 nsCOMPtr
<nsIHandleReportCallback
> mHandleReport
;
3580 nsCOMPtr
<nsISupports
> mHandlerData
;
3583 AppendMessage(MakeUnique
<Message
>(this, aHandleReport
, aData
));
3588 void MediaTrackGraphImpl::CollectSizesForMemoryReport(
3589 already_AddRefed
<nsIHandleReportCallback
> aHandleReport
,
3590 already_AddRefed
<nsISupports
> aHandlerData
) {
3591 class FinishCollectRunnable final
: public Runnable
{
3593 explicit FinishCollectRunnable(
3594 already_AddRefed
<nsIHandleReportCallback
> aHandleReport
,
3595 already_AddRefed
<nsISupports
> aHandlerData
)
3596 : mozilla::Runnable("FinishCollectRunnable"),
3597 mHandleReport(aHandleReport
),
3598 mHandlerData(aHandlerData
) {}
3600 NS_IMETHOD
Run() override
{
3601 TRACE("MTG::FinishCollectReports ControlMessage");
3602 MediaTrackGraphImpl::FinishCollectReports(mHandleReport
, mHandlerData
,
3603 std::move(mAudioTrackSizes
));
3607 nsTArray
<AudioNodeSizes
> mAudioTrackSizes
;
3610 ~FinishCollectRunnable() = default;
3612 // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
3613 // constructor modifies the ref-count, which cannot be done off main
3615 RefPtr
<nsIHandleReportCallback
> mHandleReport
;
3616 RefPtr
<nsISupports
> mHandlerData
;
3619 RefPtr
<FinishCollectRunnable
> runnable
= new FinishCollectRunnable(
3620 std::move(aHandleReport
), std::move(aHandlerData
));
3622 auto audioTrackSizes
= &runnable
->mAudioTrackSizes
;
3624 for (MediaTrack
* t
: AllTracks()) {
3625 AudioNodeTrack
* track
= t
->AsAudioNodeTrack();
3627 AudioNodeSizes
* usage
= audioTrackSizes
->AppendElement();
3628 track
->SizeOfAudioNodesIncludingThis(MallocSizeOf
, *usage
);
3632 mMainThread
->Dispatch(runnable
.forget());
3635 void MediaTrackGraphImpl::FinishCollectReports(
3636 nsIHandleReportCallback
* aHandleReport
, nsISupports
* aData
,
3637 const nsTArray
<AudioNodeSizes
>& aAudioTrackSizes
) {
3638 MOZ_ASSERT(NS_IsMainThread());
3640 nsCOMPtr
<nsIMemoryReporterManager
> manager
=
3641 do_GetService("@mozilla.org/memory-reporter-manager;1");
3643 if (!manager
) return;
3645 #define REPORT(_path, _amount, _desc) \
3646 aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
3647 nsLiteralCString(_desc), aData);
3649 for (size_t i
= 0; i
< aAudioTrackSizes
.Length(); i
++) {
3650 const AudioNodeSizes
& usage
= aAudioTrackSizes
[i
];
3651 const char* const nodeType
=
3652 usage
.mNodeType
? usage
.mNodeType
: "<unknown>";
3654 nsPrintfCString
enginePath("explicit/webaudio/audio-node/%s/engine-objects",
3656 REPORT(enginePath
, usage
.mEngine
,
3657 "Memory used by AudioNode engine objects (Web Audio).");
3659 nsPrintfCString
trackPath("explicit/webaudio/audio-node/%s/track-objects",
3661 REPORT(trackPath
, usage
.mTrack
,
3662 "Memory used by AudioNode track objects (Web Audio).");
3665 size_t hrtfLoaders
= WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf
);
3667 REPORT(nsLiteralCString(
3668 "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
3669 hrtfLoaders
, "Memory used by PannerNode databases (Web Audio).");
3674 manager
->EndReport();
3677 SourceMediaTrack
* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType
) {
3678 SourceMediaTrack
* track
= new SourceMediaTrack(aType
, GraphRate());
3683 ProcessedMediaTrack
* MediaTrackGraph::CreateForwardedInputTrack(
3684 MediaSegment::Type aType
) {
3685 ForwardedInputTrack
* track
= new ForwardedInputTrack(GraphRate(), aType
);
3690 AudioCaptureTrack
* MediaTrackGraph::CreateAudioCaptureTrack() {
3691 AudioCaptureTrack
* track
= new AudioCaptureTrack(GraphRate());
3696 CrossGraphTransmitter
* MediaTrackGraph::CreateCrossGraphTransmitter(
3697 CrossGraphReceiver
* aReceiver
) {
3698 CrossGraphTransmitter
* track
=
3699 new CrossGraphTransmitter(GraphRate(), aReceiver
);
3704 CrossGraphReceiver
* MediaTrackGraph::CreateCrossGraphReceiver(
3705 TrackRate aTransmitterRate
) {
3706 CrossGraphReceiver
* track
=
3707 new CrossGraphReceiver(GraphRate(), aTransmitterRate
);
3712 void MediaTrackGraph::AddTrack(MediaTrack
* aTrack
) {
3713 MediaTrackGraphImpl
* graph
= static_cast<MediaTrackGraphImpl
*>(this);
3714 MOZ_ASSERT(NS_IsMainThread());
3715 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
3716 if (graph
->mRealtime
) {
3717 GraphHashSet::Ptr p
= Graphs()->lookup(*graph
);
3718 MOZ_DIAGNOSTIC_ASSERT(p
, "Graph must not be shutting down");
3722 aTrack
->SetGraphImpl(graph
);
3723 ++graph
->mMainThreadTrackCount
;
3724 graph
->AppendMessage(MakeUnique
<CreateMessage
>(aTrack
));
3727 void MediaTrackGraphImpl::RemoveTrack(MediaTrack
* aTrack
) {
3728 MOZ_ASSERT(NS_IsMainThread());
3729 MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount
> 0);
3731 mAudioOutputParams
.RemoveElementsBy(
3732 [&](const TrackKeyDeviceAndVolume
& aElement
) {
3733 if (aElement
.mTrack
!= aTrack
) {
3736 DecrementOutputDeviceRefCnt(aElement
.mDeviceID
);
3740 if (--mMainThreadTrackCount
== 0) {
3741 LOG(LogLevel::Info
, ("MediaTrackGraph %p, last track %p removed from "
3742 "main thread. Graph will shut down.",
3745 // Find the graph in the hash table and remove it.
3746 GraphHashSet
* graphs
= Graphs();
3747 GraphHashSet::Ptr p
= graphs
->lookup(*this);
3748 MOZ_ASSERT(*p
== this);
3751 nsCOMPtr
<nsIObserverService
> observerService
=
3752 mozilla::services::GetObserverService();
3753 if (observerService
) {
3754 observerService
->RemoveObserver(this, "document-title-changed");
3757 // The graph thread will shut itself down soon, but won't be able to do
3758 // that if JS continues to run.
3763 auto MediaTrackGraph::NotifyWhenDeviceStarted(MediaTrack
* aTrack
)
3764 -> RefPtr
<GraphStartedPromise
> {
3765 MOZ_ASSERT(NS_IsMainThread());
3766 MozPromiseHolder
<GraphStartedPromise
> h
;
3767 RefPtr
<GraphStartedPromise
> p
= h
.Ensure(__func__
);
3768 aTrack
->GraphImpl()->NotifyWhenGraphStarted(aTrack
, std::move(h
));
3772 void MediaTrackGraphImpl::NotifyWhenGraphStarted(
3773 RefPtr
<MediaTrack
> aTrack
,
3774 MozPromiseHolder
<GraphStartedPromise
>&& aHolder
) {
3775 MOZ_ASSERT(NS_IsMainThread());
3776 if (aTrack
->IsDestroyed()) {
3777 aHolder
.Reject(NS_ERROR_NOT_AVAILABLE
, __func__
);
3781 QueueControlOrShutdownMessage(
3782 [self
= RefPtr
{this}, this, track
= std::move(aTrack
),
3783 holder
= std::move(aHolder
)](IsInShutdown aInShutdown
) mutable {
3784 if (aInShutdown
== IsInShutdown::Yes
) {
3785 holder
.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN
, __func__
);
3789 TRACE("MTG::GraphStartedNotificationControlMessage ControlMessage");
3790 // This runs on the graph thread, so when this runs, and the current
3791 // driver is an AudioCallbackDriver, we know the audio hardware is
3792 // started. If not, we are going to switch soon, keep reposting this
3794 if (CurrentDriver()->AsAudioCallbackDriver() &&
3795 CurrentDriver()->ThreadRunning() &&
3796 !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
3797 // Avoid Resolve's locking on the graph thread by doing it on main.
3798 Dispatch(NS_NewRunnableFunction(
3799 "MediaTrackGraphImpl::NotifyWhenGraphStarted::Resolver",
3800 [holder
= std::move(holder
)]() mutable {
3801 holder
.Resolve(true, __func__
);
3804 DispatchToMainThreadStableState(
3806 StoreCopyPassByRRef
<RefPtr
<MediaTrack
>>,
3807 StoreCopyPassByRRef
<MozPromiseHolder
<GraphStartedPromise
>>>(
3808 "MediaTrackGraphImpl::NotifyWhenGraphStarted", this,
3809 &MediaTrackGraphImpl::NotifyWhenGraphStarted
,
3810 std::move(track
), std::move(holder
)));
3815 class AudioContextOperationControlMessage
: public ControlMessage
{
3816 using AudioContextOperationPromise
=
3817 MediaTrackGraph::AudioContextOperationPromise
;
3820 AudioContextOperationControlMessage(
3821 MediaTrack
* aDestinationTrack
, nsTArray
<RefPtr
<MediaTrack
>> aTracks
,
3822 AudioContextOperation aOperation
,
3823 MozPromiseHolder
<AudioContextOperationPromise
>&& aHolder
)
3824 : ControlMessage(aDestinationTrack
),
3825 mTracks(std::move(aTracks
)),
3826 mAudioContextOperation(aOperation
),
3827 mHolder(std::move(aHolder
)) {}
3828 void Run() override
{
3829 TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
3830 kAudioContextOptionsStrings
[static_cast<uint8_t>(
3831 mAudioContextOperation
)]);
3832 mTrack
->GraphImpl()->ApplyAudioContextOperationImpl(this);
3834 void RunDuringShutdown() override
{
3835 MOZ_ASSERT(mAudioContextOperation
== AudioContextOperation::Close
,
3836 "We should be reviving the graph?");
3837 mHolder
.Reject(false, __func__
);
3840 nsTArray
<RefPtr
<MediaTrack
>> mTracks
;
3841 AudioContextOperation mAudioContextOperation
;
3842 MozPromiseHolder
<AudioContextOperationPromise
> mHolder
;
3845 void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
3846 AudioContextOperationControlMessage
* aMessage
) {
3847 MOZ_ASSERT(OnGraphThread());
3848 // Initialize state to zero. This silences a GCC warning about uninitialized
3849 // values, because although the switch below initializes state for all valid
3850 // enum values, the actual value could be any integer that fits in the enum.
3851 AudioContextState state
{0};
3852 switch (aMessage
->mAudioContextOperation
) {
3853 // Suspend and Close operations may be performed immediately because no
3854 // specific kind of GraphDriver is required. CheckDriver() will schedule
3855 // a change to a SystemCallbackDriver if all tracks are suspended.
3856 case AudioContextOperation::Suspend
:
3857 state
= AudioContextState::Suspended
;
3859 case AudioContextOperation::Close
:
3860 state
= AudioContextState::Closed
;
3862 case AudioContextOperation::Resume
:
3863 // Resume operations require an AudioCallbackDriver. CheckDriver() will
3864 // schedule an AudioCallbackDriver if necessary and process pending
3865 // operations if and when an AudioCallbackDriver is running.
3866 mPendingResumeOperations
.EmplaceBack(aMessage
);
3869 // First resolve any pending Resume promises for the same AudioContext so as
3870 // to resolve its associated promises in the same order as they were
3871 // created. These Resume operations are considered complete and immediately
3872 // canceled by the Suspend or Close.
3873 MediaTrack
* destinationTrack
= aMessage
->GetTrack();
3874 bool shrinking
= false;
3875 auto moveDest
= mPendingResumeOperations
.begin();
3876 for (PendingResumeOperation
& op
: mPendingResumeOperations
) {
3877 if (op
.DestinationTrack() == destinationTrack
) {
3882 if (shrinking
) { // Fill-in gaps in the array.
3883 *moveDest
= std::move(op
);
3887 mPendingResumeOperations
.TruncateLength(moveDest
-
3888 mPendingResumeOperations
.begin());
3890 for (MediaTrack
* track
: aMessage
->mTracks
) {
3891 track
->IncrementSuspendCount();
3893 // Resolve after main thread state is up to date with completed processing.
3894 DispatchToMainThreadStableState(NS_NewRunnableFunction(
3895 "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
3896 [holder
= std::move(aMessage
->mHolder
), state
]() mutable {
3897 holder
.Resolve(state
, __func__
);
3901 MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
3902 AudioContextOperationControlMessage
* aMessage
)
3903 : mDestinationTrack(aMessage
->GetTrack()),
3904 mTracks(std::move(aMessage
->mTracks
)),
3905 mHolder(std::move(aMessage
->mHolder
)) {
3906 MOZ_ASSERT(aMessage
->mAudioContextOperation
== AudioContextOperation::Resume
);
3909 void MediaTrackGraphImpl::PendingResumeOperation::Apply(
3910 MediaTrackGraphImpl
* aGraph
) {
3911 MOZ_ASSERT(aGraph
->OnGraphThread());
3912 for (MediaTrack
* track
: mTracks
) {
3913 track
->DecrementSuspendCount();
3915 // The graph is provided through the parameter so that it is available even
3916 // when the track is destroyed.
3917 aGraph
->DispatchToMainThreadStableState(NS_NewRunnableFunction(
3918 "PendingResumeOperation::Apply", [holder
= std::move(mHolder
)]() mutable {
3919 holder
.Resolve(AudioContextState::Running
, __func__
);
3923 void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
3924 // The graph is shutting down before the operation completed.
3925 MOZ_ASSERT(!mDestinationTrack
->GraphImpl() ||
3926 mDestinationTrack
->GraphImpl()->LifecycleStateRef() ==
3927 MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN
);
3928 mHolder
.Reject(false, __func__
);
3931 auto MediaTrackGraph::ApplyAudioContextOperation(
3932 MediaTrack
* aDestinationTrack
, nsTArray
<RefPtr
<MediaTrack
>> aTracks
,
3933 AudioContextOperation aOperation
) -> RefPtr
<AudioContextOperationPromise
> {
3934 MozPromiseHolder
<AudioContextOperationPromise
> holder
;
3935 RefPtr
<AudioContextOperationPromise
> p
= holder
.Ensure(__func__
);
3936 MediaTrackGraphImpl
* graphImpl
= static_cast<MediaTrackGraphImpl
*>(this);
3937 graphImpl
->AppendMessage(MakeUnique
<AudioContextOperationControlMessage
>(
3938 aDestinationTrack
, std::move(aTracks
), aOperation
, std::move(holder
)));
3942 uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const {
3943 MOZ_ASSERT(!mOutputDevices
[0].mReceiver
);
3944 return AudioOutputChannelCount(mOutputDevices
[0]);
3947 uint32_t MediaTrackGraphImpl::AudioOutputChannelCount(
3948 const OutputDeviceEntry
& aDevice
) const {
3949 MOZ_ASSERT(OnGraphThread());
3950 // The audio output channel count for a graph is the maximum of the output
3951 // channel count of all the tracks that are in mAudioOutputs, or the max audio
3952 // output channel count the machine can do, whichever is smaller.
3953 uint32_t channelCount
= 0;
3954 for (const auto& output
: aDevice
.mTrackOutputs
) {
3955 channelCount
= std::max(channelCount
, output
.mTrack
->NumberOfChannels());
3957 channelCount
= std::min(channelCount
, mMaxOutputChannelCount
);
3959 return channelCount
;
3961 // null aDevice.mReceiver indicates the primary graph output device.
3962 if (!aDevice
.mReceiver
&& CurrentDriver()->AsAudioCallbackDriver()) {
3963 return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
3969 double MediaTrackGraph::AudioOutputLatency() {
3970 return static_cast<MediaTrackGraphImpl
*>(this)->AudioOutputLatency();
3973 double MediaTrackGraphImpl::AudioOutputLatency() {
3974 MOZ_ASSERT(NS_IsMainThread());
3975 if (mAudioOutputLatency
!= 0.0) {
3976 return mAudioOutputLatency
;
3978 MonitorAutoLock
lock(mMonitor
);
3979 if (CurrentDriver()->AsAudioCallbackDriver()) {
3980 mAudioOutputLatency
= CurrentDriver()
3981 ->AsAudioCallbackDriver()
3982 ->AudioOutputLatency()
3985 // Failure mode: return 0.0 if running on a normal thread.
3986 mAudioOutputLatency
= 0.0;
3989 return mAudioOutputLatency
;
3992 bool MediaTrackGraph::IsNonRealtime() const {
3993 return !static_cast<const MediaTrackGraphImpl
*>(this)->mRealtime
;
3996 void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess
) {
3997 MOZ_ASSERT(NS_IsMainThread(), "main thread only");
3999 MediaTrackGraphImpl
* graph
= static_cast<MediaTrackGraphImpl
*>(this);
4000 NS_ASSERTION(!graph
->mRealtime
, "non-realtime only");
4002 class Message
: public ControlMessage
{
4004 explicit Message(MediaTrackGraphImpl
* aGraph
, uint32_t aTicksToProcess
)
4005 : ControlMessage(nullptr),
4007 mTicksToProcess(aTicksToProcess
) {}
4008 void Run() override
{
4009 TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
4010 MOZ_ASSERT(mGraph
->mEndTime
== 0,
4011 "StartNonRealtimeProcessing should be called only once");
4012 mGraph
->mEndTime
= mGraph
->RoundUpToEndOfAudioBlock(
4013 mGraph
->mStateComputedTime
+ mTicksToProcess
);
4015 // The graph owns this message.
4016 MediaTrackGraphImpl
* MOZ_NON_OWNING_REF mGraph
;
4017 uint32_t mTicksToProcess
;
4020 graph
->AppendMessage(MakeUnique
<Message
>(graph
, aTicksToProcess
));
4023 void MediaTrackGraphImpl::InterruptJS() {
4024 MonitorAutoLock
lock(mMonitor
);
4025 mInterruptJSCalled
= true;
4027 JS_RequestInterruptCallback(mJSContext
);
4031 static bool InterruptCallback(JSContext
* aCx
) {
4032 // Interrupt future calls also.
4033 JS_RequestInterruptCallback(aCx
);
4038 void MediaTrackGraph::NotifyJSContext(JSContext
* aCx
) {
4039 MOZ_ASSERT(OnGraphThread());
4042 auto* impl
= static_cast<MediaTrackGraphImpl
*>(this);
4043 MonitorAutoLock
lock(impl
->mMonitor
);
4044 if (impl
->mJSContext
) {
4045 MOZ_ASSERT(impl
->mJSContext
== aCx
);
4048 JS_AddInterruptCallback(aCx
, InterruptCallback
);
4049 impl
->mJSContext
= aCx
;
4050 if (impl
->mInterruptJSCalled
) {
4051 JS_RequestInterruptCallback(aCx
);
4055 void ProcessedMediaTrack::AddInput(MediaInputPort
* aPort
) {
4056 MediaTrack
* t
= aPort
->GetSource();
4057 if (!t
->IsSuspended()) {
4058 mInputs
.AppendElement(aPort
);
4060 mSuspendedInputs
.AppendElement(aPort
);
4062 GraphImpl()->SetTrackOrderDirty();
4065 void ProcessedMediaTrack::InputSuspended(MediaInputPort
* aPort
) {
4066 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4067 mInputs
.RemoveElement(aPort
);
4068 mSuspendedInputs
.AppendElement(aPort
);
4069 GraphImpl()->SetTrackOrderDirty();
4072 void ProcessedMediaTrack::InputResumed(MediaInputPort
* aPort
) {
4073 GraphImpl()->AssertOnGraphThreadOrNotRunning();
4074 mSuspendedInputs
.RemoveElement(aPort
);
4075 mInputs
.AppendElement(aPort
);
4076 GraphImpl()->SetTrackOrderDirty();
4079 void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver
* aNextDriver
) {
4080 MOZ_ASSERT(OnGraphThread());
4081 LOG(LogLevel::Debug
, ("%p: Switching to new driver: %p", this, aNextDriver
));
4082 if (GraphDriver
* nextDriver
= NextDriver()) {
4083 if (nextDriver
!= CurrentDriver()) {
4084 LOG(LogLevel::Debug
,
4085 ("%p: Discarding previous next driver: %p", this, nextDriver
));
4088 mNextDriver
= aNextDriver
;
4091 void MediaTrackGraph::RegisterCaptureTrackForWindow(
4092 uint64_t aWindowId
, ProcessedMediaTrack
* aCaptureTrack
) {
4093 MOZ_ASSERT(NS_IsMainThread());
4094 MediaTrackGraphImpl
* graphImpl
= static_cast<MediaTrackGraphImpl
*>(this);
4095 graphImpl
->RegisterCaptureTrackForWindow(aWindowId
, aCaptureTrack
);
4098 void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
4099 uint64_t aWindowId
, ProcessedMediaTrack
* aCaptureTrack
) {
4100 MOZ_ASSERT(NS_IsMainThread());
4101 WindowAndTrack winAndTrack
;
4102 winAndTrack
.mWindowId
= aWindowId
;
4103 winAndTrack
.mCaptureTrackSink
= aCaptureTrack
;
4104 mWindowCaptureTracks
.AppendElement(winAndTrack
);
4107 void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId
) {
4108 MOZ_ASSERT(NS_IsMainThread());
4109 MediaTrackGraphImpl
* graphImpl
= static_cast<MediaTrackGraphImpl
*>(this);
4110 graphImpl
->UnregisterCaptureTrackForWindow(aWindowId
);
4113 void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId
) {
4114 MOZ_ASSERT(NS_IsMainThread());
4115 mWindowCaptureTracks
.RemoveElementsBy(
4116 [aWindowId
](const auto& track
) { return track
.mWindowId
== aWindowId
; });
4119 already_AddRefed
<MediaInputPort
> MediaTrackGraph::ConnectToCaptureTrack(
4120 uint64_t aWindowId
, MediaTrack
* aMediaTrack
) {
4121 return aMediaTrack
->GraphImpl()->ConnectToCaptureTrack(aWindowId
,
4125 already_AddRefed
<MediaInputPort
> MediaTrackGraphImpl::ConnectToCaptureTrack(
4126 uint64_t aWindowId
, MediaTrack
* aMediaTrack
) {
4127 MOZ_ASSERT(NS_IsMainThread());
4128 for (uint32_t i
= 0; i
< mWindowCaptureTracks
.Length(); i
++) {
4129 if (mWindowCaptureTracks
[i
].mWindowId
== aWindowId
) {
4130 ProcessedMediaTrack
* sink
= mWindowCaptureTracks
[i
].mCaptureTrackSink
;
4131 return sink
->AllocateInputPort(aMediaTrack
);
4137 void MediaTrackGraph::DispatchToMainThreadStableState(
4138 already_AddRefed
<nsIRunnable
> aRunnable
) {
4139 AssertOnGraphThreadOrNotRunning();
4140 static_cast<MediaTrackGraphImpl
*>(this)
4141 ->mPendingUpdateRunnables
.AppendElement(std::move(aRunnable
));
4144 Watchable
<mozilla::GraphTime
>& MediaTrackGraphImpl::CurrentTime() {
4145 MOZ_ASSERT(NS_IsMainThread());
4146 return mMainThreadGraphTime
;
4149 GraphTime
MediaTrackGraph::ProcessedTime() const {
4150 AssertOnGraphThreadOrNotRunning();
4151 return static_cast<const MediaTrackGraphImpl
*>(this)->mProcessedTime
;
4154 void* MediaTrackGraph::CurrentDriver() const {
4155 AssertOnGraphThreadOrNotRunning();
4156 return static_cast<const MediaTrackGraphImpl
*>(this)->mDriver
;
4159 uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
4160 CubebUtils::AudioDeviceID aID
) {
4161 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4162 DeviceInputTrack
* t
=
4163 mDeviceInputTrackManagerGraphThread
.GetDeviceInputTrack(aID
);
4164 return t
? t
->MaxRequestedInputChannels() : 0;
4167 AudioInputType
MediaTrackGraphImpl::AudioInputDevicePreference(
4168 CubebUtils::AudioDeviceID aID
) {
4169 MOZ_ASSERT(OnGraphThreadOrNotRunning());
4170 DeviceInputTrack
* t
=
4171 mDeviceInputTrackManagerGraphThread
.GetDeviceInputTrack(aID
);
4172 return t
&& t
->HasVoiceInput() ? AudioInputType::Voice
4173 : AudioInputType::Unknown
;
4176 void MediaTrackGraphImpl::SetNewNativeInput() {
4177 MOZ_ASSERT(NS_IsMainThread());
4178 MOZ_ASSERT(!mDeviceInputTrackManagerMainThread
.GetNativeInputTrack());
4180 LOG(LogLevel::Debug
, ("%p SetNewNativeInput", this));
4182 NonNativeInputTrack
* track
=
4183 mDeviceInputTrackManagerMainThread
.GetFirstNonNativeInputTrack();
4185 LOG(LogLevel::Debug
, ("%p No other devices opened. Do nothing", this));
4189 const CubebUtils::AudioDeviceID deviceId
= track
->mDeviceId
;
4190 const PrincipalHandle principal
= track
->mPrincipalHandle
;
4192 LOG(LogLevel::Debug
,
4193 ("%p Select device %p as the new native input device", this, deviceId
));
4195 struct TrackListener
{
4196 DeviceInputConsumerTrack
* track
;
4197 // Keep its reference so it won't be dropped when after
4198 // DisconnectDeviceInput().
4199 RefPtr
<AudioDataListener
> listener
;
4201 nsTArray
<TrackListener
> pairs
;
4203 for (const auto& t
: track
->GetConsumerTracks()) {
4204 pairs
.AppendElement(
4205 TrackListener
{t
.get(), t
->GetAudioDataListener().get()});
4208 for (TrackListener
& pair
: pairs
) {
4209 pair
.track
->DisconnectDeviceInput();
4212 for (TrackListener
& pair
: pairs
) {
4213 pair
.track
->ConnectDeviceInput(deviceId
, pair
.listener
.get(), principal
);
4214 LOG(LogLevel::Debug
,
4215 ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
4216 pair
.track
, deviceId
));
4219 LOG(LogLevel::Debug
,
4220 ("%p Native input device is set to device %p now", this, deviceId
));
4222 MOZ_ASSERT(mDeviceInputTrackManagerMainThread
.GetNativeInputTrack());
4225 // nsIThreadObserver methods
4228 MediaTrackGraphImpl::OnDispatchedEvent() {
4229 MonitorAutoLock
lock(mMonitor
);
4230 EnsureNextIteration();
4235 MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal
*, bool) {
4240 MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal
*, bool) {
4243 } // namespace mozilla