1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MOZILLA_MEDIATRACKGRAPHIMPL_H_
7 #define MOZILLA_MEDIATRACKGRAPHIMPL_H_
9 #include "MediaTrackGraph.h"
11 #include "AudioMixer.h"
12 #include "GraphDriver.h"
13 #include "DeviceInputTrack.h"
14 #include "mozilla/Atomics.h"
15 #include "mozilla/Maybe.h"
16 #include "mozilla/Monitor.h"
17 #include "mozilla/TimeStamp.h"
18 #include "mozilla/UniquePtr.h"
19 #include "mozilla/WeakPtr.h"
20 #include "nsClassHashtable.h"
21 #include "nsIMemoryReporter.h"
23 #include "nsIRunnable.h"
24 #include "nsIThreadInternal.h"
26 #include "AsyncLogger.h"
31 class ShutdownBlocker
;
34 class AudioContextOperationControlMessage
;
39 class DeviceInputTrackManager
{
41 DeviceInputTrackManager() = default;
43 // Returns the current NativeInputTrack.
44 NativeInputTrack
* GetNativeInputTrack();
45 // Returns the DeviceInputTrack paired with the device of aID if it exists.
46 // Otherwise, returns nullptr.
47 DeviceInputTrack
* GetDeviceInputTrack(CubebUtils::AudioDeviceID aID
);
48 // Returns the first added NonNativeInputTrack if any. Otherwise, returns
50 NonNativeInputTrack
* GetFirstNonNativeInputTrack();
51 // Adds DeviceInputTrack to the managing list.
52 void Add(DeviceInputTrack
* aTrack
);
53 // Removes DeviceInputTrack from the managing list.
54 void Remove(DeviceInputTrack
* aTrack
);
57 RefPtr
<NativeInputTrack
> mNativeInputTrack
;
58 nsTArray
<RefPtr
<NonNativeInputTrack
>> mNonNativeInputTracks
;
62 * A per-track update message passed from the media graph thread to the
66 RefPtr
<MediaTrack
> mTrack
;
67 TrackTime mNextMainThreadCurrentTime
;
68 bool mNextMainThreadEnded
;
72 * This represents a message run on the graph thread to modify track or graph
73 * state. These are passed from main thread to graph thread through
74 * AppendMessage(), or scheduled on the graph thread with
75 * RunMessageAfterProcessing(). A ControlMessage
76 * always has a weak reference to a particular affected track.
78 class ControlMessage
{
80 explicit ControlMessage(MediaTrack
* aTrack
) : mTrack(aTrack
) {
81 MOZ_COUNT_CTOR(ControlMessage
);
83 // All these run on the graph thread
84 MOZ_COUNTED_DTOR_VIRTUAL(ControlMessage
)
85 // Do the action of this message on the MediaTrackGraph thread. Any actions
86 // affecting graph processing should take effect at mProcessedTime.
87 // All track data for times < mProcessedTime has already been
89 virtual void Run() = 0;
90 // RunDuringShutdown() is only relevant to messages generated on the main
91 // thread (for AppendMessage()).
92 // When we're shutting down the application, most messages are ignored but
93 // some cleanup messages should still be processed (on the main thread).
94 // This must not add new control messages to the graph.
95 virtual void RunDuringShutdown() {}
96 MediaTrack
* GetTrack() { return mTrack
; }
99 // We do not hold a reference to mTrack. The graph will be holding a reference
100 // to the track until the Destroy message is processed. The last message
101 // referencing a track is the Destroy message for that track.
107 nsTArray
<UniquePtr
<ControlMessage
>> mMessages
;
111 * The implementation of a media track graph. This class is private to this
112 * file. It's not in the anonymous namespace because MediaTrack needs to
113 * be able to friend it.
115 * There can be multiple MediaTrackGraph per process: one per document.
116 * Additionaly, each OfflineAudioContext object creates its own MediaTrackGraph
119 class MediaTrackGraphImpl
: public MediaTrackGraph
,
120 public GraphInterface
,
121 public nsIMemoryReporter
,
122 public nsIThreadObserver
,
123 public nsITimerCallback
,
126 NS_DECL_THREADSAFE_ISUPPORTS
127 NS_DECL_NSIMEMORYREPORTER
128 NS_DECL_NSITHREADOBSERVER
129 NS_DECL_NSITIMERCALLBACK
133 * Use aGraphDriverRequested with SYSTEM_THREAD_DRIVER or AUDIO_THREAD_DRIVER
134 * to create a MediaTrackGraph which provides support for real-time audio
135 * and/or video. Set it to OFFLINE_THREAD_DRIVER in order to create a
136 * non-realtime instance which just churns through its inputs and produces
137 * output. Those objects currently only support audio, and are used to
138 * implement OfflineAudioContext. They do not support MediaTrack inputs.
140 explicit MediaTrackGraphImpl(GraphDriverType aGraphDriverRequested
,
141 GraphRunType aRunTypeRequested
,
142 TrackRate aSampleRate
, uint32_t aChannelCount
,
143 CubebUtils::AudioDeviceID aOutputDeviceID
,
144 nsISerialEventTarget
* aMainThread
);
145 static MediaTrackGraphImpl
* GetInstance(
146 GraphDriverType aGraphDriverRequested
, uint64_t aWindowID
,
147 bool aShouldResistFingerprinting
, TrackRate aSampleRate
,
148 CubebUtils::AudioDeviceID aOutputDeviceID
,
149 nsISerialEventTarget
* aMainThread
);
150 static MediaTrackGraphImpl
* GetInstanceIfExists(
151 uint64_t aWindowID
, bool aShouldResistFingerprinting
,
152 TrackRate aSampleRate
, CubebUtils::AudioDeviceID aOutputDeviceID
);
154 // Intended only for assertions, either on graph thread or not running (in
155 // which case we must be on the main thread).
156 bool OnGraphThreadOrNotRunning() const override
;
157 bool OnGraphThread() const override
;
159 bool Destroyed() const override
;
163 * True if we're on aDriver's thread, or if we're on mGraphRunner's thread
164 * and mGraphRunner is currently run by aDriver.
166 bool InDriverIteration(const GraphDriver
* aDriver
) const override
;
170 * Unregisters memory reporting and deletes this instance. This should be
171 * called instead of calling the destructor directly.
177 * This runs every time we need to sync state from the media graph thread
178 * to the main thread while the main thread is not in the middle
179 * of a script. It runs during a "stable state" (per HTML5) or during
180 * an event posted to the main thread.
181 * The boolean affects which boolean controlling runnable dispatch is cleared
183 void RunInStableState(bool aSourceIsMTG
);
185 * Ensure a runnable to run RunInStableState is posted to the appshell to
186 * run at the next stable state (per HTML5).
187 * See EnsureStableStateEventPosted.
189 void EnsureRunInStableState();
191 * Called to apply a TrackUpdate to its track.
193 void ApplyTrackUpdate(TrackUpdate
* aUpdate
) MOZ_REQUIRES(mMonitor
);
195 * Append a ControlMessage to the message queue. This queue is drained
196 * during RunInStableState; the messages will run on the graph thread.
198 virtual void AppendMessage(UniquePtr
<ControlMessage
> aMessage
);
201 * Dispatches a runnable from any thread to the correct main thread for this
204 void Dispatch(already_AddRefed
<nsIRunnable
>&& aRunnable
);
207 * Make this MediaTrackGraph enter forced-shutdown state. This state
208 * will be noticed by the media graph thread, which will shut down all tracks
209 * and other state controlled by the media graph thread.
210 * This is called during application shutdown, and on document unload if an
211 * AudioContext is using the graph.
213 void ForceShutDown();
216 * Sets mShutdownBlocker and makes it block shutdown.
217 * Main thread only. Not idempotent. Returns true if a blocker was added,
218 * false if this failed.
220 bool AddShutdownBlocker();
223 * Removes mShutdownBlocker and unblocks shutdown.
224 * Main thread only. Idempotent.
226 void RemoveShutdownBlocker();
229 * Called before the thread runs.
234 * Respond to CollectReports with sizes collected on the graph thread.
236 static void FinishCollectReports(
237 nsIHandleReportCallback
* aHandleReport
, nsISupports
* aData
,
238 const nsTArray
<AudioNodeSizes
>& aAudioTrackSizes
);
240 // The following methods run on the graph thread (or possibly the main thread
241 // if mLifecycleState > LIFECYCLE_RUNNING)
242 void CollectSizesForMemoryReport(
243 already_AddRefed
<nsIHandleReportCallback
> aHandleReport
,
244 already_AddRefed
<nsISupports
> aHandlerData
);
247 * Returns true if this MediaTrackGraph should keep running
249 bool UpdateMainThreadState();
252 * Proxy method called by GraphDriver to iterate the graph.
253 * If this graph was created with GraphRunType SINGLE_THREAD, mGraphRunner
254 * will take care of calling OneIterationImpl from its thread. Otherwise,
255 * OneIterationImpl is called directly. Output from the graph gets mixed into
256 * aMixer, if it is non-null.
258 IterationResult
OneIteration(GraphTime aStateTime
, GraphTime aIterationEnd
,
259 AudioMixer
* aMixer
) override
;
262 * Returns true if this MediaTrackGraph should keep running
264 IterationResult
OneIterationImpl(GraphTime aStateTime
,
265 GraphTime aIterationEnd
, AudioMixer
* aMixer
);
268 * Called from the driver, when the graph thread is about to stop, to tell
269 * the main thread to attempt to begin cleanup. The main thread may either
270 * shutdown or revive the graph depending on whether it receives new
273 void SignalMainThreadCleanup();
275 /* This is the end of the current iteration, that is, the current time of the
277 GraphTime
IterationEnd() const;
280 * Ensure there is an event posted to the main thread to run RunInStableState.
281 * mMonitor must be held.
282 * See EnsureRunInStableState
284 void EnsureStableStateEventPosted() MOZ_REQUIRES(mMonitor
);
286 * Generate messages to the main thread to update it for all state changes.
287 * mMonitor must be held.
289 void PrepareUpdatesToMainThreadState(bool aFinalUpdate
)
290 MOZ_REQUIRES(mMonitor
);
292 * If we are rendering in non-realtime mode, we don't want to send messages to
293 * the main thread at each iteration for performance reasons. We instead
294 * notify the main thread at the same rate
296 bool ShouldUpdateMainThread();
297 // The following methods are the various stages of RunThread processing.
299 * Advance all track state to mStateComputedTime.
301 void UpdateCurrentTimeForTracks(GraphTime aPrevCurrentTime
);
303 * Process chunks for all tracks and raise events for properties that have
304 * changed, such as principalId.
306 void ProcessChunkMetadata(GraphTime aPrevCurrentTime
);
308 * Process chunks for the given track and interval, and raise events for
309 * properties that have changed, such as principalHandle.
311 template <typename C
, typename Chunk
>
312 void ProcessChunkMetadataForInterval(MediaTrack
* aTrack
, C
& aSegment
,
313 TrackTime aStart
, TrackTime aEnd
);
315 * Process graph messages in mFrontMessageQueue.
317 void RunMessagesInQueue();
319 * Update track processing order and recompute track blocking until
320 * aEndBlockingDecisions.
322 void UpdateGraph(GraphTime aEndBlockingDecisions
);
324 void SwapMessageQueues() MOZ_REQUIRES(mMonitor
) {
325 MOZ_ASSERT(OnGraphThreadOrNotRunning());
326 mMonitor
.AssertCurrentThreadOwns();
327 MOZ_ASSERT(mFrontMessageQueue
.IsEmpty());
328 mFrontMessageQueue
.SwapElements(mBackMessageQueue
);
329 if (!mFrontMessageQueue
.IsEmpty()) {
330 EnsureNextIteration();
334 * Do all the processing and play the audio and video, from
335 * mProcessedTime to mStateComputedTime.
337 void Process(AudioMixer
* aMixer
);
340 * For use during ProcessedMediaTrack::ProcessInput() or
341 * MediaTrackListener callbacks, when graph state cannot be changed.
342 * Schedules |aMessage| to run after processing, at a time when graph state
343 * can be changed. Graph thread.
345 void RunMessageAfterProcessing(UniquePtr
<ControlMessage
> aMessage
);
348 * Resolve the GraphStartedPromise when the driver has started processing on
349 * the audio thread after the device has started.
351 void NotifyWhenGraphStarted(RefPtr
<MediaTrack
> aTrack
,
352 MozPromiseHolder
<GraphStartedPromise
>&& aHolder
);
355 * Apply an AudioContext operation (suspend/resume/close), on the graph
358 void ApplyAudioContextOperationImpl(
359 AudioContextOperationControlMessage
* aMessage
);
362 * Determine if we have any audio tracks, or are about to add any audiotracks.
364 bool AudioTrackPresent();
367 * Schedules a replacement GraphDriver in mNextDriver, if necessary.
372 * Sort mTracks so that every track not in a cycle is after any tracks
373 * it depends on, and every track in a cycle is marked as being in a cycle.
375 void UpdateTrackOrder();
378 * Returns smallest value of t such that t is a multiple of
379 * WEBAUDIO_BLOCK_SIZE and t >= aTime.
381 static GraphTime
RoundUpToEndOfAudioBlock(GraphTime aTime
);
383 * Returns smallest value of t such that t is a multiple of
384 * WEBAUDIO_BLOCK_SIZE and t > aTime.
386 static GraphTime
RoundUpToNextAudioBlock(GraphTime aTime
);
388 * Produce data for all tracks >= aTrackIndex for the current time interval.
389 * Advances block by block, each iteration producing data for all tracks
390 * for a single block.
391 * This is called whenever we have an AudioNodeTrack in the graph.
393 void ProduceDataForTracksBlockByBlock(uint32_t aTrackIndex
,
394 TrackRate aSampleRate
);
396 * If aTrack will underrun between aTime, and aEndBlockingDecisions, returns
397 * the time at which the underrun will start. Otherwise return
398 * aEndBlockingDecisions.
400 GraphTime
WillUnderrun(MediaTrack
* aTrack
, GraphTime aEndBlockingDecisions
);
403 * Given a graph time aTime, convert it to a track time taking into
404 * account the time during which aTrack is scheduled to be blocked.
406 TrackTime
GraphTimeToTrackTimeWithBlocking(const MediaTrack
* aTrack
,
407 GraphTime aTime
) const;
410 * If aTrack needs an audio track but doesn't have one, create it.
411 * If aTrack doesn't need an audio track but has one, destroy it.
413 void CreateOrDestroyAudioTracks(MediaTrack
* aTrack
);
415 * Queue audio (mix of track audio and silence for blocked intervals)
416 * to the audio output track. Returns the number of frames played.
422 struct TrackKeyAndVolume
{
427 TrackTime
PlayAudio(AudioMixer
* aMixer
, const TrackKeyAndVolume
& aTkv
,
428 GraphTime aPlayedTime
);
430 /* Do not call this directly. For users who need to get a DeviceInputTrack,
431 * use DeviceInputTrack::OpenAudio() instead. This should only be used in
432 * DeviceInputTrack to get the existing DeviceInputTrack paired with the given
433 * device in this graph. Main thread only.*/
434 DeviceInputTrack
* GetDeviceInputTrackMainThread(
435 CubebUtils::AudioDeviceID aID
);
437 /* Do not call this directly. This should only be used in DeviceInputTrack to
438 * get the existing NativeInputTrackMain thread only.*/
439 NativeInputTrack
* GetNativeInputTrackMainThread();
441 /* Runs off a message on the graph thread when something requests audio from
442 * an input audio device of ID aID, and delivers the input audio frames to
444 void OpenAudioInputImpl(DeviceInputTrack
* aTrack
);
445 /* Called on the main thread when something requests audio from an input
446 * audio device aID. */
447 virtual void OpenAudioInput(DeviceInputTrack
* aTrack
) override
;
449 /* Runs off a message on the graph when input audio from aID is not needed
450 * anymore, for a particular track. It can be that other tracks still need
451 * audio from this audio input device. */
452 void CloseAudioInputImpl(DeviceInputTrack
* aTrack
);
453 /* Called on the main thread when input audio from aID is not needed
454 * anymore, for a particular track. It can be that other tracks still need
455 * audio from this audio input device. */
456 virtual void CloseAudioInput(DeviceInputTrack
* aTrack
) override
;
458 /* Add or remove an audio output for this track. All tracks that have an
459 * audio output are mixed and written to a single audio output stream. */
460 void RegisterAudioOutput(MediaTrack
* aTrack
, void* aKey
);
461 void UnregisterAudioOutput(MediaTrack
* aTrack
, void* aKey
);
462 void UnregisterAllAudioOutputs(MediaTrack
* aTrack
);
463 void SetAudioOutputVolume(MediaTrack
* aTrack
, void* aKey
, float aVolume
);
465 /* Called on the graph thread when the input device settings should be
466 * reevaluated, for example, if the channel count of the input track should
468 void ReevaluateInputDevice(CubebUtils::AudioDeviceID aID
);
470 /* Called on the graph thread when there is new output data for listeners.
471 * This is the mixed audio output of this MediaTrackGraph. */
472 void NotifyOutputData(AudioDataValue
* aBuffer
, size_t aFrames
,
473 TrackRate aRate
, uint32_t aChannels
) override
;
474 /* Called on the graph thread after an AudioCallbackDriver with an input
475 * stream has stopped. */
476 void NotifyInputStopped() override
;
477 /* Called on the graph thread when there is new input data for listeners. This
478 * is the raw audio input for this MediaTrackGraph. */
479 void NotifyInputData(const AudioDataValue
* aBuffer
, size_t aFrames
,
480 TrackRate aRate
, uint32_t aChannels
,
481 uint32_t aAlreadyBuffered
) override
;
482 /* Called every time there are changes to input/output audio devices like
483 * plug/unplug etc. This can be called on any thread, and posts a message to
484 * the main thread so that it can post a message to the graph thread. */
485 void DeviceChanged() override
;
486 /* Called every time there are changes to input/output audio devices. This is
487 * called on the graph thread. */
488 void DeviceChangedImpl();
491 * Compute how much track data we would like to buffer for aTrack.
493 TrackTime
GetDesiredBufferEnd(MediaTrack
* aTrack
);
495 * Returns true when there are no active tracks.
497 bool IsEmpty() const {
499 OnGraphThreadOrNotRunning() ||
500 (NS_IsMainThread() &&
501 LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
));
502 return mTracks
.IsEmpty() && mSuspendedTracks
.IsEmpty() && mPortCount
== 0;
506 * Add aTrack to the graph and initializes its graph-specific state.
508 void AddTrackGraphThread(MediaTrack
* aTrack
);
510 * Remove aTrack from the graph. Ensures that pending messages about the
511 * track back to the main thread are flushed.
513 void RemoveTrackGraphThread(MediaTrack
* aTrack
);
515 * Remove a track from the graph. Main thread.
517 void RemoveTrack(MediaTrack
* aTrack
);
519 * Remove aPort from the graph and release it.
521 void DestroyPort(MediaInputPort
* aPort
);
523 * Mark the media track order as dirty.
525 void SetTrackOrderDirty() {
526 MOZ_ASSERT(OnGraphThreadOrNotRunning());
527 mTrackOrderDirty
= true;
530 // Get the current maximum channel count. Graph thread only.
531 uint32_t AudioOutputChannelCount() const;
532 // Set a new maximum channel count. Graph thread only.
533 void SetMaxOutputChannelCount(uint32_t aMaxChannelCount
);
535 double AudioOutputLatency();
538 * The audio input channel count for a MediaTrackGraph is the max of all the
539 * channel counts requested by the listeners. The max channel count is
540 * delivered to the listeners themselves, and they take care of downmixing.
542 uint32_t AudioInputChannelCount(CubebUtils::AudioDeviceID aID
);
544 AudioInputType
AudioInputDevicePreference(CubebUtils::AudioDeviceID aID
);
546 double MediaTimeToSeconds(GraphTime aTime
) const {
547 NS_ASSERTION(aTime
> -TRACK_TIME_MAX
&& aTime
<= TRACK_TIME_MAX
,
549 return static_cast<double>(aTime
) / GraphRate();
553 * Signal to the graph that the thread has paused indefinitly,
556 void PausedIndefinitly();
557 void ResumedFromPaused();
560 * Not safe to call off the MediaTrackGraph thread unless monitor is held!
562 GraphDriver
* CurrentDriver() const MOZ_NO_THREAD_SAFETY_ANALYSIS
{
564 if (!OnGraphThreadOrNotRunning()) {
565 mMonitor
.AssertCurrentThreadOwns();
572 * Effectively set the new driver, while we are switching.
573 * It is only safe to call this at the very end of an iteration, when there
574 * has been a SwitchAtNextIteration call during the iteration. The driver
575 * should return and pass the control to the new driver shortly after.
576 * Monitor must be held.
578 void SetCurrentDriver(GraphDriver
* aDriver
) {
579 MOZ_ASSERT_IF(mGraphDriverRunning
, InDriverIteration(mDriver
));
580 MOZ_ASSERT_IF(!mGraphDriverRunning
, NS_IsMainThread());
581 MonitorAutoLock
lock(GetMonitor());
585 GraphDriver
* NextDriver() const {
586 MOZ_ASSERT(OnGraphThread());
590 bool Switching() const { return NextDriver(); }
592 void SwitchAtNextIteration(GraphDriver
* aNextDriver
);
594 Monitor
& GetMonitor() { return mMonitor
; }
596 void EnsureNextIteration() { CurrentDriver()->EnsureNextIteration(); }
598 // Capture API. This allows to get a mixed-down output for a window.
599 void RegisterCaptureTrackForWindow(uint64_t aWindowId
,
600 ProcessedMediaTrack
* aCaptureTrack
);
601 void UnregisterCaptureTrackForWindow(uint64_t aWindowId
);
602 already_AddRefed
<MediaInputPort
> ConnectToCaptureTrack(
603 uint64_t aWindowId
, MediaTrack
* aMediaTrack
);
605 Watchable
<GraphTime
>& CurrentTime() override
;
608 * Interrupt any JS running on the graph thread.
609 * Called on the main thread when shutting down the graph.
617 explicit iterator(MediaTrackGraphImpl
& aGraph
)
618 : mGraph(&aGraph
), mArrayNum(-1), mArrayIndex(0) {
621 iterator() : mGraph(nullptr), mArrayNum(2), mArrayIndex(0) {}
622 MediaTrack
* operator*() { return Array()->ElementAt(mArrayIndex
); }
623 iterator
operator++() {
625 while (mArrayNum
< 2 &&
626 (mArrayNum
< 0 || mArrayIndex
>= Array()->Length())) {
632 bool operator==(const iterator
& aOther
) const {
633 return mArrayNum
== aOther
.mArrayNum
&&
634 mArrayIndex
== aOther
.mArrayIndex
;
636 bool operator!=(const iterator
& aOther
) const {
637 return !(*this == aOther
);
641 nsTArray
<MediaTrack
*>* Array() {
642 return mArrayNum
== 0 ? &mGraph
->mTracks
: &mGraph
->mSuspendedTracks
;
644 MediaTrackGraphImpl
* mGraph
;
646 uint32_t mArrayIndex
;
649 explicit TrackSet(MediaTrackGraphImpl
& aGraph
) : mGraph(aGraph
) {}
650 iterator
begin() { return iterator(mGraph
); }
651 iterator
end() { return iterator(); }
654 MediaTrackGraphImpl
& mGraph
;
656 TrackSet
AllTracks() { return TrackSet(*this); }
661 * If set, the GraphRunner class handles handing over data from audio
662 * callbacks to a common single thread, shared across GraphDrivers.
664 const RefPtr
<GraphRunner
> mGraphRunner
;
667 * Main-thread view of the number of tracks in this graph, for lifetime
670 * When this becomes zero, the graph is marked as forbidden to add more
671 * tracks to. It will be shut down shortly after.
673 size_t mMainThreadTrackCount
= 0;
676 * Main-thread view of the number of ports in this graph, to catch bugs.
678 * When this becomes zero, and mMainThreadTrackCount is 0, the graph is
679 * marked as forbidden to add more ControlMessages to. It will be shut down
682 size_t mMainThreadPortCount
= 0;
685 * Graphs own owning references to their driver, until shutdown. When a driver
686 * switch occur, previous driver is either deleted, or it's ownership is
687 * passed to a event that will take care of the asynchronous cleanup, as
688 * audio track can take some time to shut down.
689 * Accessed on both the main thread and the graph thread; both read and write.
690 * Must hold monitor to access it.
692 RefPtr
<GraphDriver
> mDriver
;
694 // Set during an iteration to switch driver after the iteration has finished.
695 // Should the current iteration be the last iteration, the next driver will be
696 // discarded. Access through SwitchAtNextIteration()/NextDriver(). Graph
698 RefPtr
<GraphDriver
> mNextDriver
;
700 // The following state is managed on the graph thread only, unless
701 // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread
702 // is not running and this state can be used from the main thread.
705 * The graph keeps a reference to each track.
706 * References are maintained manually to simplify reordering without
707 * unnecessary thread-safe refcount changes.
708 * Must satisfy OnGraphThreadOrNotRunning().
710 nsTArray
<MediaTrack
*> mTracks
;
712 * This stores MediaTracks that are part of suspended AudioContexts.
713 * mTracks and mSuspendTracks are disjoint sets: a track is either suspended
714 * or not suspended. Suspended tracks are not ordered in UpdateTrackOrder,
715 * and are therefore not doing any processing.
716 * Must satisfy OnGraphThreadOrNotRunning().
718 nsTArray
<MediaTrack
*> mSuspendedTracks
;
720 * Tracks from mFirstCycleBreaker to the end of mTracks produce output before
721 * they receive input. They correspond to DelayNodes that are in cycles.
723 uint32_t mFirstCycleBreaker
;
725 * Blocking decisions have been computed up to this time.
726 * Between each iteration, this is the same as mProcessedTime.
728 GraphTime mStateComputedTime
= 0;
730 * All track contents have been computed up to this time.
731 * The next batch of updates from the main thread will be processed
732 * at this time. This is behind mStateComputedTime during processing.
734 GraphTime mProcessedTime
= 0;
736 * The end of the current iteration. Only access on the graph thread.
738 GraphTime mIterationEndTime
= 0;
740 * The graph should stop processing at this time.
744 * Date of the last time we updated the main thread with the graph state.
746 TimeStamp mLastMainThreadUpdate
;
748 * Number of active MediaInputPorts
752 * Runnables to run after the next update to main thread state, but that are
753 * still waiting for the next iteration to finish.
755 nsTArray
<nsCOMPtr
<nsIRunnable
>> mPendingUpdateRunnables
;
758 * Devices to use for cubeb output, or nullptr for default device.
759 * A MediaTrackGraph always has an output (even if silent).
761 * All mOutputDeviceID access is on the graph thread.
763 CubebUtils::AudioDeviceID mOutputDeviceID
;
766 * List of resume operations waiting for a switch to an AudioCallbackDriver.
768 class PendingResumeOperation
{
770 explicit PendingResumeOperation(
771 AudioContextOperationControlMessage
* aMessage
);
772 void Apply(MediaTrackGraphImpl
* aGraph
);
774 MediaTrack
* DestinationTrack() const { return mDestinationTrack
; }
777 RefPtr
<MediaTrack
> mDestinationTrack
;
778 nsTArray
<RefPtr
<MediaTrack
>> mTracks
;
779 MozPromiseHolder
<AudioContextOperationPromise
> mHolder
;
781 AutoTArray
<PendingResumeOperation
, 1> mPendingResumeOperations
;
783 // mMonitor guards the data below.
784 // MediaTrackGraph normally does its work without holding mMonitor, so it is
785 // not safe to just grab mMonitor from some thread and start monkeying with
786 // the graph. Instead, communicate with the graph thread using provided
787 // mechanisms such as the ControlMessage queue.
790 // Data guarded by mMonitor (must always be accessed with mMonitor held,
791 // regardless of the value of mLifecycleState).
794 * State to copy to main thread
796 nsTArray
<TrackUpdate
> mTrackUpdates
MOZ_GUARDED_BY(mMonitor
);
798 * Runnables to run after the next update to main thread state.
800 nsTArray
<nsCOMPtr
<nsIRunnable
>> mUpdateRunnables
MOZ_GUARDED_BY(mMonitor
);
802 * A list of batches of messages to process. Each batch is processed
806 * Message queue processed by the MTG thread during an iteration.
807 * Accessed on graph thread only.
809 nsTArray
<MessageBlock
> mFrontMessageQueue
;
811 * Message queue in which the main thread appends messages.
812 * Access guarded by mMonitor.
814 nsTArray
<MessageBlock
> mBackMessageQueue
MOZ_GUARDED_BY(mMonitor
);
816 /* True if there will messages to process if we swap the message queues. */
817 bool MessagesQueued() const MOZ_REQUIRES(mMonitor
) {
818 mMonitor
.AssertCurrentThreadOwns();
819 return !mBackMessageQueue
.IsEmpty();
822 * This enum specifies where this graph is in its lifecycle. This is used
823 * to control shutdown.
824 * Shutdown is tricky because it can happen in two different ways:
825 * 1) Shutdown due to inactivity. RunThread() detects that it has no
826 * pending messages and no tracks, and exits. The next RunInStableState()
827 * checks if there are new pending messages from the main thread (true only
828 * if new track creation raced with shutdown); if there are, it revives
829 * RunThread(), otherwise it commits to shutting down the graph. New track
830 * creation after this point will create a new graph. An async event is
831 * dispatched to Shutdown() the graph's threads and then delete the graph
833 * 2) Forced shutdown at application shutdown, completion of a non-realtime
834 * graph, or document unload. A flag is set, RunThread() detects the flag
835 * and exits, the next RunInStableState() detects the flag, and dispatches
836 * the async event to Shutdown() the graph's threads. However the graph
837 * object is not deleted. New messages for the graph are processed
838 * synchronously on the main thread if necessary. When the last track is
839 * destroyed, the graph object is deleted.
841 * This should be kept in sync with the LifecycleState_str array in
842 * MediaTrackGraph.cpp
844 enum LifecycleState
{
845 // The graph thread hasn't started yet.
846 LIFECYCLE_THREAD_NOT_STARTED
,
847 // RunThread() is running normally.
849 // In the following states, the graph thread is not running so
850 // all "graph thread only" state in this class can be used safely
851 // on the main thread.
852 // RunThread() has exited and we're waiting for the next
853 // RunInStableState(), at which point we can clean up the main-thread
854 // side of the graph.
855 LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP
,
856 // RunInStableState() posted a ShutdownRunnable, and we're waiting for it
857 // to shut down the graph thread(s).
858 LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN
,
859 // Graph threads have shut down but we're waiting for remaining tracks
860 // to be destroyed. Only happens during application shutdown and on
861 // completed non-realtime graphs, since normally we'd only shut down a
862 // realtime graph when it has no tracks.
863 LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
867 * Modified only in mMonitor. Transitions to
868 * LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP occur on the graph thread at
869 * the end of an iteration. All other transitions occur on the main thread.
871 LifecycleState mLifecycleState
MOZ_GUARDED_BY(mMonitor
);
872 LifecycleState
& LifecycleStateRef() MOZ_NO_THREAD_SAFETY_ANALYSIS
{
874 if (mGraphDriverRunning
) {
875 mMonitor
.AssertCurrentThreadOwns();
877 MOZ_ASSERT(NS_IsMainThread());
880 return mLifecycleState
;
882 const LifecycleState
& LifecycleStateRef() const
883 MOZ_NO_THREAD_SAFETY_ANALYSIS
{
885 if (mGraphDriverRunning
) {
886 mMonitor
.AssertCurrentThreadOwns();
888 MOZ_ASSERT(NS_IsMainThread());
891 return mLifecycleState
;
895 * True once the graph thread has received the message from ForceShutDown().
896 * This is checked in the decision to shut down the
897 * graph thread so that control messages dispatched before forced shutdown
898 * are processed on the graph thread.
899 * Only set on the graph thread.
900 * Can be read safely on the thread currently owning the graph, as indicated
901 * by mLifecycleState.
903 bool mForceShutDownReceived
= false;
905 * true when InterruptJS() has been called, because shutdown (normal or
906 * forced) has commenced. Set on the main thread under mMonitor and read on
907 * the graph thread under mMonitor.
909 bool mInterruptJSCalled
MOZ_GUARDED_BY(mMonitor
) = false;
912 * Remove this blocker to unblock shutdown.
913 * Only accessed on the main thread.
915 RefPtr
<media::ShutdownBlocker
> mShutdownBlocker
;
918 * True when we have posted an event to the main thread to run
919 * RunInStableState() and the event hasn't run yet.
920 * Accessed on both main and MTG thread, mMonitor must be held.
922 bool mPostedRunInStableStateEvent
MOZ_GUARDED_BY(mMonitor
);
925 * The JSContext of the graph thread. Set under mMonitor on only the graph
926 * or GraphRunner thread. Once set this does not change until reset when
927 * the thread is about to exit. Read under mMonitor on the main thread to
928 * interrupt running JS for forced shutdown.
930 JSContext
* mJSContext
MOZ_GUARDED_BY(mMonitor
) = nullptr;
935 * Messages posted by the current event loop task. These are forwarded to
936 * the media graph thread during RunInStableState. We can't forward them
937 * immediately because we want all messages between stable states to be
938 * processed as an atomic batch.
940 nsTArray
<UniquePtr
<ControlMessage
>> mCurrentTaskMessageQueue
;
942 * True from when RunInStableState sets mLifecycleState to LIFECYCLE_RUNNING,
943 * until RunInStableState has determined that mLifecycleState is >
946 Atomic
<bool> mGraphDriverRunning
;
948 * True when a stable state runner has been posted to the appshell to run
949 * RunInStableState at the next stable state.
950 * Only accessed on the main thread.
952 bool mPostedRunInStableState
;
954 * True when processing real-time audio/video. False when processing
955 * non-realtime audio.
957 const bool mRealtime
;
959 * True when a change has happened which requires us to recompute the track
962 bool mTrackOrderDirty
;
963 const RefPtr
<nsISerialEventTarget
> mMainThread
;
965 // used to limit graph shutdown time
966 // Only accessed on the main thread.
967 nsCOMPtr
<nsITimer
> mShutdownTimer
;
970 virtual ~MediaTrackGraphImpl();
973 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf
)
975 // Set a new native iput device when the current native input device is close.
977 void SetNewNativeInput();
980 * This class uses manual memory management, and all pointers to it are raw
981 * pointers. However, in order for it to implement nsIMemoryReporter, it needs
982 * to implement nsISupports and so be ref-counted. So it maintains a single
983 * nsRefPtr to itself, giving it a ref-count of 1 during its entire lifetime,
984 * and Destroy() nulls this self-reference in order to trigger self-deletion.
986 RefPtr
<MediaTrackGraphImpl
> mSelfRef
;
988 struct WindowAndTrack
{
990 RefPtr
<ProcessedMediaTrack
> mCaptureTrackSink
;
993 * Track for window audio capture.
995 nsTArray
<WindowAndTrack
> mWindowCaptureTracks
;
997 * Tracks that have their audio output mixed and written to an audio output
1000 nsTArray
<TrackKeyAndVolume
> mAudioOutputs
;
1003 * Global volume scale. Used when running tests so that the output is not too
1006 const float mGlobalVolume
;
1010 * Used to assert when AppendMessage() runs ControlMessages synchronously.
1012 bool mCanRunMessagesSynchronously
;
1016 * The graph's main-thread observable graph time.
1017 * Updated by the stable state runnable after each iteration.
1019 Watchable
<GraphTime
> mMainThreadGraphTime
;
1022 * Set based on mProcessedTime at end of iteration.
1023 * Read by stable state runnable on main thread. Protected by mMonitor.
1025 GraphTime mNextMainThreadGraphTime
MOZ_GUARDED_BY(mMonitor
) = 0;
1028 * Cached audio output latency, in seconds. Main thread only. This is reset
1029 * whenever the audio device running this MediaTrackGraph changes.
1031 double mAudioOutputLatency
;
1034 * The max audio output channel count the default audio output device
1035 * supports. This is cached here because it can be expensive to query. The
1036 * cache is invalidated when the device is changed. This is initialized in the
1037 * ctor, and the read/write only on the graph thread.
1039 uint32_t mMaxOutputChannelCount
;
1042 * Manage the native or non-native input device in graph. Main thread only.
1044 DeviceInputTrackManager mDeviceInputTrackManagerMainThread
;
1047 * Manage the native or non-native input device in graph. Graph thread only.
1049 DeviceInputTrackManager mDeviceInputTrackManagerGraphThread
;
1052 } // namespace mozilla
1055 class nsDefaultComparator
<mozilla::MediaTrackGraphImpl::TrackKeyAndVolume
,
1056 mozilla::MediaTrackGraphImpl::TrackAndKey
> {
1059 const mozilla::MediaTrackGraphImpl::TrackKeyAndVolume
& aElement
,
1060 const mozilla::MediaTrackGraphImpl::TrackAndKey
& aTrackAndKey
) const {
1061 return aElement
.mTrack
== aTrackAndKey
.mTrack
&&
1062 aElement
.mKey
== aTrackAndKey
.mKey
;
1066 #endif /* MEDIATRACKGRAPHIMPL_H_ */