1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MOZILLA_MEDIATRACKGRAPH_H_
7 #define MOZILLA_MEDIATRACKGRAPH_H_
9 #include "AudioSampleFormat.h"
10 #include "CubebUtils.h"
11 #include "MainThreadUtils.h"
12 #include "MediaSegment.h"
13 #include "mozilla/LinkedList.h"
14 #include "mozilla/Maybe.h"
15 #include "mozilla/Mutex.h"
16 #include "mozilla/StateWatching.h"
17 #include "mozilla/TaskQueue.h"
18 #include "nsAutoRef.h"
19 #include "nsIRunnable.h"
21 #include <speex/speex_resampler.h>
24 class nsIGlobalObject
;
25 class nsPIDOMWindowInner
;
29 class AudioCaptureTrack
;
30 class CrossGraphTransmitter
;
31 class CrossGraphReceiver
;
32 }; // namespace mozilla
34 extern mozilla::AsyncLogger gMTGTraceLogger
;
37 class nsAutoRefTraits
<SpeexResamplerState
>
38 : public nsPointerRefTraits
<SpeexResamplerState
> {
40 static void Release(SpeexResamplerState
* aState
) {
41 speex_resampler_destroy(aState
);
47 extern LazyLogModule gMediaTrackGraphLog
;
50 enum class AudioContextOperation
;
51 enum class AudioContextOperationFlags
;
52 enum class AudioContextState
: uint8_t;
56 * MediaTrackGraph is a framework for synchronized audio/video processing
57 * and playback. It is designed to be used by other browser components such as
58 * HTML media elements, media capture APIs, real-time media streaming APIs,
59 * multitrack media APIs, and advanced audio APIs.
61 * The MediaTrackGraph uses a dedicated thread to process media --- the media
62 * graph thread. This ensures that we can process media through the graph
63 * without blocking on main-thread activity. The media graph is only modified
64 * on the media graph thread, to ensure graph changes can be processed without
65 * interfering with media processing. All interaction with the media graph
66 * thread is done with message passing.
68 * APIs that modify the graph or its properties are described as "control APIs".
69 * These APIs are asynchronous; they queue graph changes internally and
70 * those changes are processed all-at-once by the MediaTrackGraph. The
71 * MediaTrackGraph monitors the main thread event loop via
72 * nsIAppShell::RunInStableState to ensure that graph changes from a single
73 * event loop task are always processed all together. Control APIs should only
74 * be used on the main thread, currently; we may be able to relax that later.
76 * To allow precise synchronization of times in the control API, the
77 * MediaTrackGraph maintains a "media timeline". Control APIs that take or
78 * return times use that timeline. Those times never advance during
79 * an event loop task. This time is returned by
80 * MediaTrackGraph::GetCurrentTime().
82 * Media decoding, audio processing and media playback use thread-safe APIs to
83 * the media graph to ensure they can continue while the main thread is blocked.
85 * When the graph is changed, we may need to throw out buffered data and
86 * reprocess it. This is triggered automatically by the MediaTrackGraph.
89 class AudioInputTrack
;
90 class AudioNodeEngine
;
91 class AudioNodeExternalInputTrack
;
93 class DirectMediaTrackListener
;
94 class ForwardedInputTrack
;
97 class MediaTrackGraph
;
98 class MediaTrackGraphImpl
;
99 class MediaTrackListener
;
100 class ProcessedMediaTrack
;
101 class SourceMediaTrack
;
103 class AudioDataListenerInterface
{
105 // Protected destructor, to discourage deletion outside of Release():
106 virtual ~AudioDataListenerInterface() = default;
109 /* These are for cubeb audio input & output streams: */
111 * Output data to speakers, for use as the "far-end" data for echo
112 * cancellation. This is not guaranteed to be in any particular size
115 virtual void NotifyOutputData(MediaTrackGraphImpl
* aGraph
,
116 AudioDataValue
* aBuffer
, size_t aFrames
,
117 TrackRate aRate
, uint32_t aChannels
) = 0;
119 * An AudioCallbackDriver with an input stream signaling that it has stopped
120 * for any reason and the AudioDataListener will not be notified of input data
121 * until the driver is restarted or another driver has started.
123 virtual void NotifyInputStopped(MediaTrackGraphImpl
* aGraph
) = 0;
125 * Input data from a microphone (or other audio source. This is not
126 * guaranteed to be in any particular size chunks.
128 virtual void NotifyInputData(MediaTrackGraphImpl
* aGraph
,
129 const AudioDataValue
* aBuffer
, size_t aFrames
,
130 TrackRate aRate
, uint32_t aChannels
,
131 uint32_t aAlreadyBuffered
) = 0;
134 * Number of audio input channels.
136 virtual uint32_t RequestedInputChannelCount(MediaTrackGraphImpl
* aGraph
) = 0;
139 * Whether the underlying audio device is used for voice input.
141 virtual bool IsVoiceInput(MediaTrackGraphImpl
* aGraph
) const = 0;
143 * Called when the underlying audio device has changed.
145 virtual void DeviceChanged(MediaTrackGraphImpl
* aGraph
) = 0;
148 * Called when the underlying audio device is being closed.
150 virtual void Disconnect(MediaTrackGraphImpl
* aGraph
) = 0;
153 class AudioDataListener
: public AudioDataListenerInterface
{
155 // Protected destructor, to discourage deletion outside of Release():
156 virtual ~AudioDataListener() = default;
159 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener
)
163 * This is a base class for main-thread listener callbacks.
164 * This callback is invoked on the main thread when the main-thread-visible
165 * state of a track has changed.
167 * These methods are called with the media graph monitor held, so
168 * reentry into general media graph methods is not possible.
169 * You should do something non-blocking and non-reentrant (e.g. dispatch an
170 * event) and return. NS_DispatchToCurrentThread would be a good choice.
171 * The listener is allowed to synchronously remove itself from the track, but
172 * not add or remove any other listeners.
174 class MainThreadMediaTrackListener
{
176 virtual void NotifyMainThreadTrackEnded() = 0;
180 * Helper struct used to keep track of memory usage by AudioNodes.
182 struct AudioNodeSizes
{
183 AudioNodeSizes() : mTrack(0), mEngine(0), mNodeType() {}
186 const char* mNodeType
;
190 * Describes how a track should be disabled.
192 * ENABLED Not disabled.
193 * SILENCE_BLACK Audio data is turned into silence, video frames are made
195 * SILENCE_FREEZE Audio data is turned into silence, video freezes at
198 enum class DisabledTrackMode
{ ENABLED
, SILENCE_BLACK
, SILENCE_FREEZE
};
201 * A track of audio or video data. The media type must be known at construction
202 * and cannot change. All tracks progress at the same rate --- "real time".
203 * Tracks cannot seek. The only operation readers can perform on a track is to
204 * read the next data.
206 * Consumers of a track can be reading from it at different offsets, but that
207 * should only happen due to the order in which consumers are being run.
208 * Those offsets must not diverge in the long term, otherwise we would require
209 * unbounded buffering.
211 * (DEPRECATED to be removed in bug 1581074)
212 * Tracks can be in a "blocked" state. While blocked, a track does not
213 * produce data. A track can be explicitly blocked via the control API,
214 * or implicitly blocked by whatever's generating it (e.g. an underrun in the
215 * source resource), or implicitly blocked because something consuming it
216 * blocks, or implicitly because it has ended.
218 * A track can be in an "ended" state. "Ended" tracks are permanently blocked.
219 * The "ended" state is terminal.
221 * Transitions into and out of the "blocked" and "ended" states are managed
222 * by the MediaTrackGraph on the media graph thread.
224 * We buffer media data ahead of the consumers' reading offsets. It is possible
225 * to have buffered data but still be blocked.
227 * Any track can have its audio or video playing when requested. The media
228 * track graph plays audio by constructing audio output tracks as necessary.
229 * Video is played through a DirectMediaTrackListener managed by
232 * The data in a track is managed by mSegment. The segment starts at GraphTime
233 * mStartTime and encodes its own TrackTime duration.
235 * Tracks are explicitly managed. The client creates them via
236 * MediaTrackGraph::Create{Source|ForwardedInput}Track, and releases them by
237 * calling Destroy() when no longer needed (actual destruction will be
238 * deferred). The actual object is owned by the MediaTrackGraph. The basic idea
239 * is that main thread objects will keep Tracks alive as long as necessary
240 * (using the cycle collector to clean up whenever needed).
242 * We make them refcounted only so that track-related messages with
243 * MediaTrack* pointers can be sent to the main thread safely.
245 * The lifetimes of MediaTracks are controlled from the main thread.
246 * For MediaTracks exposed to the DOM, the lifetime is controlled by the DOM
247 * wrapper; the DOM wrappers own their associated MediaTracks. When a DOM
248 * wrapper is destroyed, it sends a Destroy message for the associated
249 * MediaTrack and clears its reference (the last main-thread reference to
250 * the object). When the Destroy message is processed on the graph thread we
251 * immediately release the affected objects (disentangling them from other
252 * objects as necessary).
254 * This could cause problems for media processing if a MediaTrack is destroyed
255 * while a downstream MediaTrack is still using it. Therefore the DOM wrappers
256 * must keep upstream MediaTracks alive as long as they could be used in the
259 * At any time, however, a set of MediaTrack wrappers could be collected via
260 * cycle collection. Destroy messages will be sent for those objects in
261 * arbitrary order and the MediaTrackGraph has to be able to handle this.
263 class MediaTrack
: public mozilla::LinkedListElement
<MediaTrack
> {
265 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaTrack
)
267 MediaTrack(TrackRate aSampleRate
, MediaSegment::Type aType
,
268 MediaSegment
* aSegment
);
270 // The sample rate of the graph.
271 const TrackRate mSampleRate
;
272 const MediaSegment::Type mType
;
275 // Protected destructor, to discourage deletion outside of Release():
276 virtual ~MediaTrack();
280 * Returns the graph that owns this track.
282 MediaTrackGraphImpl
* GraphImpl();
283 const MediaTrackGraphImpl
* GraphImpl() const;
284 MediaTrackGraph
* Graph();
285 const MediaTrackGraph
* Graph() const;
287 * Sets the graph that owns this track. Should only be called once.
289 void SetGraphImpl(MediaTrackGraphImpl
* aGraph
);
290 void SetGraphImpl(MediaTrackGraph
* aGraph
);
293 virtual void AddAudioOutput(void* aKey
);
294 virtual void SetAudioOutputVolume(void* aKey
, float aVolume
);
295 virtual void RemoveAudioOutput(void* aKey
);
296 // Explicitly suspend. Useful for example if a media element is pausing
297 // and we need to stop its track emitting its buffered data. As soon as the
298 // Suspend message reaches the graph, the track stops processing. It
299 // ignores its inputs and produces silence/no video until Resumed. Its
300 // current time does not advance.
301 virtual void Suspend();
302 virtual void Resume();
303 // Events will be dispatched by calling methods of aListener.
304 virtual void AddListener(MediaTrackListener
* aListener
);
305 virtual RefPtr
<GenericPromise
> RemoveListener(MediaTrackListener
* aListener
);
308 * Adds aListener to the source track of this track.
309 * When the MediaTrackGraph processes the added listener, it will traverse
310 * the graph and add it to the track's source track.
311 * Note that the listener will be notified on the MediaTrackGraph thread
312 * with whether the installation of it at the source was successful or not.
314 virtual void AddDirectListener(DirectMediaTrackListener
* aListener
);
317 * Removes aListener from the source track of this track.
318 * Note that the listener has already been removed if the link between the
319 * source and this track has been broken. The caller doesn't have to care
320 * about this, removing when the source cannot be found, or when the listener
321 * had already been removed does nothing.
323 virtual void RemoveDirectListener(DirectMediaTrackListener
* aListener
);
325 // A disabled track has video replaced by black, and audio replaced by
327 void SetDisabledTrackMode(DisabledTrackMode aMode
);
329 // End event will be notified by calling methods of aListener. It is the
330 // responsibility of the caller to remove aListener before it is destroyed.
331 void AddMainThreadListener(MainThreadMediaTrackListener
* aListener
);
332 // It's safe to call this even if aListener is not currently a listener;
333 // the call will be ignored.
334 void RemoveMainThreadListener(MainThreadMediaTrackListener
* aListener
) {
335 MOZ_ASSERT(NS_IsMainThread());
336 MOZ_ASSERT(aListener
);
337 mMainThreadListeners
.RemoveElement(aListener
);
341 * Ensure a runnable will run on the main thread after running all pending
342 * updates that were sent from the graph thread or will be sent before the
343 * graph thread receives the next graph update.
345 * If the graph has been shut down or destroyed, then the runnable will be
346 * dispatched to the event queue immediately. (There are no pending updates
347 * in this situation.)
351 void RunAfterPendingUpdates(already_AddRefed
<nsIRunnable
> aRunnable
);
353 // Signal that the client is done with this MediaTrack. It will be deleted
355 virtual void Destroy();
357 // Returns the main-thread's view of how much data has been processed by
359 TrackTime
GetCurrentTime() const {
360 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
361 return mMainThreadCurrentTime
;
363 // Return the main thread's view of whether this track has ended.
364 bool IsEnded() const {
365 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
366 return mMainThreadEnded
;
369 bool IsDestroyed() const {
370 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
371 return mMainThreadDestroyed
;
374 friend class MediaTrackGraphImpl
;
375 friend class MediaInputPort
;
376 friend class AudioNodeExternalInputTrack
;
378 virtual AudioInputTrack
* AsAudioInputTrack() { return nullptr; }
379 virtual SourceMediaTrack
* AsSourceTrack() { return nullptr; }
380 virtual ProcessedMediaTrack
* AsProcessedTrack() { return nullptr; }
381 virtual AudioNodeTrack
* AsAudioNodeTrack() { return nullptr; }
382 virtual ForwardedInputTrack
* AsForwardedInputTrack() { return nullptr; }
383 virtual CrossGraphTransmitter
* AsCrossGraphTransmitter() { return nullptr; }
384 virtual CrossGraphReceiver
* AsCrossGraphReceiver() { return nullptr; }
386 // These Impl methods perform the core functionality of the control methods
387 // above, on the media graph thread.
389 * Stop all track activity and disconnect it from all inputs and outputs.
390 * This must be idempotent.
392 virtual void DestroyImpl();
393 TrackTime
GetEnd() const;
394 void SetAudioOutputVolumeImpl(void* aKey
, float aVolume
);
395 void AddAudioOutputImpl(void* aKey
);
396 void RemoveAudioOutputImpl(void* aKey
);
399 * Removes all direct listeners and signals to them that they have been
402 virtual void RemoveAllDirectListenersImpl() {}
403 void RemoveAllResourcesAndListenersImpl();
405 virtual void AddListenerImpl(already_AddRefed
<MediaTrackListener
> aListener
);
406 virtual void RemoveListenerImpl(MediaTrackListener
* aListener
);
407 virtual void AddDirectListenerImpl(
408 already_AddRefed
<DirectMediaTrackListener
> aListener
);
409 virtual void RemoveDirectListenerImpl(DirectMediaTrackListener
* aListener
);
410 virtual void SetDisabledTrackModeImpl(DisabledTrackMode aMode
);
412 void AddConsumer(MediaInputPort
* aPort
) { mConsumers
.AppendElement(aPort
); }
413 void RemoveConsumer(MediaInputPort
* aPort
) {
414 mConsumers
.RemoveElement(aPort
);
416 GraphTime
StartTime() const { return mStartTime
; }
417 bool Ended() const { return mEnded
; }
419 // Returns the current number of channels this track contains if it's an audio
420 // track. Calling this on a video track will trip assertions. Graph thread
422 virtual uint32_t NumberOfChannels() const = 0;
424 // The DisabledTrackMode after combining the explicit mode and that of the
426 virtual DisabledTrackMode
CombinedDisabledMode() const {
427 return mDisabledMode
;
430 template <class SegmentType
>
431 SegmentType
* GetData() const {
435 if (mSegment
->GetType() != SegmentType::StaticType()) {
438 return static_cast<SegmentType
*>(mSegment
.get());
440 MediaSegment
* GetData() const { return mSegment
.get(); }
442 double TrackTimeToSeconds(TrackTime aTime
) const {
443 NS_ASSERTION(0 <= aTime
&& aTime
<= TRACK_TIME_MAX
, "Bad time");
444 return static_cast<double>(aTime
) / mSampleRate
;
446 int64_t TrackTimeToMicroseconds(TrackTime aTime
) const {
447 NS_ASSERTION(0 <= aTime
&& aTime
<= TRACK_TIME_MAX
, "Bad time");
448 return (aTime
* 1000000) / mSampleRate
;
450 TrackTime
SecondsToNearestTrackTime(double aSeconds
) const {
451 NS_ASSERTION(0 <= aSeconds
&& aSeconds
<= TRACK_TICKS_MAX
/ TRACK_RATE_MAX
,
453 return mSampleRate
* aSeconds
+ 0.5;
455 TrackTime
MicrosecondsToTrackTimeRoundDown(int64_t aMicroseconds
) const {
456 return (aMicroseconds
* mSampleRate
) / 1000000;
459 TrackTicks
TimeToTicksRoundUp(TrackRate aRate
, TrackTime aTime
) const {
460 return RateConvertTicksRoundUp(aRate
, mSampleRate
, aTime
);
462 TrackTime
TicksToTimeRoundDown(TrackRate aRate
, TrackTicks aTicks
) const {
463 return RateConvertTicksRoundDown(mSampleRate
, aRate
, aTicks
);
466 * Convert graph time to track time. aTime must be <= mStateComputedTime
467 * to ensure we know exactly how much time this track will be blocked during
470 TrackTime
GraphTimeToTrackTimeWithBlocking(GraphTime aTime
) const;
472 * Convert graph time to track time. This assumes there is no blocking time
473 * to take account of, which is always true except between a track
474 * having its blocking time calculated in UpdateGraph and its blocking time
475 * taken account of in UpdateCurrentTimeForTracks.
477 TrackTime
GraphTimeToTrackTime(GraphTime aTime
) const;
479 * Convert track time to graph time. This assumes there is no blocking time
480 * to take account of, which is always true except between a track
481 * having its blocking time calculated in UpdateGraph and its blocking time
482 * taken account of in UpdateCurrentTimeForTracks.
484 GraphTime
TrackTimeToGraphTime(TrackTime aTime
) const;
486 virtual void ApplyTrackDisabling(MediaSegment
* aSegment
,
487 MediaSegment
* aRawSegment
= nullptr);
489 // Return true if the main thread needs to observe updates from this track.
490 virtual bool MainThreadNeedsUpdates() const { return true; }
492 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const;
493 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf
) const;
495 bool IsSuspended() const { return mSuspendedCount
> 0; }
497 * Increment suspend count and move it to mGraph->mSuspendedTracks if
498 * necessary. Graph thread.
500 void IncrementSuspendCount();
502 * Increment suspend count on aTrack and move it to mGraph->mTracks if
503 * necessary. GraphThread.
505 virtual void DecrementSuspendCount();
508 // Called on graph thread either during destroy handling or before handing
509 // graph control to the main thread to release tracks.
510 virtual void OnGraphThreadDone() {}
512 // |AdvanceTimeVaryingValuesToCurrentTime| will be override in
514 virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime
,
515 GraphTime aBlockedTime
);
517 void NotifyMainThreadListeners() {
518 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
520 for (int32_t i
= mMainThreadListeners
.Length() - 1; i
>= 0; --i
) {
521 mMainThreadListeners
[i
]->NotifyMainThreadTrackEnded();
523 mMainThreadListeners
.Clear();
526 bool ShouldNotifyTrackEnded() {
527 NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
528 if (!mMainThreadEnded
|| mEndedNotificationSent
) {
532 mEndedNotificationSent
= true;
536 // Notifies listeners and consumers of the change in disabled mode when the
537 // current combined mode is different from aMode.
538 void NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode
);
540 // This state is all initialized on the main thread but
541 // otherwise modified only on the media graph thread.
543 // Buffered data. The start of the buffer corresponds to mStartTime.
544 // Conceptually the buffer contains everything this track has ever played,
545 // but we forget some prefix of the buffered data to bound the space usage.
546 // Note that this may be null for tracks that never contain data, like
547 // non-external AudioNodeTracks.
548 const UniquePtr
<MediaSegment
> mSegment
;
550 // The time when the buffered data could be considered to have started
551 // playing. This increases over time to account for time the track was
552 // blocked before mCurrentTime.
553 GraphTime mStartTime
;
555 // The time until which we last called mSegment->ForgetUpTo().
556 TrackTime mForgottenTime
;
558 // True once we've processed mSegment until the end and no more data will be
559 // added. Note that mSegment might still contain data for the current
563 // True after track listeners have been notified that this track has ended.
566 // Client-set volume of this track
567 nsTArray
<RefPtr
<MediaTrackListener
>> mTrackListeners
;
568 nsTArray
<MainThreadMediaTrackListener
*> mMainThreadListeners
;
569 // This track's associated disabled mode. It can either by disabled by frames
570 // being replaced by black, or by retaining the previous frame.
571 DisabledTrackMode mDisabledMode
;
573 // GraphTime at which this track starts blocking.
574 // This is only valid up to mStateComputedTime. The track is considered to
575 // have not been blocked before mCurrentTime (its mStartTime is
576 // increased as necessary to account for that time instead).
577 GraphTime mStartBlocking
;
579 // MediaInputPorts to which this is connected
580 nsTArray
<MediaInputPort
*> mConsumers
;
583 * Number of outstanding suspend operations on this track. Track is
584 * suspended when this is > 0.
586 int32_t mSuspendedCount
;
588 // Main-thread views of state
589 TrackTime mMainThreadCurrentTime
;
590 bool mMainThreadEnded
;
591 bool mEndedNotificationSent
;
592 bool mMainThreadDestroyed
;
594 // Our media track graph. null if destroyed on the graph thread.
595 MediaTrackGraphImpl
* mGraph
;
599 * This is a track into which a decoder can write audio or video.
601 * Audio or video can be written on any thread, but you probably want to
602 * always write from the same thread to avoid unexpected interleavings.
604 * For audio the sample rate of the written data can differ from the sample rate
605 * of the graph itself. Use SetAppendDataSourceRate to inform the track what
606 * rate written audio data will be sampled in.
608 class SourceMediaTrack
: public MediaTrack
{
610 SourceMediaTrack(MediaSegment::Type aType
, TrackRate aSampleRate
);
612 SourceMediaTrack
* AsSourceTrack() override
{ return this; }
617 * Enable or disable pulling.
618 * When pulling is enabled, NotifyPull gets called on the
619 * MediaTrackListeners for this track during the MediaTrackGraph
620 * control loop. Pulling is initially disabled. Due to unavoidable race
621 * conditions, after a call to SetPullingEnabled(false) it is still possible
622 * for a NotifyPull to occur.
624 void SetPullingEnabled(bool aEnabled
);
626 // MediaTrackGraph thread only
627 void DestroyImpl() override
;
629 // Call these on any thread.
631 * Call all MediaTrackListeners to request new data via the NotifyPull
633 * aDesiredUpToTime (in): end time of new data requested.
635 * Returns true if new data is about to be added.
637 bool PullNewData(GraphTime aDesiredUpToTime
);
640 * Extract any state updates pending in the track, and apply them.
642 void ExtractPendingInput(GraphTime aCurrentTime
, GraphTime aDesiredUpToTime
);
645 * All data appended with AppendData() from this point on will be resampled
646 * from aRate to the graph rate.
648 * Resampling for video does not make sense and is forbidden.
650 void SetAppendDataSourceRate(TrackRate aRate
);
653 * Append media data to this track. Ownership of aSegment remains with the
654 * caller, but aSegment is emptied. Returns 0 if the data was not appended
655 * because the stream has ended. Returns the duration of the appended data in
656 * the graph's track rate otherwise.
658 TrackTime
AppendData(MediaSegment
* aSegment
,
659 MediaSegment
* aRawSegment
= nullptr);
662 * Clear any data appended with AppendData() that hasn't entered the graph
663 * yet. Returns the duration of the cleared data in the graph's track rate.
665 TrackTime
ClearFutureData();
668 * Indicate that this track has ended. Do not do any more API calls affecting
673 // Overriding allows us to hold the mMutex lock while changing the track
675 void SetDisabledTrackModeImpl(DisabledTrackMode aMode
) override
;
677 // Overriding allows us to ensure mMutex is locked while changing the track
679 void ApplyTrackDisabling(MediaSegment
* aSegment
,
680 MediaSegment
* aRawSegment
= nullptr) override
{
681 mMutex
.AssertCurrentThreadOwns();
682 MediaTrack::ApplyTrackDisabling(aSegment
, aRawSegment
);
685 uint32_t NumberOfChannels() const override
;
687 void RemoveAllDirectListenersImpl() override
;
689 // The value set here is applied in MoveToSegment so we can avoid the
690 // buffering delay in applying the change. See Bug 1443511.
691 void SetVolume(float aVolume
);
692 float GetVolumeLocked();
694 friend class MediaTrackGraphImpl
;
697 enum TrackCommands
: uint32_t;
699 virtual ~SourceMediaTrack();
702 * Data to cater for appending media data to this track.
705 // Sample rate of the input data.
706 TrackRate mInputRate
;
707 // Resampler if the rate of the input track does not match the
708 // MediaTrackGraph's.
709 nsAutoRef
<SpeexResamplerState
> mResampler
;
710 uint32_t mResamplerChannelCount
;
711 // Each time the track updates are flushed to the media graph thread,
712 // the segment buffer is emptied.
713 UniquePtr
<MediaSegment
> mData
;
714 // True once the producer has signaled that no more data is coming.
716 // True if the producer of this track is having data pulled by the graph.
717 bool mPullingEnabled
;
718 // True if the graph has notified this track that it will not be used
719 // again on the graph thread.
720 bool mGraphThreadDone
;
725 void ResampleAudioToGraphSampleRate(MediaSegment
* aSegment
);
727 void AddDirectListenerImpl(
728 already_AddRefed
<DirectMediaTrackListener
> aListener
) override
;
729 void RemoveDirectListenerImpl(DirectMediaTrackListener
* aListener
) override
;
732 * Notify direct consumers of new data to this track.
733 * The data doesn't have to be resampled (though it may be). This is called
734 * from AppendData on the thread providing the data, and will call
735 * the Listeners on this thread.
737 void NotifyDirectConsumers(MediaSegment
* aSegment
);
739 void OnGraphThreadDone() override
{
740 MutexAutoLock
lock(mMutex
);
744 mUpdateTrack
->mGraphThreadDone
= true;
745 if (!mUpdateTrack
->mData
) {
748 mUpdateTrack
->mData
->Clear();
751 virtual void AdvanceTimeVaryingValuesToCurrentTime(
752 GraphTime aCurrentTime
, GraphTime aBlockedTime
) override
;
754 // This must be acquired *before* MediaTrackGraphImpl's lock, if they are
757 // protected by mMutex
759 UniquePtr
<TrackData
> mUpdateTrack
;
760 nsTArray
<RefPtr
<DirectMediaTrackListener
>> mDirectTrackListeners
;
764 * A ref-counted wrapper of a MediaTrack that allows multiple users to share a
765 * reference to the same MediaTrack with the purpose of being guaranteed that
766 * the graph it is in is kept alive.
768 * Automatically suspended on creation and destroyed on destruction. Main thread
771 struct SharedDummyTrack
{
772 NS_INLINE_DECL_REFCOUNTING(SharedDummyTrack
)
773 explicit SharedDummyTrack(MediaTrack
* aTrack
) : mTrack(aTrack
) {
776 const RefPtr
<MediaTrack
> mTrack
;
779 ~SharedDummyTrack() { mTrack
->Destroy(); }
783 * Represents a connection between a ProcessedMediaTrack and one of its
785 * We make these refcounted so that track-related messages with MediaInputPort*
786 * pointers can be sent to the main thread safely.
788 * When a port's source or destination track dies, the track's DestroyImpl
789 * calls MediaInputPort::Disconnect to disconnect the port from
790 * the source and destination tracks.
792 * The lifetimes of MediaInputPort are controlled from the main thread.
793 * The media graph adds a reference to the port. When a MediaInputPort is no
794 * longer needed, main-thread code sends a Destroy message for the port and
795 * clears its reference (the last main-thread reference to the object). When
796 * the Destroy message is processed on the graph manager thread we disconnect
797 * the port and drop the graph's reference, destroying the object.
799 class MediaInputPort final
{
801 // Do not call this constructor directly. Instead call
802 // aDest->AllocateInputPort.
803 MediaInputPort(MediaTrackGraphImpl
* aGraph
, MediaTrack
* aSource
,
804 ProcessedMediaTrack
* aDest
, uint16_t aInputNumber
,
805 uint16_t aOutputNumber
)
808 mInputNumber(aInputNumber
),
809 mOutputNumber(aOutputNumber
),
811 MOZ_COUNT_CTOR(MediaInputPort
);
814 // Private destructor, to discourage deletion outside of Release():
815 MOZ_COUNTED_DTOR(MediaInputPort
)
818 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort
)
821 * Disconnects and destroys the port. The caller must not reference this
822 * object again. Main thread.
826 // The remaining methods and members must always be called on the graph thread
827 // from within MediaTrackGraph.cpp.
830 // Called during message processing to trigger removal of this port's source
831 // and destination tracks.
834 MediaTrack
* GetSource() const { return mSource
; }
835 ProcessedMediaTrack
* GetDestination() const { return mDest
; }
837 uint16_t InputNumber() const { return mInputNumber
; }
838 uint16_t OutputNumber() const { return mOutputNumber
; }
840 struct InputInterval
{
843 bool mInputIsBlocked
;
845 // Find the next time interval starting at or after aTime during which
846 // aPort->mDest is not blocked and aPort->mSource's blocking status does not
847 // change. A null aPort returns a blocked interval starting at aTime.
848 static InputInterval
GetNextInputInterval(MediaInputPort
const* aPort
,
852 * Returns the graph that owns this port.
854 MediaTrackGraphImpl
* GraphImpl() const;
855 MediaTrackGraph
* Graph() const;
858 * Sets the graph that owns this track. Should only be called once.
860 void SetGraphImpl(MediaTrackGraphImpl
* aGraph
);
863 * Notify the port that the source MediaTrack has been suspended.
868 * Notify the port that the source MediaTrack has been resumed.
872 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const {
882 size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf
) const {
883 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf
);
887 friend class ProcessedMediaTrack
;
888 // Never modified after Init()
890 ProcessedMediaTrack
* mDest
;
891 // The input and output numbers are optional, and are currently only used by
893 const uint16_t mInputNumber
;
894 const uint16_t mOutputNumber
;
896 // Our media track graph
897 MediaTrackGraphImpl
* mGraph
;
901 * This track processes zero or more input tracks in parallel to produce
902 * its output. The details of how the output is produced are handled by
903 * subclasses overriding the ProcessInput method.
905 class ProcessedMediaTrack
: public MediaTrack
{
907 ProcessedMediaTrack(TrackRate aSampleRate
, MediaSegment::Type aType
,
908 MediaSegment
* aSegment
)
909 : MediaTrack(aSampleRate
, aType
, aSegment
),
915 * Allocates a new input port attached to source aTrack.
916 * This port can be removed by calling MediaInputPort::Destroy().
918 already_AddRefed
<MediaInputPort
> AllocateInputPort(
919 MediaTrack
* aTrack
, uint16_t aInputNumber
= 0,
920 uint16_t aOutputNumber
= 0);
922 * Queue a message to set the autoend flag on this track (defaults to
923 * true). When this flag is set, and the input track has ended playout
924 * (including if there is no input track), this track automatically
925 * enters the ended state.
927 virtual void QueueSetAutoend(bool aAutoend
);
929 ProcessedMediaTrack
* AsProcessedTrack() override
{ return this; }
931 friend class MediaTrackGraphImpl
;
933 // Do not call these from outside MediaTrackGraph.cpp!
934 virtual void AddInput(MediaInputPort
* aPort
);
935 virtual void RemoveInput(MediaInputPort
* aPort
) {
936 mInputs
.RemoveElement(aPort
) || mSuspendedInputs
.RemoveElement(aPort
);
938 bool HasInputPort(MediaInputPort
* aPort
) const {
939 return mInputs
.Contains(aPort
) || mSuspendedInputs
.Contains(aPort
);
941 uint32_t InputPortCount() const {
942 return mInputs
.Length() + mSuspendedInputs
.Length();
944 void InputSuspended(MediaInputPort
* aPort
);
945 void InputResumed(MediaInputPort
* aPort
);
946 void DestroyImpl() override
;
947 void DecrementSuspendCount() override
;
949 * This gets called after we've computed the blocking states for all
950 * tracks (mBlocked is up to date up to mStateComputedTime).
951 * Also, we've produced output for all tracks up to this one. If this track
952 * is not in a cycle, then all its source tracks have produced data.
953 * Generate output from aFrom to aTo.
954 * This will be called on tracks that have ended. Most track types should
955 * just return immediately if they're ended, but some may wish to update
956 * internal state (see AudioNodeTrack).
957 * ProcessInput is allowed to set mEnded only if ALLOW_END is in aFlags. (This
958 * flag will be set when aTo >= mStateComputedTime, i.e. when we've produced
959 * the last block of data we need to produce.) Otherwise we can get into a
960 * situation where we've determined the track should not block before
961 * mStateComputedTime, but the track ends before mStateComputedTime, violating
962 * the invariant that ended tracks are blocked.
964 enum { ALLOW_END
= 0x01 };
965 virtual void ProcessInput(GraphTime aFrom
, GraphTime aTo
,
966 uint32_t aFlags
) = 0;
967 void SetAutoendImpl(bool aAutoend
) { mAutoend
= aAutoend
; }
969 // Only valid after MediaTrackGraphImpl::UpdateTrackOrder() has run.
970 // A DelayNode is considered to break a cycle and so this will not return
971 // true for echo loops, only for muted cycles.
972 bool InMutedCycle() const { return mCycleMarker
; }
974 // Used by ForwardedInputTrack to propagate the disabled mode along the graph.
975 virtual void OnInputDisabledModeChanged(DisabledTrackMode aMode
) {}
977 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const override
{
978 size_t amount
= MediaTrack::SizeOfExcludingThis(aMallocSizeOf
);
980 // - mInputs elements
981 // - mSuspendedInputs elements
982 amount
+= mInputs
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
983 amount
+= mSuspendedInputs
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
987 size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf
) const override
{
988 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf
);
992 // This state is all accessed only on the media graph thread.
994 // The list of all inputs that are not currently suspended.
995 nsTArray
<MediaInputPort
*> mInputs
;
996 // The list of all inputs that are currently suspended.
997 nsTArray
<MediaInputPort
*> mSuspendedInputs
;
999 // After UpdateTrackOrder(), mCycleMarker is either 0 or 1 to indicate
1000 // whether this track is in a muted cycle. During ordering it can contain
1001 // other marker values - see MediaTrackGraphImpl::UpdateTrackOrder().
1002 uint32_t mCycleMarker
;
1006 * There is a single MediaTrackGraph per window.
1007 * Additionaly, each OfflineAudioContext object creates its own MediaTrackGraph
1010 class MediaTrackGraph
{
1012 // We ensure that the graph current time advances in multiples of
1013 // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A track that
1014 // never blocks and has the ideal audio rate will produce audio in multiples
1015 // of the block size.
1017 // Initializing a graph that outputs audio can take quite long on some
1018 // platforms. Code that want to output audio at some point can express the
1019 // fact that they will need an audio track at some point by passing
1020 // AUDIO_THREAD_DRIVER when getting an instance of MediaTrackGraph, so that
1021 // the graph starts with the right driver.
1022 enum GraphDriverType
{
1023 AUDIO_THREAD_DRIVER
,
1024 SYSTEM_THREAD_DRIVER
,
1025 OFFLINE_THREAD_DRIVER
1027 // A MediaTrackGraph running an AudioWorklet must always be run from the
1028 // same thread, in order to run js. To acheive this, create the graph with
1029 // a SINGLE_THREAD RunType. DIRECT_DRIVER will run the graph directly off
1030 // the GraphDriver's thread.
1035 static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT
= 20 * 1000;
1036 static const TrackRate REQUEST_DEFAULT_SAMPLE_RATE
= 0;
1037 constexpr static const CubebUtils::AudioDeviceID DEFAULT_OUTPUT_DEVICE
=
1041 static MediaTrackGraph
* GetInstanceIfExists(
1042 nsPIDOMWindowInner
* aWindow
, TrackRate aSampleRate
,
1043 CubebUtils::AudioDeviceID aOutputDeviceID
);
1044 static MediaTrackGraph
* GetInstance(
1045 GraphDriverType aGraphDriverRequested
, nsPIDOMWindowInner
* aWindow
,
1046 TrackRate aSampleRate
, CubebUtils::AudioDeviceID aOutputDeviceID
);
1047 static MediaTrackGraph
* CreateNonRealtimeInstance(
1048 TrackRate aSampleRate
, nsPIDOMWindowInner
* aWindowId
);
1051 void ForceShutDown();
1053 virtual nsresult
OpenAudioInput(CubebUtils::AudioDeviceID aID
,
1054 AudioDataListener
* aListener
) = 0;
1055 virtual void CloseAudioInput(CubebUtils::AudioDeviceID aID
,
1056 AudioDataListener
* aListener
) = 0;
1060 * Create a track that a media decoder (or some other source of
1061 * media data, such as a camera) can write to.
1063 SourceMediaTrack
* CreateSourceTrack(MediaSegment::Type aType
);
1065 * Create a track that will forward data from its input track.
1067 * A TrackUnionStream can have 0 or 1 input streams. Adding more than that is
1070 * A TrackUnionStream will end when autoending is enabled (default) and there
1071 * is no input, or the input's source is ended. If there is no input and
1072 * autoending is disabled, TrackUnionStream will continue to produce silence
1073 * for audio or the last video frame for video.
1075 ProcessedMediaTrack
* CreateForwardedInputTrack(MediaSegment::Type aType
);
1077 * Create a track that will mix all its audio inputs.
1079 AudioCaptureTrack
* CreateAudioCaptureTrack();
1081 CrossGraphTransmitter
* CreateCrossGraphTransmitter(
1082 CrossGraphReceiver
* aReceiver
);
1083 CrossGraphReceiver
* CreateCrossGraphReceiver(TrackRate aTransmitterRate
);
1086 * Add a new track to the graph. Main thread.
1088 void AddTrack(MediaTrack
* aTrack
);
1090 /* From the main thread, ask the MTG to resolve the returned promise when
1091 * the device has started.
1092 * The promise is rejected with NS_ERROR_NOT_AVAILABLE if aTrack
1093 * is destroyed, or NS_ERROR_ILLEGAL_DURING_SHUTDOWN if the graph is shut
1094 * down, before the promise could be resolved.
1095 * (Audio is initially processed in the FallbackDriver's thread while the
1096 * device is starting up.)
1098 using GraphStartedPromise
= GenericPromise
;
1099 RefPtr
<GraphStartedPromise
> NotifyWhenDeviceStarted(MediaTrack
* aTrack
);
1101 /* From the main thread, suspend, resume or close an AudioContext. Calls
1102 * are not counted. Even Resume calls can be more frequent than Suspend
1105 * aTracks are the tracks of all the AudioNodes of the AudioContext that
1106 * need to be suspended or resumed. Suspend and Resume operations on these
1107 * tracks are counted. Resume operations must not outnumber Suspends and a
1108 * track will not resume until the number of Resume operations matches the
1109 * number of Suspends. This array may be empty if, for example, this is a
1110 * second consecutive suspend call and all the nodes are already suspended.
1112 * This can possibly pause the graph thread, releasing system resources, if
1113 * all tracks have been suspended/closed.
1115 * When the operation is complete, the returned promise is resolved.
1117 using AudioContextOperationPromise
=
1118 MozPromise
<dom::AudioContextState
, bool, true>;
1119 RefPtr
<AudioContextOperationPromise
> ApplyAudioContextOperation(
1120 MediaTrack
* aDestinationTrack
, nsTArray
<RefPtr
<MediaTrack
>> aTracks
,
1121 dom::AudioContextOperation aOperation
);
1123 bool IsNonRealtime() const;
1125 * Start processing non-realtime for a specific number of ticks.
1127 void StartNonRealtimeProcessing(uint32_t aTicksToProcess
);
1130 * NotifyJSContext() is called on the graph thread before content script
1133 void NotifyJSContext(JSContext
* aCx
);
1136 * Media graph thread only.
1137 * Dispatches a runnable that will run on the main thread after all
1138 * main-thread track state has been updated, i.e., during stable state.
1140 * Should only be called during MediaTrackListener callbacks or during
1141 * ProcessedMediaTrack::ProcessInput().
1143 * Note that if called during shutdown the runnable will be ignored and
1144 * released on main thread.
1146 void DispatchToMainThreadStableState(already_AddRefed
<nsIRunnable
> aRunnable
);
1149 * Returns graph sample rate in Hz.
1151 TrackRate
GraphRate() const { return mSampleRate
; }
1153 double AudioOutputLatency();
1155 void RegisterCaptureTrackForWindow(uint64_t aWindowId
,
1156 ProcessedMediaTrack
* aCaptureTrack
);
1157 void UnregisterCaptureTrackForWindow(uint64_t aWindowId
);
1158 already_AddRefed
<MediaInputPort
> ConnectToCaptureTrack(
1159 uint64_t aWindowId
, MediaTrack
* aMediaTrack
);
1161 void AssertOnGraphThreadOrNotRunning() const {
1162 MOZ_ASSERT(OnGraphThreadOrNotRunning());
1166 * Returns a watchable of the graph's main-thread observable graph time.
1169 virtual Watchable
<GraphTime
>& CurrentTime() = 0;
1172 * Graph thread function to return the time at which all processing has been
1173 * completed. Some tracks may have performed processing beyond this time.
1175 GraphTime
ProcessedTime() const;
1178 explicit MediaTrackGraph(TrackRate aSampleRate
) : mSampleRate(aSampleRate
) {
1179 MOZ_COUNT_CTOR(MediaTrackGraph
);
1181 MOZ_COUNTED_DTOR_VIRTUAL(MediaTrackGraph
)
1183 // Intended only for assertions, either on graph thread or not running (in
1184 // which case we must be on the main thread).
1185 virtual bool OnGraphThreadOrNotRunning() const = 0;
1186 virtual bool OnGraphThread() const = 0;
1188 // Intended only for internal assertions. Main thread only.
1189 virtual bool Destroyed() const = 0;
1192 * Sample rate at which this graph runs. For real time graphs, this is
1193 * the rate of the audio mixer. For offline graphs, this is the rate specified
1196 const TrackRate mSampleRate
;
1199 } // namespace mozilla
1201 #endif /* MOZILLA_MEDIATRACKGRAPH_H_ */