1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
10 #include "nsIThread.h"
11 #include "AudioBufferUtils.h"
12 #include "AudioMixer.h"
13 #include "AudioSegment.h"
15 #include "mozilla/Atomics.h"
16 #include "mozilla/dom/AudioContext.h"
17 #include "mozilla/DataMutex.h"
18 #include "mozilla/TaskQueue.h"
19 #include "mozilla/StaticPtr.h"
20 #include "WavDumper.h"
27 class nsAutoRefTraits
<cubeb_stream
> : public nsPointerRefTraits
<cubeb_stream
> {
29 static void Release(cubeb_stream
* aStream
) { cubeb_stream_destroy(aStream
); }
34 * Assume we can run an iteration of the MediaTrackGraph loop in this much time
36 * We try to run the control loop at this rate.
38 static const int MEDIA_GRAPH_TARGET_PERIOD_MS
= 10;
41 * Assume that we might miss our scheduled wakeup of the MediaTrackGraph by
44 static const int SCHEDULE_SAFETY_MARGIN_MS
= 10;
47 * Try have this much audio buffered in streams and queued to the hardware.
48 * The maximum delay to the end of the next control loop
49 * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
50 * There is no point in buffering more audio than this in a stream at any
51 * given time (until we add processing).
52 * This is not optimal yet.
54 static const int AUDIO_TARGET_MS
=
55 2 * MEDIA_GRAPH_TARGET_PERIOD_MS
+ SCHEDULE_SAFETY_MARGIN_MS
;
58 * After starting a fallback driver, wait this long before attempting to re-init
59 * the audio stream the first time.
61 static const int AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS
= 10;
64 * The backoff step duration for when to next attempt to re-init the audio
65 * stream is capped at this value.
67 static const int AUDIO_MAX_FALLBACK_BACKOFF_STEP_MS
= 1000;
69 class AudioCallbackDriver
;
72 class OfflineClockDriver
;
73 class SystemClockDriver
;
76 enum class AudioContextOperation
: uint8_t;
79 struct GraphInterface
: public nsISupports
{
81 * Object returned from OneIteration() instructing the iterating GraphDriver
84 * - If the result is StillProcessing: keep the iterations coming.
85 * - If the result is Stop: the driver potentially updates its internal state
86 * and interacts with the graph (e.g., NotifyOutputData), then it must call
87 * Stopped() exactly once.
88 * - If the result is SwitchDriver: the driver updates internal state as for
89 * the Stop result, then it must call Switched() exactly once and start
92 class IterationResult final
{
94 struct StillProcessing
{};
96 explicit Stop(RefPtr
<Runnable
> aStoppedRunnable
)
97 : mStoppedRunnable(std::move(aStoppedRunnable
)) {}
98 Stop(const Stop
&) = delete;
99 Stop(Stop
&& aOther
) noexcept
100 : mStoppedRunnable(std::move(aOther
.mStoppedRunnable
)) {}
101 ~Stop() { MOZ_ASSERT(!mStoppedRunnable
); }
102 RefPtr
<Runnable
> mStoppedRunnable
;
104 mStoppedRunnable
->Run();
105 mStoppedRunnable
= nullptr;
108 struct SwitchDriver
{
109 SwitchDriver(RefPtr
<GraphDriver
> aDriver
,
110 RefPtr
<Runnable
> aSwitchedRunnable
)
111 : mDriver(std::move(aDriver
)),
112 mSwitchedRunnable(std::move(aSwitchedRunnable
)) {}
113 SwitchDriver(const SwitchDriver
&) = delete;
114 SwitchDriver(SwitchDriver
&& aOther
) noexcept
115 : mDriver(std::move(aOther
.mDriver
)),
116 mSwitchedRunnable(std::move(aOther
.mSwitchedRunnable
)) {}
117 ~SwitchDriver() { MOZ_ASSERT(!mSwitchedRunnable
); }
118 RefPtr
<GraphDriver
> mDriver
;
119 RefPtr
<Runnable
> mSwitchedRunnable
;
121 mSwitchedRunnable
->Run();
122 mSwitchedRunnable
= nullptr;
125 Variant
<Undefined
, StillProcessing
, Stop
, SwitchDriver
> mResult
;
127 explicit IterationResult(StillProcessing
&& aArg
)
128 : mResult(std::move(aArg
)) {}
129 explicit IterationResult(Stop
&& aArg
) : mResult(std::move(aArg
)) {}
130 explicit IterationResult(SwitchDriver
&& aArg
) : mResult(std::move(aArg
)) {}
133 IterationResult() : mResult(Undefined()) {}
134 IterationResult(const IterationResult
&) = delete;
135 IterationResult(IterationResult
&&) = default;
137 IterationResult
& operator=(const IterationResult
&) = delete;
138 IterationResult
& operator=(IterationResult
&&) = default;
140 static IterationResult
CreateStillProcessing() {
141 return IterationResult(StillProcessing());
143 static IterationResult
CreateStop(RefPtr
<Runnable
> aStoppedRunnable
) {
144 return IterationResult(Stop(std::move(aStoppedRunnable
)));
146 static IterationResult
CreateSwitchDriver(
147 RefPtr
<GraphDriver
> aDriver
, RefPtr
<Runnable
> aSwitchedRunnable
) {
148 return IterationResult(
149 SwitchDriver(std::move(aDriver
), std::move(aSwitchedRunnable
)));
152 bool IsStillProcessing() const { return mResult
.is
<StillProcessing
>(); }
153 bool IsStop() const { return mResult
.is
<Stop
>(); }
154 bool IsSwitchDriver() const { return mResult
.is
<SwitchDriver
>(); }
157 MOZ_ASSERT(IsStop());
158 mResult
.as
<Stop
>().Stopped();
161 GraphDriver
* NextDriver() const {
162 if (!IsSwitchDriver()) {
165 return mResult
.as
<SwitchDriver
>().mDriver
;
169 MOZ_ASSERT(IsSwitchDriver());
170 mResult
.as
<SwitchDriver
>().Switched();
174 /* Called on the graph thread after an AudioCallbackDriver with an input
175 * stream has stopped. */
176 virtual void NotifyInputStopped() = 0;
177 /* Called on the graph thread when there is new input data for listeners. This
178 * is the raw audio input for this MediaTrackGraph. */
179 virtual void NotifyInputData(const AudioDataValue
* aBuffer
, size_t aFrames
,
180 TrackRate aRate
, uint32_t aChannels
,
181 uint32_t aAlreadyBuffered
) = 0;
182 /* Called on the main thread after an AudioCallbackDriver has attempted an
183 * operation to set aRequestedParams on the cubeb stream. */
184 virtual void NotifySetRequestedInputProcessingParamsResult(
185 AudioCallbackDriver
* aDriver
,
186 cubeb_input_processing_params aRequestedParams
,
187 Result
<cubeb_input_processing_params
, int>&& aResult
) = 0;
188 /* Called every time there are changes to input/output audio devices like
189 * plug/unplug etc. This can be called on any thread, and posts a message to
190 * the main thread so that it can post a message to the graph thread. */
191 virtual void DeviceChanged() = 0;
192 /* Called by GraphDriver to iterate the graph. Mixed audio output from the
193 * graph is passed into aMixerReceiver, if it is non-null. */
194 virtual IterationResult
OneIteration(
195 GraphTime aStateComputedEnd
, GraphTime aIterationEnd
,
196 MixerCallbackReceiver
* aMixerReceiver
) = 0;
198 /* True if we're on aDriver's thread, or if we're on mGraphRunner's thread
199 * and mGraphRunner is currently run by aDriver. */
200 virtual bool InDriverIteration(const GraphDriver
* aDriver
) const = 0;
205 * A driver is responsible for the scheduling of the processing, the thread
206 * management, and give the different clocks to a MediaTrackGraph. This is an
207 * abstract base class. A MediaTrackGraph can be driven by an
208 * OfflineClockDriver, if the graph is offline, or a SystemClockDriver or an
209 * AudioCallbackDriver, if the graph is real time.
210 * A MediaTrackGraph holds an owning reference to its driver.
212 * The lifetime of drivers is a complicated affair. Here are the different
213 * scenarii that can happen:
215 * Starting a MediaTrackGraph with an AudioCallbackDriver
216 * - A new thread T is created, from the main thread.
217 * - On this thread T, cubeb is initialized if needed, and a cubeb_stream is
218 * created and started
219 * - The thread T posts a message to the main thread to terminate itself.
220 * - The graph runs off the audio thread
222 * Starting a MediaTrackGraph with a SystemClockDriver:
223 * - A new thread T is created from the main thread.
224 * - The graph runs off this thread.
226 * Switching from a SystemClockDriver to an AudioCallbackDriver:
227 * - At the end of the MTG iteration, the graph tells the current driver to
228 * switch to an AudioCallbackDriver, which is created and initialized on the
230 * - At the end of the MTG iteration, the SystemClockDriver transfers its timing
231 * info and a reference to itself to the AudioCallbackDriver. It then starts
232 * the AudioCallbackDriver.
233 * - When the AudioCallbackDriver starts, it:
234 * - Starts a fallback SystemClockDriver that runs until the
235 * AudioCallbackDriver is running, in case it takes a long time to start (it
236 * could block on I/O, e.g., negotiating a bluetooth connection).
237 * - Checks if it has been switched from a SystemClockDriver, and if that is
238 * the case, sends a message to the main thread to shut the
239 * SystemClockDriver thread down.
240 * - When the AudioCallbackDriver is running, data callbacks are blocked. The
241 * fallback driver detects this in its callback and stops itself. The first
242 * DataCallback after the fallback driver had stopped goes through.
243 * - The graph now runs off an audio callback.
245 * Switching from an AudioCallbackDriver to a SystemClockDriver:
246 * - At the end of the MTG iteration, the graph tells the current driver to
247 * switch to a SystemClockDriver.
248 * - the AudioCallbackDriver transfers its timing info and a reference to itself
249 * to the SystemClockDriver. A new SystemClockDriver is started from the
250 * current audio thread.
251 * - When starting, the SystemClockDriver checks if it has been switched from an
252 * AudioCallbackDriver. If yes, it creates a new temporary thread to release
253 * the cubeb_streams. This temporary thread closes the cubeb_stream, and then
254 * dispatches a message to the main thread to be terminated.
255 * - The graph now runs off a normal thread.
257 * Two drivers cannot run at the same time for the same graph. The thread safety
258 * of the different members of drivers, and their access pattern is documented
259 * next to the members themselves.
263 using IterationResult
= GraphInterface::IterationResult
;
265 GraphDriver(GraphInterface
* aGraphInterface
, GraphDriver
* aPreviousDriver
,
266 uint32_t aSampleRate
);
268 NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
270 /* Start the graph, init the driver, start the thread.
271 * A driver cannot be started twice, it must be shutdown
272 * before being started again. */
273 virtual void Start() = 0;
274 /* Shutdown GraphDriver */
275 MOZ_CAN_RUN_SCRIPT
virtual void Shutdown() = 0;
276 /* Set the UTF-8 name for system audio streams.
277 * Graph thread, or main thread if the graph is not running. */
278 virtual void SetStreamName(const nsACString
& aStreamName
);
279 /* Rate at which the GraphDriver runs, in ms. This can either be user
280 * controlled (because we are using a {System,Offline}ClockDriver, and decide
281 * how often we want to wakeup/how much we want to process per iteration), or
282 * it can be indirectly set by the latency of the audio backend, and the
283 * number of buffers of this audio backend: say we have four buffers, and 40ms
284 * latency, we will get a callback approximately every 10ms. */
285 virtual uint32_t IterationDuration() = 0;
287 * Signaled by the graph when it needs another iteration. Goes unhandled for
288 * GraphDrivers that are not able to sleep indefinitely (i.e., all drivers but
289 * ThreadedDriver). Can be called on any thread.
291 virtual void EnsureNextIteration() = 0;
293 // Those are simply for accessing the associated pointer. Graph thread only,
294 // or if one is not running, main thread.
295 GraphDriver
* PreviousDriver();
296 void SetPreviousDriver(GraphDriver
* aPreviousDriver
);
298 virtual AudioCallbackDriver
* AsAudioCallbackDriver() { return nullptr; }
299 virtual const AudioCallbackDriver
* AsAudioCallbackDriver() const {
303 virtual OfflineClockDriver
* AsOfflineClockDriver() { return nullptr; }
304 virtual const OfflineClockDriver
* AsOfflineClockDriver() const {
308 virtual SystemClockDriver
* AsSystemClockDriver() { return nullptr; }
309 virtual const SystemClockDriver
* AsSystemClockDriver() const {
314 * Set the state of the driver so it can start at the right point in time,
315 * after switching from another driver.
317 void SetState(const nsACString
& aStreamName
, GraphTime aIterationEnd
,
318 GraphTime aStateComputedTime
);
320 GraphInterface
* Graph() const { return mGraphInterface
; }
323 // True if the current thread is currently iterating the MTG.
324 bool InIteration() const;
326 // True if the current thread is the GraphDriver's thread.
327 virtual bool OnThread() const = 0;
328 // GraphDriver's thread has started and the thread is running.
329 virtual bool ThreadRunning() const = 0;
331 double MediaTimeToSeconds(GraphTime aTime
) const {
332 NS_ASSERTION(aTime
> -TRACK_TIME_MAX
&& aTime
<= TRACK_TIME_MAX
,
334 return static_cast<double>(aTime
) / mSampleRate
;
337 GraphTime
SecondsToMediaTime(double aS
) const {
338 NS_ASSERTION(0 <= aS
&& aS
<= TRACK_TICKS_MAX
/ TRACK_RATE_MAX
,
340 return mSampleRate
* aS
;
343 GraphTime
MillisecondsToMediaTime(int32_t aMS
) const {
344 return RateConvertTicksRoundDown(mSampleRate
, 1000, aMS
);
348 // The UTF-8 name for system audio streams. Graph thread.
349 nsCString mStreamName
;
350 // Time of the end of this graph iteration.
351 GraphTime mIterationEnd
= 0;
352 // Time until which the graph has processed data.
353 GraphTime mStateComputedTime
= 0;
354 // The GraphInterface this driver is currently iterating.
355 const RefPtr
<GraphInterface
> mGraphInterface
;
356 // The sample rate for the graph, and in case of an audio driver, also for the
358 const uint32_t mSampleRate
;
360 // This is non-null only when this driver has recently switched from an other
361 // driver, and has not cleaned it up yet (for example because the audio stream
362 // is currently calling the callback during initialization).
364 // This is written to when changing driver, from the previous driver's thread,
365 // or a thread created for the occasion. This is read each time we need to
366 // check whether we're changing driver (in Switching()), from the graph
368 // This must be accessed using the {Set,Get}PreviousDriver methods.
369 RefPtr
<GraphDriver
> mPreviousDriver
;
371 virtual ~GraphDriver() = default;
374 class MediaTrackGraphInitThreadRunnable
;
377 * This class is a driver that manages its own thread.
379 class ThreadedDriver
: public GraphDriver
{
380 class IterationWaitHelper
{
381 Monitor mMonitor MOZ_UNANNOTATED
;
382 // The below members are guarded by mMonitor.
383 bool mNeedAnotherIteration
= false;
387 IterationWaitHelper() : mMonitor("IterationWaitHelper::mMonitor") {}
390 * If another iteration is needed we wait for aDuration, otherwise we wait
391 * for a wake-up. If a wake-up occurs before aDuration time has passed, we
392 * wait for aDuration nonetheless.
394 void WaitForNextIterationAtLeast(TimeDuration aDuration
) {
395 MonitorAutoLock
lock(mMonitor
);
396 TimeStamp now
= TimeStamp::Now();
397 mWakeTime
= now
+ aDuration
;
399 if (mNeedAnotherIteration
&& now
>= mWakeTime
) {
402 if (mNeedAnotherIteration
) {
403 lock
.Wait(mWakeTime
- now
);
405 lock
.Wait(TimeDuration::Forever());
407 now
= TimeStamp::Now();
409 mWakeTime
= TimeStamp();
410 mNeedAnotherIteration
= false;
414 * Sets mNeedAnotherIteration to true and notifies the monitor, in case a
415 * driver is currently waiting.
417 void EnsureNextIteration() {
418 MonitorAutoLock
lock(mMonitor
);
419 mNeedAnotherIteration
= true;
425 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ThreadedDriver
, override
);
427 ThreadedDriver(GraphInterface
* aGraphInterface
, GraphDriver
* aPreviousDriver
,
428 uint32_t aSampleRate
);
430 void EnsureNextIteration() override
;
431 void Start() override
;
432 MOZ_CAN_RUN_SCRIPT
void Shutdown() override
;
434 * Runs main control loop on the graph thread. Normally a single invocation
435 * of this runs for the entire lifetime of the graph thread.
437 virtual void RunThread();
438 friend class MediaTrackGraphInitThreadRunnable
;
439 uint32_t IterationDuration() override
{ return MEDIA_GRAPH_TARGET_PERIOD_MS
; }
441 nsIThread
* Thread() const { return mThread
; }
443 bool OnThread() const override
{
444 return !mThread
|| mThread
->IsOnCurrentThread();
447 bool ThreadRunning() const override
{ return mThreadRunning
; }
450 /* Waits until it's time to process more data. */
451 void WaitForNextIteration();
452 /* Implementation dependent time the ThreadedDriver should wait between
454 virtual TimeDuration
WaitInterval() = 0;
455 /* When the graph wakes up to do an iteration, implementations return the
456 * range of time that will be processed. This is called only once per
457 * iteration; it may determine the interval from state in a previous
459 virtual MediaTime
GetIntervalForIteration() = 0;
461 virtual ~ThreadedDriver();
463 nsCOMPtr
<nsIThread
> mThread
;
466 // This is true if the thread is running. It is false
467 // before starting the thread and after stopping it.
468 Atomic
<bool> mThreadRunning
;
471 IterationWaitHelper mWaitHelper
;
475 * A SystemClockDriver drives a GraphInterface using a system clock, and waits
476 * using a monitor, between each iteration.
478 class SystemClockDriver
: public ThreadedDriver
{
480 SystemClockDriver(GraphInterface
* aGraphInterface
,
481 GraphDriver
* aPreviousDriver
, uint32_t aSampleRate
);
482 virtual ~SystemClockDriver();
483 SystemClockDriver
* AsSystemClockDriver() override
{ return this; }
484 const SystemClockDriver
* AsSystemClockDriver() const override
{ return this; }
487 /* Return the TimeDuration to wait before the next rendering iteration. */
488 TimeDuration
WaitInterval() override
;
489 MediaTime
GetIntervalForIteration() override
;
492 // Those are only modified (after initialization) on the graph thread. The
493 // graph thread does not run during the initialization.
494 TimeStamp mInitialTimeStamp
;
495 TimeStamp mCurrentTimeStamp
;
496 TimeStamp mLastTimeStamp
;
500 * An OfflineClockDriver runs the graph as fast as possible, without waiting
503 class OfflineClockDriver
: public ThreadedDriver
{
505 OfflineClockDriver(GraphInterface
* aGraphInterface
, uint32_t aSampleRate
,
507 virtual ~OfflineClockDriver();
508 OfflineClockDriver
* AsOfflineClockDriver() override
{ return this; }
509 const OfflineClockDriver
* AsOfflineClockDriver() const override
{
513 void RunThread() override
;
516 TimeDuration
WaitInterval() override
{ return TimeDuration(); }
517 MediaTime
GetIntervalForIteration() override
;
520 // Time, in GraphTime, for each iteration
524 enum class AudioInputType
{ Unknown
, Voice
};
527 * This is a graph driver that is based on callback functions called by the
528 * audio api. This ensures minimal audio latency, because it means there is no
529 * buffering happening: the audio is generated inside the callback.
531 * This design is less flexible than running our own thread:
532 * - We have no control over the thread:
533 * - It cannot block, and it has to run for a shorter amount of time than the
534 * buffer it is going to fill, or an under-run is going to occur (short burst
535 * of silence in the final audio output).
536 * - We can't know for sure when the callback function is going to be called
537 * (although we compute an estimation so we can schedule video frames)
538 * - Creating and shutting the thread down is a blocking operation, that can
539 * take _seconds_ in some cases (because IPC has to be set up, and
540 * sometimes hardware components are involved and need to be warmed up)
541 * - We have no control on how much audio we generate, we have to return exactly
542 * the number of frames asked for by the callback. Since for the Web Audio
543 * API, we have to do block processing at 128 frames per block, we need to
544 * keep a little spill buffer to store the extra frames.
546 class AudioCallbackDriver
: public GraphDriver
, public MixerCallbackReceiver
{
547 using IterationResult
= GraphInterface::IterationResult
;
548 enum class FallbackDriverState
;
549 class FallbackWrapper
;
552 NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DELETE_ON_EVENT_TARGET(
553 AudioCallbackDriver
, mCubebOperationThread
, override
);
555 /** If aInputChannelCount is zero, then this driver is output-only. */
557 GraphInterface
* aGraphInterface
, GraphDriver
* aPreviousDriver
,
558 uint32_t aSampleRate
, uint32_t aOutputChannelCount
,
559 uint32_t aInputChannelCount
, CubebUtils::AudioDeviceID aOutputDeviceID
,
560 CubebUtils::AudioDeviceID aInputDeviceID
, AudioInputType aAudioInputType
,
561 cubeb_input_processing_params aRequestedInputProcessingParams
);
563 void Start() override
;
564 MOZ_CAN_RUN_SCRIPT
void Shutdown() override
;
565 void SetStreamName(const nsACString
& aStreamName
) override
;
567 /* Static wrapper function cubeb calls back. */
568 static long DataCallback_s(cubeb_stream
* aStream
, void* aUser
,
569 const void* aInputBuffer
, void* aOutputBuffer
,
571 static void StateCallback_s(cubeb_stream
* aStream
, void* aUser
,
573 static void DeviceChangedCallback_s(void* aUser
);
575 /* This function is called by the underlying audio backend when a refill is
576 * needed. This is what drives the whole graph when it is used to output
577 * audio. If the return value is exactly aFrames, this function will get
578 * called again. If it is less than aFrames, the stream will go in draining
579 * mode, and this function will not be called again. */
580 long DataCallback(const AudioDataValue
* aInputBuffer
,
581 AudioDataValue
* aOutputBuffer
, long aFrames
);
582 /* This function is called by the underlying audio backend, but is only used
583 * for informational purposes at the moment. */
584 void StateCallback(cubeb_state aState
);
585 /* This is an approximation of the number of millisecond there are between two
586 * iterations of the graph. */
587 uint32_t IterationDuration() override
;
588 /* If the audio stream has started, this does nothing. There will be another
589 * iteration. If there is an active fallback driver, we forward the call so it
591 void EnsureNextIteration() override
;
593 /* This function gets called when the graph has produced the audio frames for
595 void MixerCallback(AudioChunk
* aMixedBuffer
, uint32_t aSampleRate
) override
;
597 AudioCallbackDriver
* AsAudioCallbackDriver() override
{ return this; }
598 const AudioCallbackDriver
* AsAudioCallbackDriver() const override
{
602 uint32_t OutputChannelCount() { return mOutputChannelCount
; }
604 uint32_t InputChannelCount() { return mInputChannelCount
; }
606 AudioInputType
InputDevicePreference() {
607 if (mInputDevicePreference
== CUBEB_DEVICE_PREF_VOICE
) {
608 return AudioInputType::Voice
;
610 return AudioInputType::Unknown
;
613 /* Get the input processing params requested from this driver, so that an
614 * external caller can decide whether it is necessary to call the setter,
615 * since it may allocate or dispatch. */
616 cubeb_input_processing_params
RequestedInputProcessingParams() const;
618 /* Set the input processing params requested from this driver. */
619 void SetRequestedInputProcessingParams(cubeb_input_processing_params aParams
);
621 std::thread::id
ThreadId() const { return mAudioThreadIdInCb
.load(); }
623 /* Called at the beginning of the audio callback to check if the thread id has
625 bool CheckThreadIdChanged();
627 bool OnThread() const override
{
628 return mAudioThreadIdInCb
.load() == std::this_thread::get_id();
631 /* Returns true if this driver has started (perhaps with a fallback driver)
632 * and not yet stopped. */
633 bool ThreadRunning() const override
{
634 return mAudioStreamState
== AudioStreamState::Running
||
635 mFallbackDriverState
== FallbackDriverState::Running
;
638 /* Whether the underlying cubeb stream has been started and has not stopped
640 bool IsStarted() { return mAudioStreamState
> AudioStreamState::Starting
; };
642 // Returns the output latency for the current audio output stream.
643 TimeDuration
AudioOutputLatency();
645 /* Returns true if this driver has a fallback driver and handover to the audio
646 * callback has not been completed. */
647 bool HasFallback() const;
648 /* Returns true if this driver is currently driven by the fallback driver. */
649 bool OnFallback() const;
653 * On certain MacBookPro, the microphone is located near the left speaker.
654 * We need to pan the sound output to the right speaker if we are using the
655 * mic and the built-in speaker, or we will have terrible echo. */
656 void PanOutputIfNeeded(bool aMicrophoneActive
);
658 * This is called when the output device used by the cubeb stream changes. */
659 void DeviceChangedCallback();
660 /* Start the cubeb stream */
662 friend class MediaTrackGraphInitThreadRunnable
;
663 void Init(const nsCString
& aStreamName
);
664 void SetCubebStreamName(const nsCString
& aStreamName
);
666 /* After the requested input processing params has changed, this applies them
667 * on the cubeb stream. */
668 void SetInputProcessingParams(cubeb_input_processing_params aParams
);
669 /* Calls FallbackToSystemClockDriver() if in FallbackDriverState::None.
670 * Returns Ok(true) if the fallback driver was started, or the old
671 * FallbackDriverState in an Err otherwise. */
672 Result
<bool, FallbackDriverState
> TryStartingFallbackDriver();
673 /* Fall back to a SystemClockDriver using a normal thread. If needed, the
674 * graph will try to re-open an audio stream later. */
675 void FallbackToSystemClockDriver();
676 /* Called by the fallback driver when it has fully stopped, after finishing
677 * its last iteration. If it stopped after the audio stream started, aState
678 * will be None. If it stopped after the graph told it to stop, or switch,
679 * aState will be Stopped. Hands over state to the audio driver that may
680 * iterate the graph after this has been called. */
681 void FallbackDriverStopped(GraphTime aIterationEnd
,
682 GraphTime aStateComputedTime
,
683 FallbackDriverState aState
);
685 /* Called at the end of the fallback driver's iteration to see whether we
686 * should attempt to start the AudioStream again. */
687 void MaybeStartAudioStream();
689 /* This is true when the method is executed on CubebOperation thread pool. */
690 bool OnCubebOperationThread() {
691 return mCubebOperationThread
->IsOnCurrentThreadInfallible();
694 /* MediaTrackGraphs are always down/up mixed to output channels. */
695 const uint32_t mOutputChannelCount
;
696 /* The size of this buffer comes from the fact that some audio backends can
697 * call back with a number of frames lower than one block (128 frames), so we
698 * need to keep at most two block in the SpillBuffer, because we always round
699 * up to block boundaries during an iteration.
700 * This is only ever accessed on the audio callback thread. */
701 SpillBuffer
<AudioDataValue
, WEBAUDIO_BLOCK_SIZE
* 2> mScratchBuffer
;
702 /* Wrapper to ensure we write exactly the number of frames we need in the
703 * audio buffer cubeb passes us. This is only ever accessed on the audio
704 * callback thread. */
705 AudioCallbackBufferWrapper
<AudioDataValue
> mBuffer
;
706 // mAudioStream (a cubeb_stream) has a bare pointer to the cubeb context, so
707 // we hold a strong reference on its behalf.
708 RefPtr
<CubebUtils::CubebHandle
> mCubeb
;
709 /* cubeb stream for this graph. This is non-null after a successful
710 * cubeb_stream_init(). CubebOperation thread only. */
711 nsAutoRef
<cubeb_stream
> mAudioStream
;
712 /* The number of input channels from cubeb. Set before opening cubeb. If it is
713 * zero then the driver is output-only. */
714 const uint32_t mInputChannelCount
;
716 * Devices to use for cubeb input & output, or nullptr for default device.
718 const CubebUtils::AudioDeviceID mOutputDeviceID
;
719 const CubebUtils::AudioDeviceID mInputDeviceID
;
720 /* Approximation of the time between two callbacks. This is used to schedule
721 * video frames. This is in milliseconds. Only even used (after
722 * inizatialization) on the audio callback thread. */
723 uint32_t mIterationDurationMS
;
725 struct AutoInCallback
{
726 explicit AutoInCallback(AudioCallbackDriver
* aDriver
);
728 AudioCallbackDriver
* mDriver
;
731 static already_AddRefed
<TaskQueue
> CreateTaskQueue();
733 /* Shared thread pool with up to one thread for off-main-thread
734 * initialization and shutdown of the audio stream and for other tasks that
735 * must run serially for access to mAudioStream. */
736 const RefPtr
<TaskQueue
> mCubebOperationThread
;
737 cubeb_device_pref mInputDevicePreference
;
738 /* Params that have been attempted to set on mAudioStream, after filtering by
739 * supported processing params. Cubeb operation thread only. */
740 cubeb_input_processing_params mConfiguredInputProcessingParams
=
741 CUBEB_INPUT_PROCESSING_PARAM_NONE
;
742 /* The input processing params requested from this audio driver. Once started,
743 * audio callback thread only. */
744 cubeb_input_processing_params mRequestedInputProcessingParams
;
745 /* Contains the id of the audio thread, from profiler_current_thread_id. */
746 std::atomic
<ProfilerThreadId
> mAudioThreadId
;
747 /* This allows implementing AutoInCallback. This is equal to the current
748 * thread id when in an audio callback, and is an invalid thread id otherwise.
750 std::atomic
<std::thread::id
> mAudioThreadIdInCb
;
751 /* State of the audio stream, see inline comments. */
752 enum class AudioStreamState
{
753 /* There is no cubeb_stream or mAudioStream is in CUBEB_STATE_ERROR or
754 * CUBEB_STATE_STOPPED and no pending task exists to Init() a new
757 /* A task to Init() a new cubeb_stream is pending. */
759 /* cubeb_start_stream() is about to be or has been called on mAudioStream.
760 * Any previous cubeb_streams have been destroyed. */
762 /* mAudioStream has advertised it will change device. In this state we
763 ignore all data callbacks until the fallback driver has started. */
765 /* mAudioStream is running. */
767 /* mAudioStream is draining, and will soon stop. */
770 Atomic
<AudioStreamState
> mAudioStreamState
{AudioStreamState::None
};
771 /* State of the fallback driver, see inline comments. */
772 enum class FallbackDriverState
{
773 /* There is no fallback driver. */
775 /* There is a fallback driver trying to iterate us. */
777 /* There was a fallback driver and the graph stopped it. No audio callback
778 may iterate the graph. */
781 Atomic
<FallbackDriverState
> mFallbackDriverState
{FallbackDriverState::None
};
782 /* SystemClockDriver used as fallback if this AudioCallbackDriver fails to
784 DataMutex
<RefPtr
<FallbackWrapper
>> mFallback
;
785 /* If using a fallback driver, this is the duration to wait after failing to
786 * start it before attempting to start it again. */
787 TimeDuration mNextReInitBackoffStep
;
788 /* If using a fallback driver, this is the next time we'll try to start the
790 TimeStamp mNextReInitAttempt
;
791 /* The time mAudioStreamState was changed to ChangingDevice.
792 * Synchronized by the mAudioStreamState atomic, i.e. written *before* writing
793 * the atomic, and read *after* reading the atomic. */
794 TimeStamp mChangingDeviceStartTime
;
796 /* When using the built-in speakers on macbook pro (13 and 15, all models),
797 * it's best to hard pan the audio on the right, to avoid feedback into the
798 * microphone that is located next to the left speaker. */
799 Atomic
<bool> mNeedsPanning
;
802 WavDumper mInputStreamFile
;
803 WavDumper mOutputStreamFile
;
805 virtual ~AudioCallbackDriver();
806 const bool mSandboxed
= false;
809 } // namespace mozilla
811 #endif // GRAPHDRIVER_H_