Bug 1913176 - Set micro surveys off by default in tests r=aaronmt
[gecko.git] / dom / media / GraphDriver.h
blob4608913c7fcbfd6cde777e0b4d6999711e981b88
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef GRAPHDRIVER_H_
7 #define GRAPHDRIVER_H_
9 #include "nsAutoRef.h"
10 #include "nsIThread.h"
11 #include "AudioBufferUtils.h"
12 #include "AudioMixer.h"
13 #include "AudioSegment.h"
14 #include "SelfRef.h"
15 #include "mozilla/Atomics.h"
16 #include "mozilla/dom/AudioContext.h"
17 #include "mozilla/DataMutex.h"
18 #include "mozilla/TaskQueue.h"
19 #include "mozilla/StaticPtr.h"
20 #include "WavDumper.h"
22 #include <thread>
24 struct cubeb_stream;
26 template <>
27 class nsAutoRefTraits<cubeb_stream> : public nsPointerRefTraits<cubeb_stream> {
28 public:
29 static void Release(cubeb_stream* aStream) { cubeb_stream_destroy(aStream); }
32 namespace mozilla {
33 /**
34 * Assume we can run an iteration of the MediaTrackGraph loop in this much time
35 * or less.
36 * We try to run the control loop at this rate.
38 static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
40 /**
41 * Assume that we might miss our scheduled wakeup of the MediaTrackGraph by
42 * this much.
44 static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
46 /**
47 * Try have this much audio buffered in streams and queued to the hardware.
48 * The maximum delay to the end of the next control loop
49 * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
50 * There is no point in buffering more audio than this in a stream at any
51 * given time (until we add processing).
52 * This is not optimal yet.
54 static const int AUDIO_TARGET_MS =
55 2 * MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS;
57 /**
58 * After starting a fallback driver, wait this long before attempting to re-init
59 * the audio stream the first time.
61 static const int AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS = 10;
63 /**
64 * The backoff step duration for when to next attempt to re-init the audio
65 * stream is capped at this value.
67 static const int AUDIO_MAX_FALLBACK_BACKOFF_STEP_MS = 1000;
69 class AudioCallbackDriver;
70 class GraphDriver;
71 class MediaTrack;
72 class OfflineClockDriver;
73 class SystemClockDriver;
75 namespace dom {
76 enum class AudioContextOperation : uint8_t;
79 struct GraphInterface : public nsISupports {
80 /**
81 * Object returned from OneIteration() instructing the iterating GraphDriver
82 * what to do.
84 * - If the result is StillProcessing: keep the iterations coming.
85 * - If the result is Stop: the driver potentially updates its internal state
86 * and interacts with the graph (e.g., NotifyOutputData), then it must call
87 * Stopped() exactly once.
88 * - If the result is SwitchDriver: the driver updates internal state as for
89 * the Stop result, then it must call Switched() exactly once and start
90 * NextDriver().
92 class IterationResult final {
93 struct Undefined {};
94 struct StillProcessing {};
95 struct Stop {
96 explicit Stop(RefPtr<Runnable> aStoppedRunnable)
97 : mStoppedRunnable(std::move(aStoppedRunnable)) {}
98 Stop(const Stop&) = delete;
99 Stop(Stop&& aOther) noexcept
100 : mStoppedRunnable(std::move(aOther.mStoppedRunnable)) {}
101 ~Stop() { MOZ_ASSERT(!mStoppedRunnable); }
102 RefPtr<Runnable> mStoppedRunnable;
103 void Stopped() {
104 mStoppedRunnable->Run();
105 mStoppedRunnable = nullptr;
108 struct SwitchDriver {
109 SwitchDriver(RefPtr<GraphDriver> aDriver,
110 RefPtr<Runnable> aSwitchedRunnable)
111 : mDriver(std::move(aDriver)),
112 mSwitchedRunnable(std::move(aSwitchedRunnable)) {}
113 SwitchDriver(const SwitchDriver&) = delete;
114 SwitchDriver(SwitchDriver&& aOther) noexcept
115 : mDriver(std::move(aOther.mDriver)),
116 mSwitchedRunnable(std::move(aOther.mSwitchedRunnable)) {}
117 ~SwitchDriver() { MOZ_ASSERT(!mSwitchedRunnable); }
118 RefPtr<GraphDriver> mDriver;
119 RefPtr<Runnable> mSwitchedRunnable;
120 void Switched() {
121 mSwitchedRunnable->Run();
122 mSwitchedRunnable = nullptr;
125 Variant<Undefined, StillProcessing, Stop, SwitchDriver> mResult;
127 explicit IterationResult(StillProcessing&& aArg)
128 : mResult(std::move(aArg)) {}
129 explicit IterationResult(Stop&& aArg) : mResult(std::move(aArg)) {}
130 explicit IterationResult(SwitchDriver&& aArg) : mResult(std::move(aArg)) {}
132 public:
133 IterationResult() : mResult(Undefined()) {}
134 IterationResult(const IterationResult&) = delete;
135 IterationResult(IterationResult&&) = default;
137 IterationResult& operator=(const IterationResult&) = delete;
138 IterationResult& operator=(IterationResult&&) = default;
140 static IterationResult CreateStillProcessing() {
141 return IterationResult(StillProcessing());
143 static IterationResult CreateStop(RefPtr<Runnable> aStoppedRunnable) {
144 return IterationResult(Stop(std::move(aStoppedRunnable)));
146 static IterationResult CreateSwitchDriver(
147 RefPtr<GraphDriver> aDriver, RefPtr<Runnable> aSwitchedRunnable) {
148 return IterationResult(
149 SwitchDriver(std::move(aDriver), std::move(aSwitchedRunnable)));
152 bool IsStillProcessing() const { return mResult.is<StillProcessing>(); }
153 bool IsStop() const { return mResult.is<Stop>(); }
154 bool IsSwitchDriver() const { return mResult.is<SwitchDriver>(); }
156 void Stopped() {
157 MOZ_ASSERT(IsStop());
158 mResult.as<Stop>().Stopped();
161 GraphDriver* NextDriver() const {
162 if (!IsSwitchDriver()) {
163 return nullptr;
165 return mResult.as<SwitchDriver>().mDriver;
168 void Switched() {
169 MOZ_ASSERT(IsSwitchDriver());
170 mResult.as<SwitchDriver>().Switched();
174 /* Called on the graph thread after an AudioCallbackDriver with an input
175 * stream has stopped. */
176 virtual void NotifyInputStopped() = 0;
177 /* Called on the graph thread when there is new input data for listeners. This
178 * is the raw audio input for this MediaTrackGraph. */
179 virtual void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
180 TrackRate aRate, uint32_t aChannels,
181 uint32_t aAlreadyBuffered) = 0;
182 /* Called on the main thread after an AudioCallbackDriver has attempted an
183 * operation to set aRequestedParams on the cubeb stream. */
184 virtual void NotifySetRequestedInputProcessingParamsResult(
185 AudioCallbackDriver* aDriver,
186 cubeb_input_processing_params aRequestedParams,
187 Result<cubeb_input_processing_params, int>&& aResult) = 0;
188 /* Called every time there are changes to input/output audio devices like
189 * plug/unplug etc. This can be called on any thread, and posts a message to
190 * the main thread so that it can post a message to the graph thread. */
191 virtual void DeviceChanged() = 0;
192 /* Called by GraphDriver to iterate the graph. Mixed audio output from the
193 * graph is passed into aMixerReceiver, if it is non-null. */
194 virtual IterationResult OneIteration(
195 GraphTime aStateComputedEnd, GraphTime aIterationEnd,
196 MixerCallbackReceiver* aMixerReceiver) = 0;
197 #ifdef DEBUG
198 /* True if we're on aDriver's thread, or if we're on mGraphRunner's thread
199 * and mGraphRunner is currently run by aDriver. */
200 virtual bool InDriverIteration(const GraphDriver* aDriver) const = 0;
201 #endif
205 * A driver is responsible for the scheduling of the processing, the thread
206 * management, and give the different clocks to a MediaTrackGraph. This is an
207 * abstract base class. A MediaTrackGraph can be driven by an
208 * OfflineClockDriver, if the graph is offline, or a SystemClockDriver or an
209 * AudioCallbackDriver, if the graph is real time.
210 * A MediaTrackGraph holds an owning reference to its driver.
212 * The lifetime of drivers is a complicated affair. Here are the different
213 * scenarii that can happen:
215 * Starting a MediaTrackGraph with an AudioCallbackDriver
216 * - A new thread T is created, from the main thread.
217 * - On this thread T, cubeb is initialized if needed, and a cubeb_stream is
218 * created and started
219 * - The thread T posts a message to the main thread to terminate itself.
220 * - The graph runs off the audio thread
222 * Starting a MediaTrackGraph with a SystemClockDriver:
223 * - A new thread T is created from the main thread.
224 * - The graph runs off this thread.
226 * Switching from a SystemClockDriver to an AudioCallbackDriver:
227 * - At the end of the MTG iteration, the graph tells the current driver to
228 * switch to an AudioCallbackDriver, which is created and initialized on the
229 * graph thread.
230 * - At the end of the MTG iteration, the SystemClockDriver transfers its timing
231 * info and a reference to itself to the AudioCallbackDriver. It then starts
232 * the AudioCallbackDriver.
233 * - When the AudioCallbackDriver starts, it:
234 * - Starts a fallback SystemClockDriver that runs until the
235 * AudioCallbackDriver is running, in case it takes a long time to start (it
236 * could block on I/O, e.g., negotiating a bluetooth connection).
237 * - Checks if it has been switched from a SystemClockDriver, and if that is
238 * the case, sends a message to the main thread to shut the
239 * SystemClockDriver thread down.
240 * - When the AudioCallbackDriver is running, data callbacks are blocked. The
241 * fallback driver detects this in its callback and stops itself. The first
242 * DataCallback after the fallback driver had stopped goes through.
243 * - The graph now runs off an audio callback.
245 * Switching from an AudioCallbackDriver to a SystemClockDriver:
246 * - At the end of the MTG iteration, the graph tells the current driver to
247 * switch to a SystemClockDriver.
248 * - the AudioCallbackDriver transfers its timing info and a reference to itself
249 * to the SystemClockDriver. A new SystemClockDriver is started from the
250 * current audio thread.
251 * - When starting, the SystemClockDriver checks if it has been switched from an
252 * AudioCallbackDriver. If yes, it creates a new temporary thread to release
253 * the cubeb_streams. This temporary thread closes the cubeb_stream, and then
254 * dispatches a message to the main thread to be terminated.
255 * - The graph now runs off a normal thread.
257 * Two drivers cannot run at the same time for the same graph. The thread safety
258 * of the different members of drivers, and their access pattern is documented
259 * next to the members themselves.
261 class GraphDriver {
262 public:
263 using IterationResult = GraphInterface::IterationResult;
265 GraphDriver(GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
266 uint32_t aSampleRate);
268 NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
270 /* Start the graph, init the driver, start the thread.
271 * A driver cannot be started twice, it must be shutdown
272 * before being started again. */
273 virtual void Start() = 0;
274 /* Shutdown GraphDriver */
275 MOZ_CAN_RUN_SCRIPT virtual void Shutdown() = 0;
276 /* Set the UTF-8 name for system audio streams.
277 * Graph thread, or main thread if the graph is not running. */
278 virtual void SetStreamName(const nsACString& aStreamName);
279 /* Rate at which the GraphDriver runs, in ms. This can either be user
280 * controlled (because we are using a {System,Offline}ClockDriver, and decide
281 * how often we want to wakeup/how much we want to process per iteration), or
282 * it can be indirectly set by the latency of the audio backend, and the
283 * number of buffers of this audio backend: say we have four buffers, and 40ms
284 * latency, we will get a callback approximately every 10ms. */
285 virtual uint32_t IterationDuration() = 0;
287 * Signaled by the graph when it needs another iteration. Goes unhandled for
288 * GraphDrivers that are not able to sleep indefinitely (i.e., all drivers but
289 * ThreadedDriver). Can be called on any thread.
291 virtual void EnsureNextIteration() = 0;
293 // Those are simply for accessing the associated pointer. Graph thread only,
294 // or if one is not running, main thread.
295 GraphDriver* PreviousDriver();
296 void SetPreviousDriver(GraphDriver* aPreviousDriver);
298 virtual AudioCallbackDriver* AsAudioCallbackDriver() { return nullptr; }
299 virtual const AudioCallbackDriver* AsAudioCallbackDriver() const {
300 return nullptr;
303 virtual OfflineClockDriver* AsOfflineClockDriver() { return nullptr; }
304 virtual const OfflineClockDriver* AsOfflineClockDriver() const {
305 return nullptr;
308 virtual SystemClockDriver* AsSystemClockDriver() { return nullptr; }
309 virtual const SystemClockDriver* AsSystemClockDriver() const {
310 return nullptr;
314 * Set the state of the driver so it can start at the right point in time,
315 * after switching from another driver.
317 void SetState(const nsACString& aStreamName, GraphTime aIterationEnd,
318 GraphTime aStateComputedTime);
320 GraphInterface* Graph() const { return mGraphInterface; }
322 #ifdef DEBUG
323 // True if the current thread is currently iterating the MTG.
324 bool InIteration() const;
325 #endif
326 // True if the current thread is the GraphDriver's thread.
327 virtual bool OnThread() const = 0;
328 // GraphDriver's thread has started and the thread is running.
329 virtual bool ThreadRunning() const = 0;
331 double MediaTimeToSeconds(GraphTime aTime) const {
332 NS_ASSERTION(aTime > -TRACK_TIME_MAX && aTime <= TRACK_TIME_MAX,
333 "Bad time");
334 return static_cast<double>(aTime) / mSampleRate;
337 GraphTime SecondsToMediaTime(double aS) const {
338 NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX / TRACK_RATE_MAX,
339 "Bad seconds");
340 return mSampleRate * aS;
343 GraphTime MillisecondsToMediaTime(int32_t aMS) const {
344 return RateConvertTicksRoundDown(mSampleRate, 1000, aMS);
347 protected:
348 // The UTF-8 name for system audio streams. Graph thread.
349 nsCString mStreamName;
350 // Time of the end of this graph iteration.
351 GraphTime mIterationEnd = 0;
352 // Time until which the graph has processed data.
353 GraphTime mStateComputedTime = 0;
354 // The GraphInterface this driver is currently iterating.
355 const RefPtr<GraphInterface> mGraphInterface;
356 // The sample rate for the graph, and in case of an audio driver, also for the
357 // cubeb stream.
358 const uint32_t mSampleRate;
360 // This is non-null only when this driver has recently switched from an other
361 // driver, and has not cleaned it up yet (for example because the audio stream
362 // is currently calling the callback during initialization).
364 // This is written to when changing driver, from the previous driver's thread,
365 // or a thread created for the occasion. This is read each time we need to
366 // check whether we're changing driver (in Switching()), from the graph
367 // thread.
368 // This must be accessed using the {Set,Get}PreviousDriver methods.
369 RefPtr<GraphDriver> mPreviousDriver;
371 virtual ~GraphDriver() = default;
374 class MediaTrackGraphInitThreadRunnable;
377 * This class is a driver that manages its own thread.
379 class ThreadedDriver : public GraphDriver {
380 class IterationWaitHelper {
381 Monitor mMonitor MOZ_UNANNOTATED;
382 // The below members are guarded by mMonitor.
383 bool mNeedAnotherIteration = false;
384 TimeStamp mWakeTime;
386 public:
387 IterationWaitHelper() : mMonitor("IterationWaitHelper::mMonitor") {}
390 * If another iteration is needed we wait for aDuration, otherwise we wait
391 * for a wake-up. If a wake-up occurs before aDuration time has passed, we
392 * wait for aDuration nonetheless.
394 void WaitForNextIterationAtLeast(TimeDuration aDuration) {
395 MonitorAutoLock lock(mMonitor);
396 TimeStamp now = TimeStamp::Now();
397 mWakeTime = now + aDuration;
398 while (true) {
399 if (mNeedAnotherIteration && now >= mWakeTime) {
400 break;
402 if (mNeedAnotherIteration) {
403 lock.Wait(mWakeTime - now);
404 } else {
405 lock.Wait(TimeDuration::Forever());
407 now = TimeStamp::Now();
409 mWakeTime = TimeStamp();
410 mNeedAnotherIteration = false;
414 * Sets mNeedAnotherIteration to true and notifies the monitor, in case a
415 * driver is currently waiting.
417 void EnsureNextIteration() {
418 MonitorAutoLock lock(mMonitor);
419 mNeedAnotherIteration = true;
420 lock.Notify();
424 public:
425 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ThreadedDriver, override);
427 ThreadedDriver(GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
428 uint32_t aSampleRate);
430 void EnsureNextIteration() override;
431 void Start() override;
432 MOZ_CAN_RUN_SCRIPT void Shutdown() override;
434 * Runs main control loop on the graph thread. Normally a single invocation
435 * of this runs for the entire lifetime of the graph thread.
437 virtual void RunThread();
438 friend class MediaTrackGraphInitThreadRunnable;
439 uint32_t IterationDuration() override { return MEDIA_GRAPH_TARGET_PERIOD_MS; }
441 nsIThread* Thread() const { return mThread; }
443 bool OnThread() const override {
444 return !mThread || mThread->IsOnCurrentThread();
447 bool ThreadRunning() const override { return mThreadRunning; }
449 protected:
450 /* Waits until it's time to process more data. */
451 void WaitForNextIteration();
452 /* Implementation dependent time the ThreadedDriver should wait between
453 * iterations. */
454 virtual TimeDuration WaitInterval() = 0;
455 /* When the graph wakes up to do an iteration, implementations return the
456 * range of time that will be processed. This is called only once per
457 * iteration; it may determine the interval from state in a previous
458 * call. */
459 virtual MediaTime GetIntervalForIteration() = 0;
461 virtual ~ThreadedDriver();
463 nsCOMPtr<nsIThread> mThread;
465 private:
466 // This is true if the thread is running. It is false
467 // before starting the thread and after stopping it.
468 Atomic<bool> mThreadRunning;
470 // Any thread.
471 IterationWaitHelper mWaitHelper;
475 * A SystemClockDriver drives a GraphInterface using a system clock, and waits
476 * using a monitor, between each iteration.
478 class SystemClockDriver : public ThreadedDriver {
479 public:
480 SystemClockDriver(GraphInterface* aGraphInterface,
481 GraphDriver* aPreviousDriver, uint32_t aSampleRate);
482 virtual ~SystemClockDriver();
483 SystemClockDriver* AsSystemClockDriver() override { return this; }
484 const SystemClockDriver* AsSystemClockDriver() const override { return this; }
486 protected:
487 /* Return the TimeDuration to wait before the next rendering iteration. */
488 TimeDuration WaitInterval() override;
489 MediaTime GetIntervalForIteration() override;
491 private:
492 // Those are only modified (after initialization) on the graph thread. The
493 // graph thread does not run during the initialization.
494 TimeStamp mInitialTimeStamp;
495 TimeStamp mCurrentTimeStamp;
496 TimeStamp mLastTimeStamp;
500 * An OfflineClockDriver runs the graph as fast as possible, without waiting
501 * between iteration.
503 class OfflineClockDriver : public ThreadedDriver {
504 public:
505 OfflineClockDriver(GraphInterface* aGraphInterface, uint32_t aSampleRate,
506 GraphTime aSlice);
507 virtual ~OfflineClockDriver();
508 OfflineClockDriver* AsOfflineClockDriver() override { return this; }
509 const OfflineClockDriver* AsOfflineClockDriver() const override {
510 return this;
513 void RunThread() override;
515 protected:
516 TimeDuration WaitInterval() override { return TimeDuration(); }
517 MediaTime GetIntervalForIteration() override;
519 private:
520 // Time, in GraphTime, for each iteration
521 GraphTime mSlice;
524 enum class AudioInputType { Unknown, Voice };
527 * This is a graph driver that is based on callback functions called by the
528 * audio api. This ensures minimal audio latency, because it means there is no
529 * buffering happening: the audio is generated inside the callback.
531 * This design is less flexible than running our own thread:
532 * - We have no control over the thread:
533 * - It cannot block, and it has to run for a shorter amount of time than the
534 * buffer it is going to fill, or an under-run is going to occur (short burst
535 * of silence in the final audio output).
536 * - We can't know for sure when the callback function is going to be called
537 * (although we compute an estimation so we can schedule video frames)
538 * - Creating and shutting the thread down is a blocking operation, that can
539 * take _seconds_ in some cases (because IPC has to be set up, and
540 * sometimes hardware components are involved and need to be warmed up)
541 * - We have no control on how much audio we generate, we have to return exactly
542 * the number of frames asked for by the callback. Since for the Web Audio
543 * API, we have to do block processing at 128 frames per block, we need to
544 * keep a little spill buffer to store the extra frames.
546 class AudioCallbackDriver : public GraphDriver, public MixerCallbackReceiver {
547 using IterationResult = GraphInterface::IterationResult;
548 enum class FallbackDriverState;
549 class FallbackWrapper;
551 public:
552 NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DELETE_ON_EVENT_TARGET(
553 AudioCallbackDriver, mCubebOperationThread, override);
555 /** If aInputChannelCount is zero, then this driver is output-only. */
556 AudioCallbackDriver(
557 GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
558 uint32_t aSampleRate, uint32_t aOutputChannelCount,
559 uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
560 CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType,
561 cubeb_input_processing_params aRequestedInputProcessingParams);
563 void Start() override;
564 MOZ_CAN_RUN_SCRIPT void Shutdown() override;
565 void SetStreamName(const nsACString& aStreamName) override;
567 /* Static wrapper function cubeb calls back. */
568 static long DataCallback_s(cubeb_stream* aStream, void* aUser,
569 const void* aInputBuffer, void* aOutputBuffer,
570 long aFrames);
571 static void StateCallback_s(cubeb_stream* aStream, void* aUser,
572 cubeb_state aState);
573 static void DeviceChangedCallback_s(void* aUser);
575 /* This function is called by the underlying audio backend when a refill is
576 * needed. This is what drives the whole graph when it is used to output
577 * audio. If the return value is exactly aFrames, this function will get
578 * called again. If it is less than aFrames, the stream will go in draining
579 * mode, and this function will not be called again. */
580 long DataCallback(const AudioDataValue* aInputBuffer,
581 AudioDataValue* aOutputBuffer, long aFrames);
582 /* This function is called by the underlying audio backend, but is only used
583 * for informational purposes at the moment. */
584 void StateCallback(cubeb_state aState);
585 /* This is an approximation of the number of millisecond there are between two
586 * iterations of the graph. */
587 uint32_t IterationDuration() override;
588 /* If the audio stream has started, this does nothing. There will be another
589 * iteration. If there is an active fallback driver, we forward the call so it
590 * can wake up. */
591 void EnsureNextIteration() override;
593 /* This function gets called when the graph has produced the audio frames for
594 * this iteration. */
595 void MixerCallback(AudioChunk* aMixedBuffer, uint32_t aSampleRate) override;
597 AudioCallbackDriver* AsAudioCallbackDriver() override { return this; }
598 const AudioCallbackDriver* AsAudioCallbackDriver() const override {
599 return this;
602 uint32_t OutputChannelCount() { return mOutputChannelCount; }
604 uint32_t InputChannelCount() { return mInputChannelCount; }
606 AudioInputType InputDevicePreference() {
607 if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
608 return AudioInputType::Voice;
610 return AudioInputType::Unknown;
613 /* Get the input processing params requested from this driver, so that an
614 * external caller can decide whether it is necessary to call the setter,
615 * since it may allocate or dispatch. */
616 cubeb_input_processing_params RequestedInputProcessingParams() const;
618 /* Set the input processing params requested from this driver. */
619 void SetRequestedInputProcessingParams(cubeb_input_processing_params aParams);
621 std::thread::id ThreadId() const { return mAudioThreadIdInCb.load(); }
623 /* Called at the beginning of the audio callback to check if the thread id has
624 * changed. */
625 bool CheckThreadIdChanged();
627 bool OnThread() const override {
628 return mAudioThreadIdInCb.load() == std::this_thread::get_id();
631 /* Returns true if this driver has started (perhaps with a fallback driver)
632 * and not yet stopped. */
633 bool ThreadRunning() const override {
634 return mAudioStreamState == AudioStreamState::Running ||
635 mFallbackDriverState == FallbackDriverState::Running;
638 /* Whether the underlying cubeb stream has been started and has not stopped
639 * or errored. */
640 bool IsStarted() { return mAudioStreamState > AudioStreamState::Starting; };
642 // Returns the output latency for the current audio output stream.
643 TimeDuration AudioOutputLatency();
645 /* Returns true if this driver has a fallback driver and handover to the audio
646 * callback has not been completed. */
647 bool HasFallback() const;
648 /* Returns true if this driver is currently driven by the fallback driver. */
649 bool OnFallback() const;
651 private:
653 * On certain MacBookPro, the microphone is located near the left speaker.
654 * We need to pan the sound output to the right speaker if we are using the
655 * mic and the built-in speaker, or we will have terrible echo. */
656 void PanOutputIfNeeded(bool aMicrophoneActive);
658 * This is called when the output device used by the cubeb stream changes. */
659 void DeviceChangedCallback();
660 /* Start the cubeb stream */
661 bool StartStream();
662 friend class MediaTrackGraphInitThreadRunnable;
663 void Init(const nsCString& aStreamName);
664 void SetCubebStreamName(const nsCString& aStreamName);
665 void Stop();
666 /* After the requested input processing params has changed, this applies them
667 * on the cubeb stream. */
668 void SetInputProcessingParams(cubeb_input_processing_params aParams);
669 /* Calls FallbackToSystemClockDriver() if in FallbackDriverState::None.
670 * Returns Ok(true) if the fallback driver was started, or the old
671 * FallbackDriverState in an Err otherwise. */
672 Result<bool, FallbackDriverState> TryStartingFallbackDriver();
673 /* Fall back to a SystemClockDriver using a normal thread. If needed, the
674 * graph will try to re-open an audio stream later. */
675 void FallbackToSystemClockDriver();
676 /* Called by the fallback driver when it has fully stopped, after finishing
677 * its last iteration. If it stopped after the audio stream started, aState
678 * will be None. If it stopped after the graph told it to stop, or switch,
679 * aState will be Stopped. Hands over state to the audio driver that may
680 * iterate the graph after this has been called. */
681 void FallbackDriverStopped(GraphTime aIterationEnd,
682 GraphTime aStateComputedTime,
683 FallbackDriverState aState);
685 /* Called at the end of the fallback driver's iteration to see whether we
686 * should attempt to start the AudioStream again. */
687 void MaybeStartAudioStream();
689 /* This is true when the method is executed on CubebOperation thread pool. */
690 bool OnCubebOperationThread() {
691 return mCubebOperationThread->IsOnCurrentThreadInfallible();
694 /* MediaTrackGraphs are always down/up mixed to output channels. */
695 const uint32_t mOutputChannelCount;
696 /* The size of this buffer comes from the fact that some audio backends can
697 * call back with a number of frames lower than one block (128 frames), so we
698 * need to keep at most two block in the SpillBuffer, because we always round
699 * up to block boundaries during an iteration.
700 * This is only ever accessed on the audio callback thread. */
701 SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2> mScratchBuffer;
702 /* Wrapper to ensure we write exactly the number of frames we need in the
703 * audio buffer cubeb passes us. This is only ever accessed on the audio
704 * callback thread. */
705 AudioCallbackBufferWrapper<AudioDataValue> mBuffer;
706 // mAudioStream (a cubeb_stream) has a bare pointer to the cubeb context, so
707 // we hold a strong reference on its behalf.
708 RefPtr<CubebUtils::CubebHandle> mCubeb;
709 /* cubeb stream for this graph. This is non-null after a successful
710 * cubeb_stream_init(). CubebOperation thread only. */
711 nsAutoRef<cubeb_stream> mAudioStream;
712 /* The number of input channels from cubeb. Set before opening cubeb. If it is
713 * zero then the driver is output-only. */
714 const uint32_t mInputChannelCount;
716 * Devices to use for cubeb input & output, or nullptr for default device.
718 const CubebUtils::AudioDeviceID mOutputDeviceID;
719 const CubebUtils::AudioDeviceID mInputDeviceID;
720 /* Approximation of the time between two callbacks. This is used to schedule
721 * video frames. This is in milliseconds. Only even used (after
722 * inizatialization) on the audio callback thread. */
723 uint32_t mIterationDurationMS;
725 struct AutoInCallback {
726 explicit AutoInCallback(AudioCallbackDriver* aDriver);
727 ~AutoInCallback();
728 AudioCallbackDriver* mDriver;
731 static already_AddRefed<TaskQueue> CreateTaskQueue();
733 /* Shared thread pool with up to one thread for off-main-thread
734 * initialization and shutdown of the audio stream and for other tasks that
735 * must run serially for access to mAudioStream. */
736 const RefPtr<TaskQueue> mCubebOperationThread;
737 cubeb_device_pref mInputDevicePreference;
738 /* Params that have been attempted to set on mAudioStream, after filtering by
739 * supported processing params. Cubeb operation thread only. */
740 cubeb_input_processing_params mConfiguredInputProcessingParams =
741 CUBEB_INPUT_PROCESSING_PARAM_NONE;
742 /* The input processing params requested from this audio driver. Once started,
743 * audio callback thread only. */
744 cubeb_input_processing_params mRequestedInputProcessingParams;
745 /* Contains the id of the audio thread, from profiler_current_thread_id. */
746 std::atomic<ProfilerThreadId> mAudioThreadId;
747 /* This allows implementing AutoInCallback. This is equal to the current
748 * thread id when in an audio callback, and is an invalid thread id otherwise.
750 std::atomic<std::thread::id> mAudioThreadIdInCb;
751 /* State of the audio stream, see inline comments. */
752 enum class AudioStreamState {
753 /* There is no cubeb_stream or mAudioStream is in CUBEB_STATE_ERROR or
754 * CUBEB_STATE_STOPPED and no pending task exists to Init() a new
755 * cubeb_stream. */
756 None,
757 /* A task to Init() a new cubeb_stream is pending. */
758 Pending,
759 /* cubeb_start_stream() is about to be or has been called on mAudioStream.
760 * Any previous cubeb_streams have been destroyed. */
761 Starting,
762 /* mAudioStream has advertised it will change device. In this state we
763 ignore all data callbacks until the fallback driver has started. */
764 ChangingDevice,
765 /* mAudioStream is running. */
766 Running,
767 /* mAudioStream is draining, and will soon stop. */
768 Stopping
770 Atomic<AudioStreamState> mAudioStreamState{AudioStreamState::None};
771 /* State of the fallback driver, see inline comments. */
772 enum class FallbackDriverState {
773 /* There is no fallback driver. */
774 None,
775 /* There is a fallback driver trying to iterate us. */
776 Running,
777 /* There was a fallback driver and the graph stopped it. No audio callback
778 may iterate the graph. */
779 Stopped,
781 Atomic<FallbackDriverState> mFallbackDriverState{FallbackDriverState::None};
782 /* SystemClockDriver used as fallback if this AudioCallbackDriver fails to
783 * init or start. */
784 DataMutex<RefPtr<FallbackWrapper>> mFallback;
785 /* If using a fallback driver, this is the duration to wait after failing to
786 * start it before attempting to start it again. */
787 TimeDuration mNextReInitBackoffStep;
788 /* If using a fallback driver, this is the next time we'll try to start the
789 * audio stream. */
790 TimeStamp mNextReInitAttempt;
791 /* The time mAudioStreamState was changed to ChangingDevice.
792 * Synchronized by the mAudioStreamState atomic, i.e. written *before* writing
793 * the atomic, and read *after* reading the atomic. */
794 TimeStamp mChangingDeviceStartTime;
795 #ifdef XP_MACOSX
796 /* When using the built-in speakers on macbook pro (13 and 15, all models),
797 * it's best to hard pan the audio on the right, to avoid feedback into the
798 * microphone that is located next to the left speaker. */
799 Atomic<bool> mNeedsPanning;
800 #endif
802 WavDumper mInputStreamFile;
803 WavDumper mOutputStreamFile;
805 virtual ~AudioCallbackDriver();
806 const bool mSandboxed = false;
809 } // namespace mozilla
811 #endif // GRAPHDRIVER_H_