dom/media/GraphDriver.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "GraphDriver.h"
   8
   9 #include "AudioNodeEngine.h"
  10 #include "cubeb/cubeb.h"
  11 #include "mozilla/dom/AudioContext.h"
  12 #include "mozilla/dom/AudioDeviceInfo.h"
  13 #include "mozilla/dom/BaseAudioContextBinding.h"
  14 #include "mozilla/SchedulerGroup.h"
  15 #include "mozilla/SharedThreadPool.h"
  16 #include "mozilla/ClearOnShutdown.h"
  17 #include "mozilla/Unused.h"
  18 #include "mozilla/MathAlgorithms.h"
  19 #include "mozilla/StaticPrefs_media.h"
  20 #include "CubebDeviceEnumerator.h"
  21 #include "MediaTrackGraphImpl.h"
  22 #include "CallbackThreadRegistry.h"
  23 #include "Tracing.h"
  24
  25 #ifdef MOZ_WEBRTC
  26 #  include "webrtc/MediaEngineWebRTC.h"
  27 #endif
  28
  29 #ifdef XP_MACOSX
  30 #  include <sys/sysctl.h>
  31 #  include "nsCocoaFeatures.h"
  32 #endif
  33
  34 extern mozilla::LazyLogModule gMediaTrackGraphLog;
  35 #ifdef LOG
  36 #  undef LOG
  37 #endif  // LOG
  38 #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
  39
  40 namespace mozilla {
  41
  42 GraphDriver::GraphDriver(GraphInterface* aGraphInterface,
  43                          GraphDriver* aPreviousDriver, uint32_t aSampleRate)
  44     : mGraphInterface(aGraphInterface),
  45       mSampleRate(aSampleRate),
  46       mPreviousDriver(aPreviousDriver) {}
  47
  48 void GraphDriver::SetStreamName(const nsACString& aStreamName) {
  49   MOZ_ASSERT(InIteration() || (!ThreadRunning() && NS_IsMainThread()));
  50   mStreamName = aStreamName;
  51   LOG(LogLevel::Debug, ("%p: GraphDriver::SetStreamName driver=%p %s", Graph(),
  52                         this, mStreamName.get()));
  53 }
  54
  55 void GraphDriver::SetState(const nsACString& aStreamName,
  56                            GraphTime aIterationEnd,
  57                            GraphTime aStateComputedTime) {
  58   MOZ_ASSERT(InIteration() || !ThreadRunning());
  59
  60   mStreamName = aStreamName;
  61   mIterationEnd = aIterationEnd;
  62   mStateComputedTime = aStateComputedTime;
  63 }
  64
  65 #ifdef DEBUG
  66 bool GraphDriver::InIteration() const {
  67   return OnThread() || Graph()->InDriverIteration(this);
  68 }
  69 #endif
  70
  71 GraphDriver* GraphDriver::PreviousDriver() {
  72   MOZ_ASSERT(InIteration() || !ThreadRunning());
  73   return mPreviousDriver;
  74 }
  75
  76 void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) {
  77   MOZ_ASSERT(InIteration() || !ThreadRunning());
  78   mPreviousDriver = aPreviousDriver;
  79 }
  80
  81 ThreadedDriver::ThreadedDriver(GraphInterface* aGraphInterface,
  82                                GraphDriver* aPreviousDriver,
  83                                uint32_t aSampleRate)
  84     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
  85       mThreadRunning(false) {}
  86
  87 class MediaTrackGraphShutdownThreadRunnable : public Runnable {
  88  public:
  89   explicit MediaTrackGraphShutdownThreadRunnable(
  90       already_AddRefed<nsIThread> aThread)
  91       : Runnable("MediaTrackGraphShutdownThreadRunnable"), mThread(aThread) {}
  92   NS_IMETHOD Run() override {
  93     TRACE("MediaTrackGraphShutdownThreadRunnable");
  94     MOZ_ASSERT(NS_IsMainThread());
  95     MOZ_ASSERT(mThread);
  96
  97     mThread->AsyncShutdown();
  98     mThread = nullptr;
  99     return NS_OK;
 100   }
 101
 102  private:
 103   nsCOMPtr<nsIThread> mThread;
 104 };
 105
 106 ThreadedDriver::~ThreadedDriver() {
 107   if (mThread) {
 108     nsCOMPtr<nsIRunnable> event =
 109         new MediaTrackGraphShutdownThreadRunnable(mThread.forget());
 110     SchedulerGroup::Dispatch(event.forget());
 111   }
 112 }
 113
 114 class MediaTrackGraphInitThreadRunnable : public Runnable {
 115  public:
 116   explicit MediaTrackGraphInitThreadRunnable(ThreadedDriver* aDriver)
 117       : Runnable("MediaTrackGraphInitThreadRunnable"), mDriver(aDriver) {}
 118   NS_IMETHOD Run() override {
 119     TRACE("MediaTrackGraphInitThreadRunnable");
 120     MOZ_ASSERT(!mDriver->ThreadRunning());
 121     LOG(LogLevel::Debug, ("Starting a new system driver for graph %p",
 122                           mDriver->mGraphInterface.get()));
 123
 124     if (GraphDriver* previousDriver = mDriver->PreviousDriver()) {
 125       LOG(LogLevel::Debug,
 126           ("%p releasing an AudioCallbackDriver(%p), for graph %p",
 127            mDriver.get(), previousDriver, mDriver->Graph()));
 128       MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
 129       AudioCallbackDriver* audioCallbackDriver =
 130           previousDriver->AsAudioCallbackDriver();
 131       audioCallbackDriver->mCubebOperationThread->Dispatch(
 132           NS_NewRunnableFunction(
 133               "ThreadedDriver previousDriver::Stop()",
 134               [audioCallbackDriver = RefPtr{audioCallbackDriver}] {
 135                 audioCallbackDriver->Stop();
 136               }));
 137       mDriver->SetPreviousDriver(nullptr);
 138     }
 139
 140     mDriver->RunThread();
 141     return NS_OK;
 142   }
 143
 144  private:
 145   RefPtr<ThreadedDriver> mDriver;
 146 };
 147
 148 void ThreadedDriver::Start() {
 149   MOZ_ASSERT(!ThreadRunning());
 150   LOG(LogLevel::Debug,
 151       ("Starting thread for a SystemClockDriver  %p", mGraphInterface.get()));
 152   Unused << NS_WARN_IF(mThread);
 153   MOZ_ASSERT(!mThread);  // Ensure we haven't already started it
 154
 155   nsCOMPtr<nsIRunnable> event = new MediaTrackGraphInitThreadRunnable(this);
 156   // Note: mThread may be null during event->Run() if we pass to NewNamedThread!
 157   // See AudioInitTask
 158   nsresult rv = NS_NewNamedThread("MediaTrackGrph", getter_AddRefs(mThread));
 159   if (NS_SUCCEEDED(rv)) {
 160     mThread->Dispatch(event.forget(), NS_DISPATCH_NORMAL);
 161   }
 162 }
 163
 164 void ThreadedDriver::Shutdown() {
 165   NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
 166   // mGraph's thread is not running so it's OK to do whatever here
 167   LOG(LogLevel::Debug, ("Stopping threads for MediaTrackGraph %p", this));
 168
 169   if (mThread) {
 170     LOG(LogLevel::Debug,
 171         ("%p: Stopping ThreadedDriver's %p thread", Graph(), this));
 172     mThread->AsyncShutdown();
 173     mThread = nullptr;
 174   }
 175 }
 176
 177 SystemClockDriver::SystemClockDriver(GraphInterface* aGraphInterface,
 178                                      GraphDriver* aPreviousDriver,
 179                                      uint32_t aSampleRate)
 180     : ThreadedDriver(aGraphInterface, aPreviousDriver, aSampleRate),
 181       mInitialTimeStamp(TimeStamp::Now()),
 182       mCurrentTimeStamp(TimeStamp::Now()),
 183       mLastTimeStamp(TimeStamp::Now()) {}
 184
 185 SystemClockDriver::~SystemClockDriver() = default;
 186
 187 void ThreadedDriver::RunThread() {
 188   mThreadRunning = true;
 189   while (true) {
 190     auto iterationStart = mIterationEnd;
 191     mIterationEnd += GetIntervalForIteration();
 192
 193     if (mStateComputedTime < mIterationEnd) {
 194       LOG(LogLevel::Warning, ("%p: Global underrun detected", Graph()));
 195       mIterationEnd = mStateComputedTime;
 196     }
 197
 198     if (iterationStart >= mIterationEnd) {
 199       NS_ASSERTION(iterationStart == mIterationEnd, "Time can't go backwards!");
 200       // This could happen due to low clock resolution, maybe?
 201       LOG(LogLevel::Debug, ("%p: Time did not advance", Graph()));
 202     }
 203
 204     GraphTime nextStateComputedTime =
 205         MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(
 206             mIterationEnd + MillisecondsToMediaTime(AUDIO_TARGET_MS));
 207     if (nextStateComputedTime < mStateComputedTime) {
 208       // A previous driver may have been processing further ahead of
 209       // iterationEnd.
 210       LOG(LogLevel::Warning,
 211           ("%p: Prevent state from going backwards. interval[%ld; %ld] "
 212            "state[%ld; "
 213            "%ld]",
 214            Graph(), (long)iterationStart, (long)mIterationEnd,
 215            (long)mStateComputedTime, (long)nextStateComputedTime));
 216       nextStateComputedTime = mStateComputedTime;
 217     }
 218     LOG(LogLevel::Verbose,
 219         ("%p: interval[%ld; %ld] state[%ld; %ld]", Graph(),
 220          (long)iterationStart, (long)mIterationEnd, (long)mStateComputedTime,
 221          (long)nextStateComputedTime));
 222
 223     mStateComputedTime = nextStateComputedTime;
 224     IterationResult result =
 225         Graph()->OneIteration(mStateComputedTime, mIterationEnd, nullptr);
 226
 227     if (result.IsStop()) {
 228       // Signal that we're done stopping.
 229       result.Stopped();
 230       break;
 231     }
 232     WaitForNextIteration();
 233     if (GraphDriver* nextDriver = result.NextDriver()) {
 234       LOG(LogLevel::Debug, ("%p: Switching to AudioCallbackDriver", Graph()));
 235       result.Switched();
 236       nextDriver->SetState(mStreamName, mIterationEnd, mStateComputedTime);
 237       nextDriver->Start();
 238       break;
 239     }
 240     MOZ_ASSERT(result.IsStillProcessing());
 241   }
 242   mThreadRunning = false;
 243 }
 244
 245 MediaTime SystemClockDriver::GetIntervalForIteration() {
 246   TimeStamp now = TimeStamp::Now();
 247   MediaTime interval =
 248       SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
 249   mCurrentTimeStamp = now;
 250
 251   MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
 252           ("%p: Updating current time to %f (real %f, StateComputedTime() %f)",
 253            Graph(), MediaTimeToSeconds(mIterationEnd + interval),
 254            (now - mInitialTimeStamp).ToSeconds(),
 255            MediaTimeToSeconds(mStateComputedTime)));
 256
 257   return interval;
 258 }
 259
 260 void ThreadedDriver::EnsureNextIteration() {
 261   mWaitHelper.EnsureNextIteration();
 262 }
 263
 264 void ThreadedDriver::WaitForNextIteration() {
 265   MOZ_ASSERT(mThread);
 266   MOZ_ASSERT(OnThread());
 267   mWaitHelper.WaitForNextIterationAtLeast(WaitInterval());
 268 }
 269
 270 TimeDuration SystemClockDriver::WaitInterval() {
 271   MOZ_ASSERT(mThread);
 272   MOZ_ASSERT(OnThread());
 273   TimeStamp now = TimeStamp::Now();
 274   int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
 275                       int64_t((now - mCurrentTimeStamp).ToMilliseconds());
 276   // Make sure timeoutMS doesn't overflow 32 bits by waking up at
 277   // least once a minute, if we need to wake up at all
 278   timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60 * 1000));
 279   LOG(LogLevel::Verbose,
 280       ("%p: Waiting for next iteration; at %f, timeout=%f", Graph(),
 281        (now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0));
 282
 283   return TimeDuration::FromMilliseconds(timeoutMS);
 284 }
 285
 286 OfflineClockDriver::OfflineClockDriver(GraphInterface* aGraphInterface,
 287                                        uint32_t aSampleRate, GraphTime aSlice)
 288     : ThreadedDriver(aGraphInterface, nullptr, aSampleRate), mSlice(aSlice) {}
 289
 290 OfflineClockDriver::~OfflineClockDriver() = default;
 291
 292 void OfflineClockDriver::RunThread() {
 293   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(mThread);
 294   nsCOMPtr<nsIThreadObserver> observer = do_QueryInterface(Graph());
 295   threadInternal->SetObserver(observer);
 296
 297   ThreadedDriver::RunThread();
 298 }
 299
 300 MediaTime OfflineClockDriver::GetIntervalForIteration() {
 301   return MillisecondsToMediaTime(mSlice);
 302 }
 303
 304 /* Helper to proxy the GraphInterface methods used by a running
 305  * mFallbackDriver. */
 306 class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
 307  public:
 308   FallbackWrapper(RefPtr<GraphInterface> aGraph,
 309                   RefPtr<AudioCallbackDriver> aOwner, uint32_t aSampleRate,
 310                   const nsACString& aStreamName, GraphTime aIterationEnd,
 311                   GraphTime aStateComputedTime)
 312       : mGraph(std::move(aGraph)),
 313         mOwner(std::move(aOwner)),
 314         mFallbackDriver(
 315             MakeRefPtr<SystemClockDriver>(this, nullptr, aSampleRate)) {
 316     mFallbackDriver->SetState(aStreamName, aIterationEnd, aStateComputedTime);
 317   }
 318
 319   NS_DECL_THREADSAFE_ISUPPORTS
 320
 321   /* Proxied SystemClockDriver methods */
 322   void Start() { mFallbackDriver->Start(); }
 323   MOZ_CAN_RUN_SCRIPT void Shutdown() {
 324     RefPtr<SystemClockDriver> driver = mFallbackDriver;
 325     driver->Shutdown();
 326   }
 327   void SetStreamName(const nsACString& aStreamName) {
 328     mFallbackDriver->SetStreamName(aStreamName);
 329   }
 330   void EnsureNextIteration() { mFallbackDriver->EnsureNextIteration(); }
 331 #ifdef DEBUG
 332   bool InIteration() { return mFallbackDriver->InIteration(); }
 333 #endif
 334   bool OnThread() { return mFallbackDriver->OnThread(); }
 335
 336   /* GraphInterface methods */
 337   void NotifyInputStopped() override {
 338     MOZ_CRASH("Unexpected NotifyInputStopped from fallback SystemClockDriver");
 339   }
 340   void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
 341                        TrackRate aRate, uint32_t aChannels,
 342                        uint32_t aAlreadyBuffered) override {
 343     MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
 344   }
 345   void DeviceChanged() override {
 346     MOZ_CRASH("Unexpected DeviceChanged from fallback SystemClockDriver");
 347   }
 348 #ifdef DEBUG
 349   bool InDriverIteration(const GraphDriver* aDriver) const override {
 350     return mGraph->InDriverIteration(mOwner) && mOwner->OnFallback();
 351   }
 352 #endif
 353   IterationResult OneIteration(GraphTime aStateComputedEnd,
 354                                GraphTime aIterationEnd,
 355                                MixerCallbackReceiver* aMixerReceiver) override {
 356     MOZ_ASSERT(!aMixerReceiver);
 357
 358 #ifdef DEBUG
 359     AutoInCallback aic(mOwner);
 360 #endif
 361
 362     IterationResult result =
 363         mGraph->OneIteration(aStateComputedEnd, aIterationEnd, aMixerReceiver);
 364
 365     AudioStreamState audioState = mOwner->mAudioStreamState;
 366
 367     MOZ_ASSERT(audioState != AudioStreamState::Stopping,
 368                "The audio driver can only enter stopping if it iterated the "
 369                "graph, which it can only do if there's no fallback driver");
 370     if (audioState != AudioStreamState::Running && result.IsStillProcessing()) {
 371       mOwner->MaybeStartAudioStream();
 372       return result;
 373     }
 374
 375     MOZ_ASSERT(result.IsStillProcessing() || result.IsStop() ||
 376                result.IsSwitchDriver());
 377
 378     // Proxy the release of the fallback driver to a background thread, so it
 379     // doesn't perform unexpected suicide.
 380     IterationResult stopFallback =
 381         IterationResult::CreateStop(NS_NewRunnableFunction(
 382             "AudioCallbackDriver::FallbackDriverStopped",
 383             [self = RefPtr<FallbackWrapper>(this), this, aIterationEnd,
 384              aStateComputedEnd, result = std::move(result)]() mutable {
 385               FallbackDriverState fallbackState =
 386                   result.IsStillProcessing() ? FallbackDriverState::None
 387                                              : FallbackDriverState::Stopped;
 388               mOwner->FallbackDriverStopped(aIterationEnd, aStateComputedEnd,
 389                                             fallbackState);
 390
 391               if (fallbackState == FallbackDriverState::Stopped) {
 392 #ifdef DEBUG
 393                 // The AudioCallbackDriver may not iterate the graph, but we'll
 394                 // call into it so we need to be regarded as "in iteration".
 395                 AutoInCallback aic(mOwner);
 396 #endif
 397                 if (GraphDriver* nextDriver = result.NextDriver()) {
 398                   LOG(LogLevel::Debug,
 399                       ("%p: Switching from fallback to other driver.",
 400                        mOwner.get()));
 401                   result.Switched();
 402                   nextDriver->SetState(mOwner->mStreamName, aIterationEnd,
 403                                        aStateComputedEnd);
 404                   nextDriver->Start();
 405                 } else if (result.IsStop()) {
 406                   LOG(LogLevel::Debug,
 407                       ("%p: Stopping fallback driver.", mOwner.get()));
 408                   result.Stopped();
 409                 }
 410               }
 411               mOwner = nullptr;
 412               NS_DispatchBackgroundTask(NS_NewRunnableFunction(
 413                   "AudioCallbackDriver::FallbackDriverStopped::Release",
 414                   [fallback = std::move(self->mFallbackDriver)] {}));
 415             }));
 416
 417     return stopFallback;
 418   }
 419
 420  private:
 421   virtual ~FallbackWrapper() = default;
 422
 423   const RefPtr<GraphInterface> mGraph;
 424   // Valid until mFallbackDriver has finished its last iteration.
 425   RefPtr<AudioCallbackDriver> mOwner;
 426   RefPtr<SystemClockDriver> mFallbackDriver;
 427 };
 428
 429 NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)
 430
 431 AudioCallbackDriver::AudioCallbackDriver(
 432     GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
 433     uint32_t aSampleRate, uint32_t aOutputChannelCount,
 434     uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
 435     CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType)
 436     : GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
 437       mOutputChannelCount(aOutputChannelCount),
 438       mInputChannelCount(aInputChannelCount),
 439       mOutputDeviceID(aOutputDeviceID),
 440       mInputDeviceID(aInputDeviceID),
 441       mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
 442       mCubebOperationThread(CUBEB_TASK_THREAD),
 443       mAudioThreadId(ProfilerThreadId{}),
 444       mAudioThreadIdInCb(std::thread::id()),
 445       mFallback("AudioCallbackDriver::mFallback"),
 446       mSandboxed(CubebUtils::SandboxEnabled()) {
 447   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p ctor - input: device %p, "
 448                         "channel %d, output: device %p, channel %d",
 449                         Graph(), this, mInputDeviceID, mInputChannelCount,
 450                         mOutputDeviceID, mOutputChannelCount));
 451
 452   NS_WARNING_ASSERTION(mOutputChannelCount != 0,
 453                        "Invalid output channel count");
 454   MOZ_ASSERT(mOutputChannelCount <= 8);
 455
 456   const uint32_t kIdleThreadTimeoutMs = 2000;
 457   mCubebOperationThread->SetIdleThreadTimeout(
 458       PR_MillisecondsToInterval(kIdleThreadTimeoutMs));
 459
 460   bool allowVoice = true;
 461 #ifdef MOZ_WIDGET_COCOA
 462   // Using the VoiceProcessingIO audio unit on MacOS 12 causes crashes in
 463   // platform code.
 464   allowVoice = nsCocoaFeatures::macOSVersionMajor() != 12;
 465 #endif
 466
 467   if (aAudioInputType == AudioInputType::Voice && allowVoice) {
 468     LOG(LogLevel::Debug, ("VOICE."));
 469     mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
 470     CubebUtils::SetInCommunication(true);
 471   } else {
 472     mInputDevicePreference = CUBEB_DEVICE_PREF_ALL;
 473   }
 474 }
 475
 476 AudioCallbackDriver::~AudioCallbackDriver() {
 477   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
 478     CubebUtils::SetInCommunication(false);
 479   }
 480 }
 481
 482 bool IsMacbookOrMacbookAir() {
 483 #ifdef XP_MACOSX
 484   size_t len = 0;
 485   sysctlbyname("hw.model", NULL, &len, NULL, 0);
 486   if (len) {
 487     UniquePtr<char[]> model(new char[len]);
 488     // This string can be
 489     // MacBook%d,%d for a normal MacBook
 490     // MacBookAir%d,%d for a Macbook Air
 491     sysctlbyname("hw.model", model.get(), &len, NULL, 0);
 492     char* substring = strstr(model.get(), "MacBook");
 493     if (substring) {
 494       const size_t offset = strlen("MacBook");
 495       if (!strncmp(model.get() + offset, "Air", 3) ||
 496           isdigit(model[offset + 1])) {
 497         return true;
 498       }
 499     }
 500   }
 501 #endif
 502   return false;
 503 }
 504
 505 void AudioCallbackDriver::Init(const nsCString& aStreamName) {
 506   LOG(LogLevel::Debug,
 507       ("%p: AudioCallbackDriver::Init driver=%p", Graph(), this));
 508   TRACE("AudioCallbackDriver::Init");
 509   MOZ_ASSERT(OnCubebOperationThread());
 510   MOZ_ASSERT(mAudioStreamState == AudioStreamState::Pending);
 511   FallbackDriverState fallbackState = mFallbackDriverState;
 512   if (fallbackState == FallbackDriverState::Stopped) {
 513     // The graph has already stopped us.
 514     return;
 515   }
 516   bool fromFallback = fallbackState == FallbackDriverState::Running;
 517   cubeb* cubebContext = CubebUtils::GetCubebContext();
 518   if (!cubebContext) {
 519     NS_WARNING("Could not get cubeb context.");
 520     LOG(LogLevel::Warning, ("%s: Could not get cubeb context", __func__));
 521     mAudioStreamState = AudioStreamState::None;
 522     if (!fromFallback) {
 523       CubebUtils::ReportCubebStreamInitFailure(true);
 524       FallbackToSystemClockDriver();
 525     }
 526     return;
 527   }
 528
 529   cubeb_stream_params output;
 530   cubeb_stream_params input;
 531   bool firstStream = CubebUtils::GetFirstStream();
 532
 533   MOZ_ASSERT(!NS_IsMainThread(),
 534              "This is blocking and should never run on the main thread.");
 535
 536   output.rate = mSampleRate;
 537   output.format = CUBEB_SAMPLE_FLOAT32NE;
 538
 539   if (!mOutputChannelCount) {
 540     LOG(LogLevel::Warning, ("Output number of channels is 0."));
 541     mAudioStreamState = AudioStreamState::None;
 542     if (!fromFallback) {
 543       CubebUtils::ReportCubebStreamInitFailure(firstStream);
 544       FallbackToSystemClockDriver();
 545     }
 546     return;
 547   }
 548
 549   CubebUtils::AudioDeviceID forcedOutputDeviceId = nullptr;
 550
 551   char* forcedOutputDeviceName = CubebUtils::GetForcedOutputDevice();
 552   if (forcedOutputDeviceName) {
 553     RefPtr<CubebDeviceEnumerator> enumerator = Enumerator::GetInstance();
 554     RefPtr<AudioDeviceInfo> device = enumerator->DeviceInfoFromName(
 555         NS_ConvertUTF8toUTF16(forcedOutputDeviceName), EnumeratorSide::OUTPUT);
 556     if (device && device->DeviceID()) {
 557       forcedOutputDeviceId = device->DeviceID();
 558     }
 559   }
 560
 561   mBuffer = AudioCallbackBufferWrapper<AudioDataValue>(mOutputChannelCount);
 562   mScratchBuffer =
 563       SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2>(mOutputChannelCount);
 564
 565   output.channels = mOutputChannelCount;
 566   AudioConfig::ChannelLayout::ChannelMap channelMap =
 567       AudioConfig::ChannelLayout(mOutputChannelCount).Map();
 568
 569   output.layout = static_cast<uint32_t>(channelMap);
 570   output.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_OUTPUT);
 571   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE &&
 572       CubebUtils::RouteOutputAsVoice()) {
 573     output.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
 574   }
 575
 576   uint32_t latencyFrames = CubebUtils::GetCubebMTGLatencyInFrames(&output);
 577
 578   LOG(LogLevel::Debug, ("Minimum latency in frames: %d", latencyFrames));
 579
 580   // Macbook and MacBook air don't have enough CPU to run very low latency
 581   // MediaTrackGraphs, cap the minimal latency to 512 frames int this case.
 582   if (IsMacbookOrMacbookAir()) {
 583     latencyFrames = std::max((uint32_t)512, latencyFrames);
 584     LOG(LogLevel::Debug,
 585         ("Macbook or macbook air, new latency: %d", latencyFrames));
 586   }
 587
 588   // Buffer sizes lower than 10ms are nowadays common. It's not very useful
 589   // when doing voice, because all the WebRTC code that does audio input
 590   // processing deals in 10ms chunks of audio. Take the first power of two
 591   // above 10ms at the current rate in this case. It's probably 512, for common
 592   // rates.
 593   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
 594     if (latencyFrames < mSampleRate / 100) {
 595       latencyFrames = mozilla::RoundUpPow2(mSampleRate / 100);
 596       LOG(LogLevel::Debug,
 597           ("AudioProcessing enabled, new latency %d", latencyFrames));
 598     }
 599   }
 600
 601   // It's not useful for the graph to run with a block size lower than the Web
 602   // Audio API block size, but increasingly devices report that they can do
 603   // audio latencies lower than that.
 604   if (latencyFrames < WEBAUDIO_BLOCK_SIZE) {
 605     LOG(LogLevel::Debug,
 606         ("Latency clamped to %d from %d", WEBAUDIO_BLOCK_SIZE, latencyFrames));
 607     latencyFrames = WEBAUDIO_BLOCK_SIZE;
 608   }
 609   LOG(LogLevel::Debug, ("Effective latency in frames: %d", latencyFrames));
 610
 611   input = output;
 612   input.channels = mInputChannelCount;
 613   input.layout = CUBEB_LAYOUT_UNDEFINED;
 614   input.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_INPUT);
 615   if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
 616     input.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
 617   }
 618
 619   cubeb_stream* stream = nullptr;
 620   const char* streamName =
 621       aStreamName.IsEmpty() ? "AudioCallbackDriver" : aStreamName.get();
 622   bool inputWanted = mInputChannelCount > 0;
 623   CubebUtils::AudioDeviceID outputId = mOutputDeviceID;
 624   CubebUtils::AudioDeviceID inputId = mInputDeviceID;
 625
 626   if (CubebUtils::CubebStreamInit(
 627           cubebContext, &stream, streamName, inputId,
 628           inputWanted ? &input : nullptr,
 629           forcedOutputDeviceId ? forcedOutputDeviceId : outputId, &output,
 630           latencyFrames, DataCallback_s, StateCallback_s, this) == CUBEB_OK) {
 631     mAudioStream.own(stream);
 632     DebugOnly<int> rv =
 633         cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
 634     NS_WARNING_ASSERTION(
 635         rv == CUBEB_OK,
 636         "Could not set the audio stream volume in GraphDriver.cpp");
 637     CubebUtils::ReportCubebBackendUsed();
 638   } else {
 639     NS_WARNING(
 640         "Could not create a cubeb stream for MediaTrackGraph, falling "
 641         "back to a SystemClockDriver");
 642     mAudioStreamState = AudioStreamState::None;
 643     // Only report failures when we're not coming from a driver that was
 644     // created itself as a fallback driver because of a previous audio driver
 645     // failure.
 646     if (!fromFallback) {
 647       CubebUtils::ReportCubebStreamInitFailure(firstStream);
 648       FallbackToSystemClockDriver();
 649     }
 650     return;
 651   }
 652
 653 #ifdef XP_MACOSX
 654   PanOutputIfNeeded(inputWanted);
 655 #endif
 656
 657   cubeb_stream_register_device_changed_callback(
 658       mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s);
 659
 660   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable. This
 661   // is intended for diagnosing issues, and only works if the content sandbox is
 662   // disabled.
 663   mInputStreamFile.Open("GraphDriverInput", input.channels, input.rate);
 664   mOutputStreamFile.Open("GraphDriverOutput", output.channels, output.rate);
 665
 666   if (NS_WARN_IF(!StartStream())) {
 667     LOG(LogLevel::Warning,
 668         ("%p: AudioCallbackDriver couldn't start a cubeb stream.", Graph()));
 669     return;
 670   }
 671
 672   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver started.", Graph()));
 673 }
 674
 675 void AudioCallbackDriver::SetCubebStreamName(const nsCString& aStreamName) {
 676   MOZ_ASSERT(OnCubebOperationThread());
 677   MOZ_ASSERT(mAudioStream);
 678   cubeb_stream_set_name(mAudioStream, aStreamName.get());
 679 }
 680
 681 void AudioCallbackDriver::Start() {
 682   MOZ_ASSERT(!IsStarted());
 683   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None);
 684   MOZ_ASSERT_IF(PreviousDriver(), PreviousDriver()->InIteration());
 685   mAudioStreamState = AudioStreamState::Pending;
 686
 687   if (mFallbackDriverState == FallbackDriverState::None) {
 688     // Starting an audio driver could take a while. We start a system driver in
 689     // the meantime so that the graph is kept running.
 690     FallbackToSystemClockDriver();
 691   }
 692
 693   if (mPreviousDriver) {
 694     if (AudioCallbackDriver* previousAudioCallback =
 695             mPreviousDriver->AsAudioCallbackDriver()) {
 696       LOG(LogLevel::Debug, ("Releasing audio driver off main thread."));
 697       mCubebOperationThread->Dispatch(NS_NewRunnableFunction(
 698           "AudioCallbackDriver previousDriver::Stop()",
 699           [previousDriver = RefPtr{previousAudioCallback}] {
 700             previousDriver->Stop();
 701           }));
 702     } else {
 703       LOG(LogLevel::Debug,
 704           ("Dropping driver reference for SystemClockDriver."));
 705       MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
 706     }
 707     mPreviousDriver = nullptr;
 708   }
 709
 710   LOG(LogLevel::Debug, ("Starting new audio driver off main thread, "
 711                         "to ensure it runs after previous shutdown."));
 712   mCubebOperationThread->Dispatch(
 713       NS_NewRunnableFunction("AudioCallbackDriver Init()",
 714                              [self = RefPtr{this}, streamName = mStreamName] {
 715                                self->Init(streamName);
 716                              }));
 717 }
 718
 719 bool AudioCallbackDriver::StartStream() {
 720   TRACE("AudioCallbackDriver::StartStream");
 721   MOZ_ASSERT(!IsStarted() && OnCubebOperationThread());
 722   // Set STARTING before cubeb_stream_start, since starting the cubeb stream
 723   // can result in a callback (that may read mAudioStreamState) before
 724   // mAudioStreamState would otherwise be set.
 725   mAudioStreamState = AudioStreamState::Starting;
 726   if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
 727     NS_WARNING("Could not start cubeb stream for MTG.");
 728     return false;
 729   }
 730
 731   return true;
 732 }
 733
 734 void AudioCallbackDriver::Stop() {
 735   LOG(LogLevel::Debug,
 736       ("%p: AudioCallbackDriver::Stop driver=%p", Graph(), this));
 737   TRACE("AudioCallbackDriver::Stop");
 738   MOZ_ASSERT(OnCubebOperationThread());
 739   cubeb_stream_register_device_changed_callback(mAudioStream, nullptr);
 740   if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
 741     NS_WARNING("Could not stop cubeb stream for MTG.");
 742   } else {
 743     mAudioStreamState = AudioStreamState::None;
 744   }
 745 }
 746
 747 void AudioCallbackDriver::Shutdown() {
 748   MOZ_ASSERT(NS_IsMainThread());
 749   RefPtr<FallbackWrapper> fallback;
 750   {
 751     auto fallbackLock = mFallback.Lock();
 752     fallback = fallbackLock.ref();
 753     fallbackLock.ref() = nullptr;
 754   }
 755   if (fallback) {
 756     LOG(LogLevel::Debug,
 757         ("%p: Releasing fallback driver %p.", Graph(), fallback.get()));
 758     fallback->Shutdown();
 759   }
 760
 761   LOG(LogLevel::Debug,
 762       ("%p: Releasing audio driver off main thread (GraphDriver::Shutdown).",
 763        Graph()));
 764
 765   nsLiteralCString reason("AudioCallbackDriver::Shutdown");
 766   NS_DispatchAndSpinEventLoopUntilComplete(
 767       reason, mCubebOperationThread,
 768       NS_NewRunnableFunction(reason.get(),
 769                              [self = RefPtr{this}] { self->Stop(); }));
 770 }
 771
 772 void AudioCallbackDriver::SetStreamName(const nsACString& aStreamName) {
 773   MOZ_ASSERT(InIteration() || !ThreadRunning());
 774   if (aStreamName == mStreamName) {
 775     return;
 776   }
 777   // Record the stream name, which will be passed onto the next driver, if
 778   // any, either from this driver or the fallback driver.
 779   GraphDriver::SetStreamName(aStreamName);
 780   {
 781     auto fallbackLock = mFallback.Lock();
 782     FallbackWrapper* fallback = fallbackLock.ref().get();
 783     if (fallback) {
 784       MOZ_ASSERT(fallback->InIteration());
 785       fallback->SetStreamName(aStreamName);
 786     }
 787   }
 788   AudioStreamState streamState = mAudioStreamState;
 789   if (streamState != AudioStreamState::None &&
 790       streamState != AudioStreamState::Stopping) {
 791     mCubebOperationThread->Dispatch(
 792         NS_NewRunnableFunction("AudioCallbackDriver SetStreamName()",
 793                                [self = RefPtr{this}, streamName = mStreamName] {
 794                                  self->SetCubebStreamName(streamName);
 795                                }));
 796   }
 797 }
 798
 799 /* static */
 800 long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser,
 801                                          const void* aInputBuffer,
 802                                          void* aOutputBuffer, long aFrames) {
 803   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
 804   return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
 805                               static_cast<AudioDataValue*>(aOutputBuffer),
 806                               aFrames);
 807 }
 808
 809 /* static */
 810 void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void* aUser,
 811                                           cubeb_state aState) {
 812   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
 813   driver->StateCallback(aState);
 814 }
 815
 816 /* static */
 817 void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) {
 818   AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
 819   driver->DeviceChangedCallback();
 820 }
 821
 822 AudioCallbackDriver::AutoInCallback::AutoInCallback(
 823     AudioCallbackDriver* aDriver)
 824     : mDriver(aDriver) {
 825   MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::thread::id());
 826   mDriver->mAudioThreadIdInCb = std::this_thread::get_id();
 827 }
 828
 829 AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
 830   MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::this_thread::get_id());
 831   mDriver->mAudioThreadIdInCb = std::thread::id();
 832 }
 833
 834 bool AudioCallbackDriver::CheckThreadIdChanged() {
 835   ProfilerThreadId id = profiler_current_thread_id();
 836   if (id != mAudioThreadId) {
 837     mAudioThreadId = id;
 838     return true;
 839   }
 840   return false;
 841 }
 842
 843 long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
 844                                        AudioDataValue* aOutputBuffer,
 845                                        long aFrames) {
 846   if (!mSandboxed && CheckThreadIdChanged()) {
 847     CallbackThreadRegistry::Get()->Register(mAudioThreadId,
 848                                             "NativeAudioCallback");
 849   }
 850
 851   if (mAudioStreamState.compareExchange(AudioStreamState::Starting,
 852                                         AudioStreamState::Running)) {
 853     LOG(LogLevel::Verbose, ("%p: AudioCallbackDriver %p First audio callback "
 854                             "close the Fallback driver",
 855                             Graph(), this));
 856   }
 857
 858   FallbackDriverState fallbackState = mFallbackDriverState;
 859   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Running)) {
 860     // Wait for the fallback driver to stop. Wake it up so it can stop if it's
 861     // sleeping.
 862     LOG(LogLevel::Verbose,
 863         ("%p: AudioCallbackDriver %p Waiting for the Fallback driver to stop",
 864          Graph(), this));
 865     EnsureNextIteration();
 866     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
 867     return aFrames;
 868   }
 869
 870   if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Stopped)) {
 871     // We're supposed to stop.
 872     PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
 873     if (!mSandboxed) {
 874       CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
 875     }
 876     return aFrames - 1;
 877   }
 878
 879   MOZ_ASSERT(mAudioStreamState == AudioStreamState::Running);
 880   TRACE_AUDIO_CALLBACK_BUDGET("AudioCallbackDriver real-time budget", aFrames,
 881                               mSampleRate);
 882   TRACE("AudioCallbackDriver::DataCallback");
 883
 884 #ifdef DEBUG
 885   AutoInCallback aic(this);
 886 #endif
 887
 888   uint32_t durationMS = aFrames * 1000 / mSampleRate;
 889
 890   // For now, simply average the duration with the previous
 891   // duration so there is some damping against sudden changes.
 892   if (!mIterationDurationMS) {
 893     mIterationDurationMS = durationMS;
 894   } else {
 895     mIterationDurationMS = (mIterationDurationMS * 3) + durationMS;
 896     mIterationDurationMS /= 4;
 897   }
 898
 899   mBuffer.SetBuffer(aOutputBuffer, aFrames);
 900   // fill part or all with leftover data from last iteration (since we
 901   // align to Audio blocks)
 902   uint32_t alreadyBuffered = mScratchBuffer.Empty(mBuffer);
 903
 904   // State computed time is decided by the audio callback's buffer length. We
 905   // compute the iteration start and end from there, trying to keep the amount
 906   // of buffering in the graph constant.
 907   GraphTime nextStateComputedTime =
 908       MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(mStateComputedTime +
 909                                                     mBuffer.Available());
 910
 911   auto iterationStart = mIterationEnd;
 912   // inGraph is the number of audio frames there is between the state time and
 913   // the current time, i.e. the maximum theoretical length of the interval we
 914   // could use as [iterationStart; mIterationEnd].
 915   GraphTime inGraph = mStateComputedTime - iterationStart;
 916   // We want the interval [iterationStart; mIterationEnd] to be before the
 917   // interval [mStateComputedTime; nextStateComputedTime]. We also want
 918   // the distance between these intervals to be roughly equivalent each time, to
 919   // ensure there is no clock drift between current time and state time. Since
 920   // we can't act on the state time because we have to fill the audio buffer, we
 921   // reclock the current time against the state time, here.
 922   mIterationEnd = iterationStart + 0.8 * inGraph;
 923
 924   LOG(LogLevel::Verbose,
 925       ("%p: interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) "
 926        "(duration ticks: %ld)",
 927        Graph(), (long)iterationStart, (long)mIterationEnd,
 928        (long)mStateComputedTime, (long)nextStateComputedTime, (long)aFrames,
 929        (uint32_t)durationMS,
 930        (long)(nextStateComputedTime - mStateComputedTime)));
 931
 932   if (mStateComputedTime < mIterationEnd) {
 933     LOG(LogLevel::Error, ("%p: Media graph global underrun detected", Graph()));
 934     MOZ_ASSERT_UNREACHABLE("We should not underrun in full duplex");
 935     mIterationEnd = mStateComputedTime;
 936   }
 937
 938   // Process mic data if any/needed
 939   if (aInputBuffer && mInputChannelCount > 0) {
 940     Graph()->NotifyInputData(aInputBuffer, static_cast<size_t>(aFrames),
 941                              mSampleRate, mInputChannelCount, alreadyBuffered);
 942   }
 943
 944   IterationResult result =
 945       Graph()->OneIteration(nextStateComputedTime, mIterationEnd, this);
 946
 947   mStateComputedTime = nextStateComputedTime;
 948
 949   MOZ_ASSERT(mBuffer.Available() == 0,
 950              "The graph should have filled the buffer");
 951
 952   mBuffer.BufferFilled();
 953
 954 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
 955   // Prevent returning NaN to the OS mixer, and propagating NaN into the reverse
 956   // stream of the AEC.
 957   NaNToZeroInPlace(aOutputBuffer, aFrames * mOutputChannelCount);
 958 #endif
 959
 960 #ifdef XP_MACOSX
 961   // This only happens when the output is on a macbookpro's external speaker,
 962   // that are stereo, but let's just be safe.
 963   if (mNeedsPanning && mOutputChannelCount == 2) {
 964     // hard pan to the right
 965     for (uint32_t i = 0; i < aFrames * 2; i += 2) {
 966       aOutputBuffer[i + 1] += aOutputBuffer[i];
 967       aOutputBuffer[i] = 0.0;
 968     }
 969   }
 970 #endif
 971
 972   // No-op if MOZ_DUMP_AUDIO is not defined as an environment variable
 973   if (aInputBuffer) {
 974     mInputStreamFile.Write(static_cast<const AudioDataValue*>(aInputBuffer),
 975                            aFrames * mInputChannelCount);
 976   }
 977   mOutputStreamFile.Write(static_cast<const AudioDataValue*>(aOutputBuffer),
 978                           aFrames * mOutputChannelCount);
 979
 980   if (result.IsStop()) {
 981     if (mInputDeviceID) {
 982       mGraphInterface->NotifyInputStopped();
 983     }
 984     // Signal that we have stopped.
 985     result.Stopped();
 986     // Update the flag before handing over the graph and going to drain.
 987     mAudioStreamState = AudioStreamState::Stopping;
 988     if (!mSandboxed) {
 989       CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
 990     }
 991     return aFrames - 1;
 992   }
 993
 994   if (GraphDriver* nextDriver = result.NextDriver()) {
 995     LOG(LogLevel::Debug,
 996         ("%p: Switching to %s driver.", Graph(),
 997          nextDriver->AsAudioCallbackDriver() ? "audio" : "system"));
 998     if (mInputDeviceID) {
 999       mGraphInterface->NotifyInputStopped();
1000     }
1001     result.Switched();
1002     mAudioStreamState = AudioStreamState::Stopping;
1003     nextDriver->SetState(mStreamName, mIterationEnd, mStateComputedTime);
1004     nextDriver->Start();
1005     if (!mSandboxed) {
1006       CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
1007     }
1008     // Returning less than aFrames starts the draining and eventually stops the
1009     // audio thread. This function will never get called again.
1010     return aFrames - 1;
1011   }
1012
1013   MOZ_ASSERT(result.IsStillProcessing());
1014   return aFrames;
1015 }
1016
1017 static const char* StateToString(cubeb_state aState) {
1018   switch (aState) {
1019     case CUBEB_STATE_STARTED:
1020       return "STARTED";
1021     case CUBEB_STATE_STOPPED:
1022       return "STOPPED";
1023     case CUBEB_STATE_DRAINED:
1024       return "DRAINED";
1025     case CUBEB_STATE_ERROR:
1026       return "ERROR";
1027     default:
1028       MOZ_CRASH("Unexpected state!");
1029   }
1030 }
1031
1032 void AudioCallbackDriver::StateCallback(cubeb_state aState) {
1033   MOZ_ASSERT(!InIteration());
1034   LOG(LogLevel::Debug,
1035       ("AudioCallbackDriver(%p) State: %s", this, StateToString(aState)));
1036
1037   if (aState == CUBEB_STATE_STARTED || aState == CUBEB_STATE_STOPPED) {
1038     // Nothing to do for STARTED.
1039     //
1040     // For STOPPED, don't reset mAudioStreamState until after
1041     // cubeb_stream_stop() returns, as wasapi_stream_stop() dispatches
1042     // CUBEB_STATE_STOPPED before ensuring that data callbacks have finished.
1043     // https://searchfox.org/mozilla-central/rev/f9beb753a84aa297713d1565dcd0c5e3c66e4174/media/libcubeb/src/cubeb_wasapi.cpp#3009,3012
1044     return;
1045   }
1046
1047   AudioStreamState streamState = mAudioStreamState;
1048   if (streamState < AudioStreamState::Starting) {
1049     // mAudioStream has already entered STOPPED, DRAINED, or ERROR.
1050     // Don't reset a Pending state indicating that a task to destroy
1051     // mAudioStream and init a new cubeb_stream has already been triggered.
1052     return;
1053   }
1054
1055   // Reset for DRAINED or ERROR.
1056   streamState = mAudioStreamState.exchange(AudioStreamState::None);
1057
1058   if (aState == CUBEB_STATE_ERROR) {
1059     // About to hand over control of the graph.  Do not start a new driver if
1060     // StateCallback() receives an error for this stream while the main thread
1061     // or another driver has control of the graph.
1062     if (streamState == AudioStreamState::Running) {
1063       if (mFallbackDriverState == FallbackDriverState::None) {
1064         // Only switch to fallback if it's not already running. It could be
1065         // running with the callback driver having started but not seen a single
1066         // callback yet. I.e., handover from fallback to callback is not done.
1067         if (mInputDeviceID) {
1068 #ifdef DEBUG
1069           // No audio callback after an error. We're calling into the graph here
1070           // so we need to be regarded as "in iteration".
1071           AutoInCallback aic(this);
1072 #endif
1073           mGraphInterface->NotifyInputStopped();
1074         }
1075         FallbackToSystemClockDriver();
1076       }
1077     }
1078   }
1079 }
1080
1081 void AudioCallbackDriver::MixerCallback(AudioChunk* aMixedBuffer,
1082                                         uint32_t aSampleRate) {
1083   MOZ_ASSERT(InIteration());
1084   uint32_t toWrite = mBuffer.Available();
1085
1086   TrackTime frameCount = aMixedBuffer->mDuration;
1087   if (!mBuffer.Available() && frameCount > 0) {
1088     NS_WARNING("DataCallback buffer full, expect frame drops.");
1089   }
1090
1091   MOZ_ASSERT(mBuffer.Available() <= frameCount);
1092
1093   mBuffer.WriteFrames(*aMixedBuffer, mBuffer.Available());
1094   MOZ_ASSERT(mBuffer.Available() == 0,
1095              "Missing frames to fill audio callback's buffer.");
1096   if (toWrite == frameCount) {
1097     return;
1098   }
1099
1100   aMixedBuffer->SliceTo(toWrite, frameCount);
1101   DebugOnly<uint32_t> written = mScratchBuffer.Fill(*aMixedBuffer);
1102   NS_WARNING_ASSERTION(written == frameCount - toWrite, "Dropping frames.");
1103 };
1104
1105 void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) {
1106 #ifdef XP_MACOSX
1107   TRACE("AudioCallbackDriver::PanOutputIfNeeded");
1108   cubeb_device* out = nullptr;
1109   int rv;
1110   char name[128];
1111   size_t length = sizeof(name);
1112
1113   rv = sysctlbyname("hw.model", name, &length, NULL, 0);
1114   if (rv) {
1115     return;
1116   }
1117
1118   int major, minor;
1119   for (uint32_t i = 0; i < length; i++) {
1120     // skip the model name
1121     if (isalpha(name[i])) {
1122       continue;
1123     }
1124     sscanf(name + i, "%d,%d", &major, &minor);
1125     break;
1126   }
1127
1128   enum MacbookModel { MacBook, MacBookPro, MacBookAir, NotAMacbook };
1129
1130   MacbookModel model;
1131
1132   if (!strncmp(name, "MacBookPro", length)) {
1133     model = MacBookPro;
1134   } else if (strncmp(name, "MacBookAir", length)) {
1135     model = MacBookAir;
1136   } else if (strncmp(name, "MacBook", length)) {
1137     model = MacBook;
1138   } else {
1139     model = NotAMacbook;
1140   }
1141   // For macbook pro before 2016 model (change of chassis), hard pan the audio
1142   // to the right if the speakers are in use to avoid feedback.
1143   if (model == MacBookPro && major <= 12) {
1144     if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
1145       MOZ_ASSERT(out);
1146       // Check if we are currently outputing sound on external speakers.
1147       if (out->output_name && !strcmp(out->output_name, "ispk")) {
1148         // Pan everything to the right speaker.
1149         LOG(LogLevel::Debug, ("Using the built-in speakers, with%s audio input",
1150                               aMicrophoneActive ? "" : "out"));
1151         mNeedsPanning = aMicrophoneActive;
1152       } else {
1153         LOG(LogLevel::Debug, ("Using an external output device"));
1154         mNeedsPanning = false;
1155       }
1156       cubeb_stream_device_destroy(mAudioStream, out);
1157     }
1158   }
1159 #endif
1160 }
1161
1162 void AudioCallbackDriver::DeviceChangedCallback() {
1163   MOZ_ASSERT(!InIteration());
1164   // Tell the audio engine the device has changed, it might want to reset some
1165   // state.
1166   Graph()->DeviceChanged();
1167 #ifdef XP_MACOSX
1168   RefPtr<AudioCallbackDriver> self(this);
1169   bool hasInput = mInputChannelCount;
1170   NS_DispatchBackgroundTask(NS_NewRunnableFunction(
1171       "PanOutputIfNeeded", [self{std::move(self)}, hasInput]() {
1172         self->PanOutputIfNeeded(hasInput);
1173       }));
1174 #endif
1175 }
1176
1177 uint32_t AudioCallbackDriver::IterationDuration() {
1178   MOZ_ASSERT(InIteration());
1179   // The real fix would be to have an API in cubeb to give us the number. Short
1180   // of that, we approximate it here. bug 1019507
1181   return mIterationDurationMS;
1182 }
1183
1184 void AudioCallbackDriver::EnsureNextIteration() {
1185   if (mFallbackDriverState == FallbackDriverState::Running) {
1186     auto fallback = mFallback.Lock();
1187     if (fallback.ref()) {
1188       fallback.ref()->EnsureNextIteration();
1189     }
1190   }
1191 }
1192
1193 TimeDuration AudioCallbackDriver::AudioOutputLatency() {
1194   TRACE("AudioCallbackDriver::AudioOutputLatency");
1195   uint32_t latencyFrames;
1196   int rv = cubeb_stream_get_latency(mAudioStream, &latencyFrames);
1197   if (rv || mSampleRate == 0) {
1198     return TimeDuration::FromSeconds(0.0);
1199   }
1200
1201   return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
1202                                    mSampleRate);
1203 }
1204
1205 bool AudioCallbackDriver::OnFallback() const {
1206   MOZ_ASSERT(InIteration());
1207   return mFallbackDriverState == FallbackDriverState::Running;
1208 }
1209
1210 void AudioCallbackDriver::FallbackToSystemClockDriver() {
1211   MOZ_ASSERT(!ThreadRunning());
1212   MOZ_ASSERT(mAudioStreamState == AudioStreamState::None ||
1213              mAudioStreamState == AudioStreamState::Pending);
1214   MOZ_ASSERT(mFallbackDriverState == FallbackDriverState::None);
1215   LOG(LogLevel::Debug,
1216       ("%p: AudioCallbackDriver %p Falling back to SystemClockDriver.", Graph(),
1217        this));
1218   mFallbackDriverState = FallbackDriverState::Running;
1219   mNextReInitBackoffStep =
1220       TimeDuration::FromMilliseconds(AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS);
1221   mNextReInitAttempt = TimeStamp::Now() + mNextReInitBackoffStep;
1222   auto fallback =
1223       MakeRefPtr<FallbackWrapper>(Graph(), this, mSampleRate, mStreamName,
1224                                   mIterationEnd, mStateComputedTime);
1225   {
1226     auto driver = mFallback.Lock();
1227     driver.ref() = fallback;
1228   }
1229   fallback->Start();
1230 }
1231
1232 void AudioCallbackDriver::FallbackDriverStopped(GraphTime aIterationEnd,
1233                                                 GraphTime aStateComputedTime,
1234                                                 FallbackDriverState aState) {
1235   mIterationEnd = aIterationEnd;
1236   mStateComputedTime = aStateComputedTime;
1237   mNextReInitAttempt = TimeStamp();
1238   mNextReInitBackoffStep = TimeDuration();
1239   {
1240     auto fallback = mFallback.Lock();
1241     MOZ_ASSERT(fallback.ref()->OnThread());
1242     fallback.ref() = nullptr;
1243   }
1244
1245   MOZ_ASSERT(aState == FallbackDriverState::None ||
1246              aState == FallbackDriverState::Stopped);
1247   MOZ_ASSERT_IF(aState == FallbackDriverState::None,
1248                 mAudioStreamState == AudioStreamState::Running);
1249   mFallbackDriverState = aState;
1250 }
1251
1252 void AudioCallbackDriver::MaybeStartAudioStream() {
1253   AudioStreamState streamState = mAudioStreamState;
1254   if (streamState != AudioStreamState::None) {
1255     LOG(LogLevel::Verbose,
1256         ("%p: AudioCallbackDriver %p Cannot re-init.", Graph(), this));
1257     return;
1258   }
1259
1260   TimeStamp now = TimeStamp::Now();
1261   if (now < mNextReInitAttempt) {
1262     LOG(LogLevel::Verbose,
1263         ("%p: AudioCallbackDriver %p Not time to re-init yet. %.3fs left.",
1264          Graph(), this, (mNextReInitAttempt - now).ToSeconds()));
1265     return;
1266   }
1267
1268   LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p Attempting to re-init "
1269                         "audio stream from fallback driver.",
1270                         Graph(), this));
1271   mNextReInitBackoffStep =
1272       std::min(mNextReInitBackoffStep * 2,
1273                TimeDuration::FromMilliseconds(
1274                    StaticPrefs::media_audio_device_retry_ms()));
1275   mNextReInitAttempt = now + mNextReInitBackoffStep;
1276   Start();
1277 }
1278
1279 }  // namespace mozilla
1280
1281 // avoid redefined macro in unified build
1282 #undef LOG