dom/media/DeviceInputTrack.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef DOM_MEDIA_DEVICEINPUTTRACK_H_
   8 #define DOM_MEDIA_DEVICEINPUTTRACK_H_
   9
  10 #include <thread>
  11
  12 #include "AudioDriftCorrection.h"
  13 #include "AudioSegment.h"
  14 #include "AudioInputSource.h"
  15 #include "MediaTrackGraph.h"
  16 #include "GraphDriver.h"
  17 #include "mozilla/NotNull.h"
  18
  19 namespace mozilla {
  20
  21 class NativeInputTrack;
  22 class NonNativeInputTrack;
  23
  24 // Any MediaTrack that needs the audio data from the certain device should
  25 // inherit the this class and get the raw audio data on graph thread via
  26 // GetInputSourceData(), after calling ConnectDeviceInput() and before
  27 // DisconnectDeviceInput() on main thread. See more examples in
  28 // TestAudioTrackGraph.cpp
  29 //
  30 // Example:
  31 //
  32 // class RawAudioDataTrack : public DeviceInputConsumerTrack {
  33 //  public:
  34 //   ...
  35 //
  36 //   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override
  37 //   {
  38 //     if (aFrom >= aTo) {
  39 //       return;
  40 //     }
  41 //
  42 //     if (mInputs.IsEmpty()) {
  43 //       GetData<AudioSegment>()->AppendNullData(aTo - aFrom);
  44 //     } else {
  45 //       MOZ_ASSERT(mInputs.Length() == 1);
  46 //       AudioSegment data;
  47 //       DeviceInputConsumerTrack::GetInputSourceData(data, mInputs[0], aFrom,
  48 //                                                    aTo);
  49 //       // You can do audio data processing before appending to mSegment here.
  50 //       GetData<AudioSegment>()->AppendFrom(&data);
  51 //     }
  52 //   };
  53 //
  54 //   uint32_t NumberOfChannels() const override {
  55 //     if (mInputs.IsEmpty()) {
  56 //       return 0;
  57 //     }
  58 //     DeviceInputTrack* t = mInputs[0]->GetSource()->AsDeviceInputTrack();
  59 //     MOZ_ASSERT(t);
  60 //     return t->NumberOfChannels();
  61 //   }
  62 //
  63 //   ...
  64 //
  65 //  private:
  66 //   explicit RawAudioDataTrack(TrackRate aSampleRate)
  67 //       : DeviceInputConsumerTrack(aSampleRate) {}
  68 // };
  69 class DeviceInputConsumerTrack : public ProcessedMediaTrack {
  70  public:
  71   explicit DeviceInputConsumerTrack(TrackRate aSampleRate);
  72
  73   // Main Thread APIs:
  74   void ConnectDeviceInput(CubebUtils::AudioDeviceID aId,
  75                           AudioDataListener* aListener,
  76                           const PrincipalHandle& aPrincipal);
  77   void DisconnectDeviceInput();
  78   Maybe<CubebUtils::AudioDeviceID> DeviceId() const;
  79   NotNull<AudioDataListener*> GetAudioDataListener() const;
  80   bool ConnectToNativeDevice() const;
  81   bool ConnectToNonNativeDevice() const;
  82
  83   // Any thread:
  84   DeviceInputConsumerTrack* AsDeviceInputConsumerTrack() override {
  85     return this;
  86   }
  87
  88  protected:
  89   // Graph thread API:
  90   // Get the data in [aFrom, aTo) from aPort->GetSource() to aOutput. aOutput
  91   // needs to be empty.
  92   void GetInputSourceData(AudioSegment& aOutput, const MediaInputPort* aPort,
  93                           GraphTime aFrom, GraphTime aTo) const;
  94
  95   // Main Thread variables:
  96   RefPtr<MediaInputPort> mPort;
  97   RefPtr<DeviceInputTrack> mDeviceInputTrack;
  98   RefPtr<AudioDataListener> mListener;
  99   Maybe<CubebUtils::AudioDeviceID> mDeviceId;
 100 };
 101
 102 class DeviceInputTrack : public ProcessedMediaTrack {
 103  public:
 104   // Main Thread APIs:
 105   // Any MediaTrack that needs the audio data from the certain device should
 106   // inherit the DeviceInputConsumerTrack class and call GetInputSourceData to
 107   // get the data instead of using the below APIs.
 108   //
 109   // The following two APIs can create and destroy a DeviceInputTrack reference
 110   // on main thread, then open and close the underlying audio device accordingly
 111   // on the graph thread. The user who wants to read the audio input from a
 112   // certain device should use these APIs to obtain a DeviceInputTrack reference
 113   // and release the reference when the user no longer needs the audio data.
 114   //
 115   // Once the DeviceInputTrack is created on the main thread, the paired device
 116   // will start producing data, so its users can read the data immediately on
 117   // the graph thread, once they obtain the reference. The lifetime of
 118   // DeviceInputTrack is managed by the MediaTrackGraph itself. When the
 119   // DeviceInputTrack has no user any more, MediaTrackGraph will destroy it.
 120   // This means, it occurs when the last reference has been released by the API
 121   // below.
 122   //
 123   // The DeviceInputTrack is either a NativeInputTrack, or a
 124   // NonNativeInputTrack. We can have only one NativeInputTrack per
 125   // MediaTrackGraph, but multiple NonNativeInputTrack per graph. The audio
 126   // device paired with the NativeInputTrack is called "native device", and the
 127   // device paired with the NonNativeInputTrack is called "non-native device".
 128   // In other words, we can have only one native device per MediaTrackGraph, but
 129   // many non-native devices per graph.
 130   //
 131   // The native device is the first input device created in the MediaTrackGraph.
 132   // All other devices created after it will be non-native devices. Once the
 133   // native device is destroyed, the first non-native device will be promoted to
 134   // the new native device. The switch will be started by the MediaTrackGraph.
 135   // The MediaTrackGraph will force DeviceInputTrack's users to re-configure
 136   // their DeviceInputTrack connections with the APIs below to execute the
 137   // switching.
 138   //
 139   // The native device is also the audio input device serving the
 140   // AudioCallbackDriver, which drives the MediaTrackGraph periodically from
 141   // audio callback thread. The audio data produced by the native device and
 142   // non-native device is stored in NativeInputTrack and NonNativeInputTrack
 143   // respectively, and then accessed by their users. The only difference between
 144   // these audio data is that the data from the non-native device is
 145   // clock-drift-corrected since the non-native device may run on a different
 146   // clock than the native device's one.
 147   //
 148   // Example:
 149   //   // On main thread
 150   //   RefPtr<DeviceInputTrack> track = DeviceInputTrack::OpenAudio(...);
 151   //   ...
 152   //   // On graph thread
 153   //   AudioSegmen* data = track->GetData<AudioSegment>();
 154   //   ...
 155   //   // On main thread
 156   //   DeviceInputTrack::CloseAudio(track.forget(), ...);
 157   //
 158   // Returns a reference of DeviceInputTrack, storing the input audio data from
 159   // the given device, in the given MediaTrackGraph. The paired audio device
 160   // will be opened accordingly. The DeviceInputTrack will access its user's
 161   // audio settings via the attached AudioDataListener, and delivers the
 162   // notifications when it needs.
 163   static NotNull<RefPtr<DeviceInputTrack>> OpenAudio(
 164       MediaTrackGraphImpl* aGraph, CubebUtils::AudioDeviceID aDeviceId,
 165       const PrincipalHandle& aPrincipalHandle,
 166       DeviceInputConsumerTrack* aConsumer);
 167   // Destroy the DeviceInputTrack reference obtained by the above API. The
 168   // paired audio device will be closed accordingly.
 169   static void CloseAudio(already_AddRefed<DeviceInputTrack> aTrack,
 170                          DeviceInputConsumerTrack* aConsumer);
 171
 172   // Main thread API:
 173   const nsTArray<RefPtr<DeviceInputConsumerTrack>>& GetConsumerTracks() const;
 174
 175   // Graph thread APIs:
 176   // Query audio settings from its users.
 177   uint32_t MaxRequestedInputChannels() const;
 178   bool HasVoiceInput() const;
 179   // Deliver notification to its users.
 180   void DeviceChanged(MediaTrackGraphImpl* aGraph) const;
 181
 182   // Any thread:
 183   DeviceInputTrack* AsDeviceInputTrack() override { return this; }
 184   virtual NativeInputTrack* AsNativeInputTrack() { return nullptr; }
 185   virtual NonNativeInputTrack* AsNonNativeInputTrack() { return nullptr; }
 186
 187   // Any thread:
 188   const CubebUtils::AudioDeviceID mDeviceId;
 189   const PrincipalHandle mPrincipalHandle;
 190
 191  protected:
 192   DeviceInputTrack(TrackRate aSampleRate, CubebUtils::AudioDeviceID aDeviceId,
 193                    const PrincipalHandle& aPrincipalHandle);
 194   ~DeviceInputTrack() = default;
 195
 196  private:
 197   // Main thread APIs:
 198   void ReevaluateInputDevice();
 199   void AddDataListener(AudioDataListener* aListener);
 200   void RemoveDataListener(AudioDataListener* aListener);
 201
 202   // Only accessed on the main thread.
 203   // When this becomes empty, this DeviceInputTrack is no longer needed.
 204   nsTArray<RefPtr<DeviceInputConsumerTrack>> mConsumerTracks;
 205
 206   // Only accessed on the graph thread.
 207   nsTArray<RefPtr<AudioDataListener>> mListeners;
 208 };
 209
 210 class NativeInputTrack final : public DeviceInputTrack {
 211  public:
 212   // Do not call this directly. This can only be called in DeviceInputTrack or
 213   // tests.
 214   NativeInputTrack(TrackRate aSampleRate, CubebUtils::AudioDeviceID aDeviceId,
 215                    const PrincipalHandle& aPrincipalHandle);
 216
 217   // Graph Thread APIs, for ProcessedMediaTrack.
 218   void DestroyImpl() override;
 219   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 220   uint32_t NumberOfChannels() const override;
 221
 222   // Graph thread APIs: Get input audio data and event from graph.
 223   void NotifyInputStopped(MediaTrackGraphImpl* aGraph);
 224   void NotifyInputData(MediaTrackGraphImpl* aGraph,
 225                        const AudioDataValue* aBuffer, size_t aFrames,
 226                        TrackRate aRate, uint32_t aChannels,
 227                        uint32_t aAlreadyBuffered);
 228
 229   // Any thread
 230   NativeInputTrack* AsNativeInputTrack() override { return this; }
 231
 232  private:
 233   ~NativeInputTrack() = default;
 234
 235   // Graph thread only members:
 236   // Indicate whether we append extra frames in mPendingData. The extra number
 237   // of frames is in [0, WEBAUDIO_BLOCK_SIZE] range.
 238   bool mIsBufferingAppended = false;
 239   // Queue the audio input data coming from NotifyInputData.
 240   AudioSegment mPendingData;
 241   // The input channel count for the audio data.
 242   uint32_t mInputChannels = 0;
 243 };
 244
 245 class NonNativeInputTrack final : public DeviceInputTrack {
 246  public:
 247   // Do not call this directly. This can only be called in DeviceInputTrack or
 248   // tests.
 249   NonNativeInputTrack(TrackRate aSampleRate,
 250                       CubebUtils::AudioDeviceID aDeviceId,
 251                       const PrincipalHandle& aPrincipalHandle);
 252
 253   // Graph Thread APIs, for ProcessedMediaTrack
 254   void DestroyImpl() override;
 255   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 256   uint32_t NumberOfChannels() const override;
 257
 258   // Any thread
 259   NonNativeInputTrack* AsNonNativeInputTrack() override { return this; }
 260
 261   // Graph thread APIs:
 262   void StartAudio(RefPtr<AudioInputSource>&& aAudioInputSource);
 263   void StopAudio();
 264   AudioInputType DevicePreference() const;
 265   void NotifyDeviceChanged(AudioInputSource::Id aSourceId);
 266   void NotifyInputStopped(AudioInputSource::Id aSourceId);
 267   AudioInputSource::Id GenerateSourceId();
 268
 269  private:
 270   ~NonNativeInputTrack() = default;
 271
 272   // Graph driver thread only.
 273   bool CheckGraphDriverChanged();
 274
 275   // Graph thread only.
 276   RefPtr<AudioInputSource> mAudioSource;
 277   AudioInputSource::Id mSourceIdNumber;
 278
 279   // Graph driver thread only.
 280   std::thread::id mGraphDriverThreadId;
 281 };
 282
 283 class AudioInputSourceListener : public AudioInputSource::EventListener {
 284  public:
 285   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioInputSourceListener, override);
 286
 287   explicit AudioInputSourceListener(NonNativeInputTrack* aOwner);
 288
 289   // Main thread APIs:
 290   void AudioDeviceChanged(AudioInputSource::Id aSourceId) override;
 291   void AudioStateCallback(
 292       AudioInputSource::Id aSourceId,
 293       AudioInputSource::EventListener::State aState) override;
 294
 295  private:
 296   ~AudioInputSourceListener() = default;
 297   const RefPtr<NonNativeInputTrack> mOwner;
 298 };
 299
 300 }  // namespace mozilla
 301
 302 #endif  // DOM_MEDIA_DEVICEINPUTTRACK_H_