dom/media/webrtc/MediaEngineWebRTCAudio.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef MediaEngineWebRTCAudio_h
   7 #define MediaEngineWebRTCAudio_h
   8
   9 #include "AudioPacketizer.h"
  10 #include "AudioSegment.h"
  11 #include "AudioDeviceInfo.h"
  12 #include "DeviceInputTrack.h"
  13 #include "MediaEngineWebRTC.h"
  14 #include "MediaEnginePrefs.h"
  15 #include "MediaTrackListener.h"
  16 #include "modules/audio_processing/include/audio_processing.h"
  17
  18 namespace mozilla {
  19
  20 class AudioInputProcessing;
  21 class AudioProcessingTrack;
  22
  23 // This class is created and used exclusively on the Media Manager thread, with
  24 // exactly two exceptions:
  25 // - Pull is always called on the MTG thread. It only ever uses
  26 //   mInputProcessing. mInputProcessing is set, then a message is sent first to
  27 //   the main thread and then the MTG thread so that it can be used as part of
  28 //   the graph processing. On destruction, similarly, a message is sent to the
  29 //   graph so that it stops using it, and then it is deleted.
  30 // - mSettings is created on the MediaManager thread is always ever accessed on
  31 //   the Main Thread. It is const.
  32 class MediaEngineWebRTCMicrophoneSource : public MediaEngineSource {
  33  public:
  34   explicit MediaEngineWebRTCMicrophoneSource(const MediaDevice* aMediaDevice);
  35
  36   nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
  37                     const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
  38                     const char** aOutBadConstraint) override;
  39   nsresult Deallocate() override;
  40   void SetTrack(const RefPtr<MediaTrack>& aTrack,
  41                 const PrincipalHandle& aPrincipal) override;
  42   nsresult Start() override;
  43   nsresult Stop() override;
  44   nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,
  45                        const MediaEnginePrefs& aPrefs,
  46                        const char** aOutBadConstraint) override;
  47
  48   /**
  49    * Assigns the current settings of the capture to aOutSettings.
  50    * Main thread only.
  51    */
  52   void GetSettings(dom::MediaTrackSettings& aOutSettings) const override;
  53
  54   nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override {
  55     return NS_ERROR_NOT_IMPLEMENTED;
  56   }
  57
  58  protected:
  59   ~MediaEngineWebRTCMicrophoneSource() = default;
  60
  61  private:
  62   /**
  63    * From a set of constraints and about:config preferences, output the correct
  64    * set of preferences that can be sent to AudioInputProcessing.
  65    *
  66    * This can fail if the number of channels requested is zero, negative, or
  67    * more than the device supports.
  68    */
  69   nsresult EvaluateSettings(const NormalizedConstraints& aConstraintsUpdate,
  70                             const MediaEnginePrefs& aInPrefs,
  71                             MediaEnginePrefs* aOutPrefs,
  72                             const char** aOutBadConstraint);
  73   /**
  74    * From settings output by EvaluateSettings, send those settings to the
  75    * AudioInputProcessing instance and the main thread (for use in GetSettings).
  76    */
  77   void ApplySettings(const MediaEnginePrefs& aPrefs);
  78
  79   PrincipalHandle mPrincipal = PRINCIPAL_HANDLE_NONE;
  80
  81   const RefPtr<AudioDeviceInfo> mDeviceInfo;
  82
  83   // The maximum number of channels that this device supports.
  84   const uint32_t mDeviceMaxChannelCount;
  85   // The current settings for the underlying device.
  86   // Constructed on the MediaManager thread, and then only ever accessed on the
  87   // main thread.
  88   const nsMainThreadPtrHandle<media::Refcountable<dom::MediaTrackSettings>>
  89       mSettings;
  90
  91   // Current state of the resource for this source.
  92   MediaEngineSourceState mState;
  93
  94   // The current preferences that will be forwarded to mInputProcessing below.
  95   MediaEnginePrefs mCurrentPrefs;
  96
  97   // The AudioProcessingTrack used to inteface with the MediaTrackGraph. Set in
  98   // SetTrack as part of the initialization, and nulled in ::Deallocate.
  99   RefPtr<AudioProcessingTrack> mTrack;
 100
 101   // See note at the top of this class.
 102   RefPtr<AudioInputProcessing> mInputProcessing;
 103 };
 104
 105 // This class is created on the MediaManager thread, and then exclusively used
 106 // on the MTG thread.
 107 // All communication is done via message passing using MTG ControlMessages
 108 class AudioInputProcessing : public AudioDataListener {
 109  public:
 110   explicit AudioInputProcessing(uint32_t aMaxChannelCount);
 111   void Process(AudioProcessingTrack* aTrack, GraphTime aFrom, GraphTime aTo,
 112                AudioSegment* aInput, AudioSegment* aOutput);
 113
 114   void ProcessOutputData(AudioProcessingTrack* aTrack,
 115                          const AudioChunk& aChunk);
 116   bool IsVoiceInput(MediaTrackGraph* aGraph) const override {
 117     // If we're passing data directly without AEC or any other process, this
 118     // means that all voice-processing has been disabled intentionaly. In this
 119     // case, consider that the device is not used for voice input.
 120     return !IsPassThrough(aGraph);
 121   }
 122
 123   void Start(MediaTrackGraph* aGraph);
 124   void Stop(MediaTrackGraph* aGraph);
 125
 126   void DeviceChanged(MediaTrackGraph* aGraph) override;
 127
 128   uint32_t RequestedInputChannelCount(MediaTrackGraph*) override {
 129     return GetRequestedInputChannelCount();
 130   }
 131
 132   void Disconnect(MediaTrackGraph* aGraph) override;
 133
 134   void PacketizeAndProcess(AudioProcessingTrack* aTrack,
 135                            const AudioSegment& aSegment);
 136
 137   uint32_t GetRequestedInputChannelCount();
 138   // This is true when all processing is disabled, in which case we can skip
 139   // packetization, resampling and other processing passes.
 140   bool IsPassThrough(MediaTrackGraph* aGraph) const;
 141
 142   // This allow changing the APM options, enabling or disabling processing
 143   // steps. The settings get applied the next time we're about to process input
 144   // data.
 145   void ApplySettings(MediaTrackGraph* aGraph,
 146                      CubebUtils::AudioDeviceID aDeviceID,
 147                      const MediaEnginePrefs& aSettings);
 148
 149   void End();
 150
 151   TrackTime NumBufferedFrames(MediaTrackGraph* aGraph) const;
 152
 153   // The packet size contains samples in 10ms. The unit of aRate is hz.
 154   static uint32_t GetPacketSize(TrackRate aRate) {
 155     return webrtc::AudioProcessing::GetFrameSize(aRate);
 156   }
 157
 158   bool IsEnded() const { return mEnded; }
 159
 160  private:
 161   ~AudioInputProcessing() = default;
 162   webrtc::AudioProcessing::Config ConfigForPrefs(
 163       const MediaEnginePrefs& aPrefs);
 164   void PassThroughChanged(MediaTrackGraph* aGraph);
 165   void RequestedInputChannelCountChanged(MediaTrackGraph* aGraph,
 166                                          CubebUtils::AudioDeviceID aDeviceId);
 167   void EnsurePacketizer(AudioProcessingTrack* aTrack);
 168   void EnsureAudioProcessing(AudioProcessingTrack* aTrack);
 169   void ResetAudioProcessing(MediaTrackGraph* aGraph);
 170   PrincipalHandle GetCheckedPrincipal(const AudioSegment& aSegment);
 171   // This implements the processing algoritm to apply to the input (e.g. a
 172   // microphone). If all algorithms are disabled, this class in not used. This
 173   // class only accepts audio chunks of 10ms. It has two inputs and one output:
 174   // it is fed the speaker data and the microphone data. It outputs processed
 175   // input data.
 176   UniquePtr<webrtc::AudioProcessing> mAudioProcessing;
 177   // Packetizer to be able to feed 10ms packets to the input side of
 178   // mAudioProcessing. Not used if the processing is bypassed.
 179   Maybe<AudioPacketizer<AudioDataValue, float>> mPacketizerInput;
 180   // The current settings from about:config preferences and content-provided
 181   // constraints.
 182   MediaEnginePrefs mSettings;
 183   // Buffer for up to one 10ms packet of planar mixed audio output for the
 184   // reverse-stream (speaker data) of mAudioProcessing AEC.
 185   // Length is packet size * channel count, regardless of how many frames are
 186   // buffered.  Not used if the processing is bypassed.
 187   AlignedFloatBuffer mOutputBuffer;
 188   // Number of channels into which mOutputBuffer is divided.
 189   uint32_t mOutputBufferChannelCount = 0;
 190   // Number of frames buffered in mOutputBuffer for the reverse stream.
 191   uint32_t mOutputBufferFrameCount = 0;
 192   // Stores the input audio, to be processed by the APM.
 193   AlignedFloatBuffer mInputBuffer;
 194   // Stores the deinterleaved microphone audio
 195   AlignedFloatBuffer mDeinterleavedBuffer;
 196   // Stores the mixed down input audio
 197   AlignedFloatBuffer mInputDownmixBuffer;
 198   // Stores data waiting to be pulled.
 199   AudioSegment mSegment;
 200   // Whether or not this MediaEngine is enabled. If it's not enabled, it
 201   // operates in "pull" mode, and we append silence only, releasing the audio
 202   // input track.
 203   bool mEnabled;
 204   // Whether or not we've ended and removed the AudioProcessingTrack.
 205   bool mEnded;
 206   // When processing is enabled, the number of packets received by this
 207   // instance, to implement periodic logging.
 208   uint64_t mPacketCount;
 209   // Temporary descriptor for a slice of an AudioChunk parameter passed to
 210   // ProcessOutputData().  This is a member rather than on the stack so that
 211   // any memory allocated for its mChannelData pointer array is not
 212   // reallocated on each iteration.
 213   AudioChunk mSubChunk;
 214   // A storage holding the interleaved audio data converted the AudioSegment.
 215   // This will be used as an input parameter for PacketizeAndProcess. This
 216   // should be removed once bug 1729041 is done.
 217   AutoTArray<AudioDataValue,
 218              SilentChannel::AUDIO_PROCESSING_FRAMES * GUESS_AUDIO_CHANNELS>
 219       mInterleavedBuffer;
 220   // Tracks the pending frames with paired principals piled up in packetizer.
 221   std::deque<std::pair<TrackTime, PrincipalHandle>> mChunksInPacketizer;
 222 };
 223
 224 // MediaTrack subclass tailored for MediaEngineWebRTCMicrophoneSource.
 225 class AudioProcessingTrack : public DeviceInputConsumerTrack {
 226   // Only accessed on the graph thread.
 227   RefPtr<AudioInputProcessing> mInputProcessing;
 228
 229   explicit AudioProcessingTrack(TrackRate aSampleRate)
 230       : DeviceInputConsumerTrack(aSampleRate) {}
 231
 232   ~AudioProcessingTrack() = default;
 233
 234  public:
 235   // Main Thread API
 236   void Destroy() override;
 237   void SetInputProcessing(RefPtr<AudioInputProcessing> aInputProcessing);
 238   static AudioProcessingTrack* Create(MediaTrackGraph* aGraph);
 239
 240   // Graph Thread API
 241   void DestroyImpl() override;
 242   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 243   uint32_t NumberOfChannels() const override {
 244     MOZ_DIAGNOSTIC_ASSERT(
 245         mInputProcessing,
 246         "Must set mInputProcessing before exposing to content");
 247     return mInputProcessing->GetRequestedInputChannelCount();
 248   }
 249   // Pass the graph's mixed audio output to mInputProcessing for processing as
 250   // the reverse stream.
 251   void NotifyOutputData(MediaTrackGraph* aGraph, const AudioChunk& aChunk);
 252
 253   // Any thread
 254   AudioProcessingTrack* AsAudioProcessingTrack() override { return this; }
 255
 256  private:
 257   // Graph thread API
 258   void SetInputProcessingImpl(RefPtr<AudioInputProcessing> aInputProcessing);
 259 };
 260
 261 class MediaEngineWebRTCAudioCaptureSource : public MediaEngineSource {
 262  public:
 263   explicit MediaEngineWebRTCAudioCaptureSource(const MediaDevice* aMediaDevice);
 264   static nsString GetUUID();
 265   static nsString GetGroupId();
 266   nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
 267                     const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
 268                     const char** aOutBadConstraint) override {
 269     // Nothing to do here, everything is managed in MediaManager.cpp
 270     return NS_OK;
 271   }
 272   nsresult Deallocate() override {
 273     // Nothing to do here, everything is managed in MediaManager.cpp
 274     return NS_OK;
 275   }
 276   void SetTrack(const RefPtr<MediaTrack>& aTrack,
 277                 const PrincipalHandle& aPrincipal) override;
 278   nsresult Start() override;
 279   nsresult Stop() override;
 280   nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,
 281                        const MediaEnginePrefs& aPrefs,
 282                        const char** aOutBadConstraint) override;
 283
 284   nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override {
 285     return NS_ERROR_NOT_IMPLEMENTED;
 286   }
 287
 288   void GetSettings(dom::MediaTrackSettings& aOutSettings) const override;
 289
 290  protected:
 291   virtual ~MediaEngineWebRTCAudioCaptureSource() = default;
 292 };
 293
 294 }  // end namespace mozilla
 295
 296 #endif  // MediaEngineWebRTCAudio_h