dom/media/webrtc/MediaEngineWebRTCAudio.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef MediaEngineWebRTCAudio_h
   7 #define MediaEngineWebRTCAudio_h
   8
   9 #include "AudioPacketizer.h"
  10 #include "AudioSegment.h"
  11 #include "AudioDeviceInfo.h"
  12 #include "DeviceInputTrack.h"
  13 #include "MediaEngineWebRTC.h"
  14 #include "MediaEnginePrefs.h"
  15 #include "MediaTrackListener.h"
  16 #include "modules/audio_processing/include/audio_processing.h"
  17
  18 namespace mozilla {
  19
  20 class AudioInputProcessing;
  21 class AudioProcessingTrack;
  22
  23 // This class is created and used exclusively on the Media Manager thread, with
  24 // exactly two exceptions:
  25 // - Pull is always called on the MTG thread. It only ever uses
  26 //   mInputProcessing. mInputProcessing is set, then a message is sent first to
  27 //   the main thread and then the MTG thread so that it can be used as part of
  28 //   the graph processing. On destruction, similarly, a message is sent to the
  29 //   graph so that it stops using it, and then it is deleted.
  30 // - mSettings is created on the MediaManager thread is always ever accessed on
  31 //   the Main Thread. It is const.
  32 class MediaEngineWebRTCMicrophoneSource : public MediaEngineSource {
  33  public:
  34   explicit MediaEngineWebRTCMicrophoneSource(const MediaDevice* aMediaDevice);
  35
  36   nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
  37                     const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
  38                     const char** aOutBadConstraint) override;
  39   nsresult Deallocate() override;
  40   void SetTrack(const RefPtr<MediaTrack>& aTrack,
  41                 const PrincipalHandle& aPrincipal) override;
  42   nsresult Start() override;
  43   nsresult Stop() override;
  44   nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,
  45                        const MediaEnginePrefs& aPrefs,
  46                        const char** aOutBadConstraint) override;
  47
  48   /**
  49    * Assigns the current settings of the capture to aOutSettings.
  50    * Main thread only.
  51    */
  52   void GetSettings(dom::MediaTrackSettings& aOutSettings) const override;
  53
  54   nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override {
  55     return NS_ERROR_NOT_IMPLEMENTED;
  56   }
  57
  58  protected:
  59   ~MediaEngineWebRTCMicrophoneSource() = default;
  60
  61  private:
  62   /**
  63    * From a set of constraints and about:config preferences, output the correct
  64    * set of preferences that can be sent to AudioInputProcessing.
  65    *
  66    * This can fail if the number of channels requested is zero, negative, or
  67    * more than the device supports.
  68    */
  69   nsresult EvaluateSettings(const NormalizedConstraints& aConstraintsUpdate,
  70                             const MediaEnginePrefs& aInPrefs,
  71                             MediaEnginePrefs* aOutPrefs,
  72                             const char** aOutBadConstraint);
  73   /**
  74    * From settings output by EvaluateSettings, send those settings to the
  75    * AudioInputProcessing instance and the main thread (for use in GetSettings).
  76    */
  77   void ApplySettings(const MediaEnginePrefs& aPrefs);
  78
  79   PrincipalHandle mPrincipal = PRINCIPAL_HANDLE_NONE;
  80
  81   const RefPtr<AudioDeviceInfo> mDeviceInfo;
  82
  83   // The maximum number of channels that this device supports.
  84   const uint32_t mDeviceMaxChannelCount;
  85   // The current settings for the underlying device.
  86   // Constructed on the MediaManager thread, and then only ever accessed on the
  87   // main thread.
  88   const nsMainThreadPtrHandle<media::Refcountable<dom::MediaTrackSettings>>
  89       mSettings;
  90
  91   // Current state of the resource for this source.
  92   MediaEngineSourceState mState;
  93
  94   // The current preferences that will be forwarded to mAudioProcessingConfig
  95   // below.
  96   MediaEnginePrefs mCurrentPrefs;
  97
  98   // The AudioProcessingTrack used to inteface with the MediaTrackGraph. Set in
  99   // SetTrack as part of the initialization, and nulled in ::Deallocate.
 100   RefPtr<AudioProcessingTrack> mTrack;
 101
 102   // See note at the top of this class.
 103   RefPtr<AudioInputProcessing> mInputProcessing;
 104
 105   // Copy of the config currently applied to AudioProcessing through
 106   // mInputProcessing.
 107   webrtc::AudioProcessing::Config mAudioProcessingConfig;
 108 };
 109
 110 // This class is created on the MediaManager thread, and then exclusively used
 111 // on the MTG thread.
 112 // All communication is done via message passing using MTG ControlMessages
 113 class AudioInputProcessing : public AudioDataListener {
 114  public:
 115   explicit AudioInputProcessing(uint32_t aMaxChannelCount);
 116   void Process(MediaTrackGraph* aGraph, GraphTime aFrom, GraphTime aTo,
 117                AudioSegment* aInput, AudioSegment* aOutput);
 118
 119   void ProcessOutputData(MediaTrackGraph* aGraph, const AudioChunk& aChunk);
 120   bool IsVoiceInput(MediaTrackGraph* aGraph) const override {
 121     // If we're passing data directly without AEC or any other process, this
 122     // means that all voice-processing has been disabled intentionaly. In this
 123     // case, consider that the device is not used for voice input.
 124     return !PassThrough(aGraph);
 125   }
 126
 127   void Start(MediaTrackGraph* aGraph);
 128   void Stop(MediaTrackGraph* aGraph);
 129
 130   void DeviceChanged(MediaTrackGraph* aGraph) override;
 131
 132   uint32_t RequestedInputChannelCount(MediaTrackGraph*) override {
 133     return GetRequestedInputChannelCount();
 134   }
 135
 136   void Disconnect(MediaTrackGraph* aGraph) override;
 137
 138   void PacketizeAndProcess(MediaTrackGraph* aGraph,
 139                            const AudioSegment& aSegment);
 140
 141   void SetPassThrough(MediaTrackGraph* aGraph, bool aPassThrough);
 142   uint32_t GetRequestedInputChannelCount();
 143   void SetRequestedInputChannelCount(MediaTrackGraph* aGraph,
 144                                      CubebUtils::AudioDeviceID aDeviceId,
 145                                      uint32_t aRequestedInputChannelCount);
 146   // This is true when all processing is disabled, we can skip
 147   // packetization, resampling and other processing passes.
 148   bool PassThrough(MediaTrackGraph* aGraph) const;
 149
 150   // This allow changing the APM options, enabling or disabling processing
 151   // steps. The config gets applied the next time we're about to process input
 152   // data.
 153   void ApplyConfig(MediaTrackGraph* aGraph,
 154                    const webrtc::AudioProcessing::Config& aConfig);
 155
 156   void End();
 157
 158   TrackTime NumBufferedFrames(MediaTrackGraph* aGraph) const;
 159
 160   // The packet size contains samples in 10ms. The unit of aRate is hz.
 161   static uint32_t GetPacketSize(TrackRate aRate) {
 162     return webrtc::AudioProcessing::GetFrameSize(aRate);
 163   }
 164
 165   bool IsEnded() const { return mEnded; }
 166
 167  private:
 168   ~AudioInputProcessing() = default;
 169   void EnsureAudioProcessing(MediaTrackGraph* aGraph, uint32_t aChannels);
 170   void ResetAudioProcessing(MediaTrackGraph* aGraph);
 171   PrincipalHandle GetCheckedPrincipal(const AudioSegment& aSegment);
 172   // This implements the processing algoritm to apply to the input (e.g. a
 173   // microphone). If all algorithms are disabled, this class in not used. This
 174   // class only accepts audio chunks of 10ms. It has two inputs and one output:
 175   // it is fed the speaker data and the microphone data. It outputs processed
 176   // input data.
 177   const UniquePtr<webrtc::AudioProcessing> mAudioProcessing;
 178   // Packetizer to be able to feed 10ms packets to the input side of
 179   // mAudioProcessing. Not used if the processing is bypassed.
 180   Maybe<AudioPacketizer<AudioDataValue, float>> mPacketizerInput;
 181   // The number of channels asked for by content, after clamping to the range of
 182   // legal channel count for this particular device.
 183   uint32_t mRequestedInputChannelCount;
 184   // mSkipProcessing is true if none of the processing passes are enabled,
 185   // because of prefs or constraints. This allows simply copying the audio into
 186   // the MTG, skipping resampling and the whole webrtc.org code.
 187   bool mSkipProcessing;
 188   // Buffer for up to one 10ms packet of planar mixed audio output for the
 189   // reverse-stream (speaker data) of mAudioProcessing AEC.
 190   // Length is packet size * channel count, regardless of how many frames are
 191   // buffered.  Not used if the processing is bypassed.
 192   AlignedFloatBuffer mOutputBuffer;
 193   // Number of channels into which mOutputBuffer is divided.
 194   uint32_t mOutputBufferChannelCount = 0;
 195   // Number of frames buffered in mOutputBuffer for the reverse stream.
 196   uint32_t mOutputBufferFrameCount = 0;
 197   // Stores the input audio, to be processed by the APM.
 198   AlignedFloatBuffer mInputBuffer;
 199   // Stores the deinterleaved microphone audio
 200   AlignedFloatBuffer mDeinterleavedBuffer;
 201   // Stores the mixed down input audio
 202   AlignedFloatBuffer mInputDownmixBuffer;
 203   // Stores data waiting to be pulled.
 204   AudioSegment mSegment;
 205   // Whether or not this MediaEngine is enabled. If it's not enabled, it
 206   // operates in "pull" mode, and we append silence only, releasing the audio
 207   // input track.
 208   bool mEnabled;
 209   // Whether or not we've ended and removed the AudioProcessingTrack.
 210   bool mEnded;
 211   // When processing is enabled, the number of packets received by this
 212   // instance, to implement periodic logging.
 213   uint64_t mPacketCount;
 214   // Temporary descriptor for a slice of an AudioChunk parameter passed to
 215   // ProcessOutputData().  This is a member rather than on the stack so that
 216   // any memory allocated for its mChannelData pointer array is not
 217   // reallocated on each iteration.
 218   AudioChunk mSubChunk;
 219   // A storage holding the interleaved audio data converted the AudioSegment.
 220   // This will be used as an input parameter for PacketizeAndProcess. This
 221   // should be removed once bug 1729041 is done.
 222   AutoTArray<AudioDataValue,
 223              SilentChannel::AUDIO_PROCESSING_FRAMES * GUESS_AUDIO_CHANNELS>
 224       mInterleavedBuffer;
 225   // Tracks the pending frames with paired principals piled up in packetizer.
 226   std::deque<std::pair<TrackTime, PrincipalHandle>> mChunksInPacketizer;
 227 };
 228
 229 // MediaTrack subclass tailored for MediaEngineWebRTCMicrophoneSource.
 230 class AudioProcessingTrack : public DeviceInputConsumerTrack {
 231   // Only accessed on the graph thread.
 232   RefPtr<AudioInputProcessing> mInputProcessing;
 233
 234   explicit AudioProcessingTrack(TrackRate aSampleRate)
 235       : DeviceInputConsumerTrack(aSampleRate) {}
 236
 237   ~AudioProcessingTrack() = default;
 238
 239  public:
 240   // Main Thread API
 241   void Destroy() override;
 242   void SetInputProcessing(RefPtr<AudioInputProcessing> aInputProcessing);
 243   static AudioProcessingTrack* Create(MediaTrackGraph* aGraph);
 244
 245   // Graph Thread API
 246   void DestroyImpl() override;
 247   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 248   uint32_t NumberOfChannels() const override {
 249     MOZ_DIAGNOSTIC_ASSERT(
 250         mInputProcessing,
 251         "Must set mInputProcessing before exposing to content");
 252     return mInputProcessing->GetRequestedInputChannelCount();
 253   }
 254   // Pass the graph's mixed audio output to mInputProcessing for processing as
 255   // the reverse stream.
 256   void NotifyOutputData(MediaTrackGraph* aGraph, const AudioChunk& aChunk);
 257
 258   // Any thread
 259   AudioProcessingTrack* AsAudioProcessingTrack() override { return this; }
 260
 261  private:
 262   // Graph thread API
 263   void SetInputProcessingImpl(RefPtr<AudioInputProcessing> aInputProcessing);
 264 };
 265
 266 class MediaEngineWebRTCAudioCaptureSource : public MediaEngineSource {
 267  public:
 268   explicit MediaEngineWebRTCAudioCaptureSource(const MediaDevice* aMediaDevice);
 269   static nsString GetUUID();
 270   static nsString GetGroupId();
 271   nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
 272                     const MediaEnginePrefs& aPrefs, uint64_t aWindowID,
 273                     const char** aOutBadConstraint) override {
 274     // Nothing to do here, everything is managed in MediaManager.cpp
 275     return NS_OK;
 276   }
 277   nsresult Deallocate() override {
 278     // Nothing to do here, everything is managed in MediaManager.cpp
 279     return NS_OK;
 280   }
 281   void SetTrack(const RefPtr<MediaTrack>& aTrack,
 282                 const PrincipalHandle& aPrincipal) override;
 283   nsresult Start() override;
 284   nsresult Stop() override;
 285   nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,
 286                        const MediaEnginePrefs& aPrefs,
 287                        const char** aOutBadConstraint) override;
 288
 289   nsresult TakePhoto(MediaEnginePhotoCallback* aCallback) override {
 290     return NS_ERROR_NOT_IMPLEMENTED;
 291   }
 292
 293   void GetSettings(dom::MediaTrackSettings& aOutSettings) const override;
 294
 295  protected:
 296   virtual ~MediaEngineWebRTCAudioCaptureSource() = default;
 297 };
 298
 299 }  // end namespace mozilla
 300
 301 #endif  // MediaEngineWebRTCAudio_h