1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MediaEngineWebRTCAudio_h
7 #define MediaEngineWebRTCAudio_h
9 #include "AudioPacketizer.h"
10 #include "AudioSegment.h"
11 #include "AudioDeviceInfo.h"
12 #include "DeviceInputTrack.h"
13 #include "MediaEngineWebRTC.h"
14 #include "MediaEnginePrefs.h"
15 #include "MediaTrackListener.h"
16 #include "modules/audio_processing/include/audio_processing.h"
20 class AudioInputProcessing
;
21 class AudioProcessingTrack
;
23 // This class is created and used exclusively on the Media Manager thread, with
24 // exactly two exceptions:
25 // - Pull is always called on the MTG thread. It only ever uses
26 // mInputProcessing. mInputProcessing is set, then a message is sent first to
27 // the main thread and then the MTG thread so that it can be used as part of
28 // the graph processing. On destruction, similarly, a message is sent to the
29 // graph so that it stops using it, and then it is deleted.
30 // - mSettings is created on the MediaManager thread is always ever accessed on
31 // the Main Thread. It is const.
32 class MediaEngineWebRTCMicrophoneSource
: public MediaEngineSource
{
34 explicit MediaEngineWebRTCMicrophoneSource(const MediaDevice
* aMediaDevice
);
36 nsresult
Allocate(const dom::MediaTrackConstraints
& aConstraints
,
37 const MediaEnginePrefs
& aPrefs
, uint64_t aWindowID
,
38 const char** aOutBadConstraint
) override
;
39 nsresult
Deallocate() override
;
40 void SetTrack(const RefPtr
<MediaTrack
>& aTrack
,
41 const PrincipalHandle
& aPrincipal
) override
;
42 nsresult
Start() override
;
43 nsresult
Stop() override
;
44 nsresult
Reconfigure(const dom::MediaTrackConstraints
& aConstraints
,
45 const MediaEnginePrefs
& aPrefs
,
46 const char** aOutBadConstraint
) override
;
49 * Assigns the current settings of the capture to aOutSettings.
52 void GetSettings(dom::MediaTrackSettings
& aOutSettings
) const override
;
54 nsresult
TakePhoto(MediaEnginePhotoCallback
* aCallback
) override
{
55 return NS_ERROR_NOT_IMPLEMENTED
;
59 ~MediaEngineWebRTCMicrophoneSource() = default;
63 * From a set of constraints and about:config preferences, output the correct
64 * set of preferences that can be sent to AudioInputProcessing.
66 * This can fail if the number of channels requested is zero, negative, or
67 * more than the device supports.
69 nsresult
EvaluateSettings(const NormalizedConstraints
& aConstraintsUpdate
,
70 const MediaEnginePrefs
& aInPrefs
,
71 MediaEnginePrefs
* aOutPrefs
,
72 const char** aOutBadConstraint
);
74 * From settings output by EvaluateSettings, send those settings to the
75 * AudioInputProcessing instance and the main thread (for use in GetSettings).
77 void ApplySettings(const MediaEnginePrefs
& aPrefs
);
79 PrincipalHandle mPrincipal
= PRINCIPAL_HANDLE_NONE
;
81 const RefPtr
<AudioDeviceInfo
> mDeviceInfo
;
83 // The maximum number of channels that this device supports.
84 const uint32_t mDeviceMaxChannelCount
;
85 // The current settings for the underlying device.
86 // Constructed on the MediaManager thread, and then only ever accessed on the
88 const nsMainThreadPtrHandle
<media::Refcountable
<dom::MediaTrackSettings
>>
91 // Current state of the resource for this source.
92 MediaEngineSourceState mState
;
94 // The current preferences that will be forwarded to mInputProcessing below.
95 MediaEnginePrefs mCurrentPrefs
;
97 // The AudioProcessingTrack used to inteface with the MediaTrackGraph. Set in
98 // SetTrack as part of the initialization, and nulled in ::Deallocate.
99 RefPtr
<AudioProcessingTrack
> mTrack
;
101 // See note at the top of this class.
102 RefPtr
<AudioInputProcessing
> mInputProcessing
;
105 // This class is created on the MediaManager thread, and then exclusively used
106 // on the MTG thread.
107 // All communication is done via message passing using MTG ControlMessages
108 class AudioInputProcessing
: public AudioDataListener
{
110 explicit AudioInputProcessing(uint32_t aMaxChannelCount
);
111 void Process(AudioProcessingTrack
* aTrack
, GraphTime aFrom
, GraphTime aTo
,
112 AudioSegment
* aInput
, AudioSegment
* aOutput
);
114 void ProcessOutputData(AudioProcessingTrack
* aTrack
,
115 const AudioChunk
& aChunk
);
116 bool IsVoiceInput(MediaTrackGraph
* aGraph
) const override
{
117 // If we're passing data directly without AEC or any other process, this
118 // means that all voice-processing has been disabled intentionaly. In this
119 // case, consider that the device is not used for voice input.
120 return !IsPassThrough(aGraph
);
123 void Start(MediaTrackGraph
* aGraph
);
124 void Stop(MediaTrackGraph
* aGraph
);
126 void DeviceChanged(MediaTrackGraph
* aGraph
) override
;
128 uint32_t RequestedInputChannelCount(MediaTrackGraph
*) override
{
129 return GetRequestedInputChannelCount();
132 void Disconnect(MediaTrackGraph
* aGraph
) override
;
134 void PacketizeAndProcess(AudioProcessingTrack
* aTrack
,
135 const AudioSegment
& aSegment
);
137 uint32_t GetRequestedInputChannelCount();
138 // This is true when all processing is disabled, in which case we can skip
139 // packetization, resampling and other processing passes.
140 bool IsPassThrough(MediaTrackGraph
* aGraph
) const;
142 // This allow changing the APM options, enabling or disabling processing
143 // steps. The settings get applied the next time we're about to process input
145 void ApplySettings(MediaTrackGraph
* aGraph
,
146 CubebUtils::AudioDeviceID aDeviceID
,
147 const MediaEnginePrefs
& aSettings
);
151 TrackTime
NumBufferedFrames(MediaTrackGraph
* aGraph
) const;
153 // The packet size contains samples in 10ms. The unit of aRate is hz.
154 static uint32_t GetPacketSize(TrackRate aRate
) {
155 return webrtc::AudioProcessing::GetFrameSize(aRate
);
158 bool IsEnded() const { return mEnded
; }
161 ~AudioInputProcessing() = default;
162 webrtc::AudioProcessing::Config
ConfigForPrefs(
163 const MediaEnginePrefs
& aPrefs
);
164 void PassThroughChanged(MediaTrackGraph
* aGraph
);
165 void RequestedInputChannelCountChanged(MediaTrackGraph
* aGraph
,
166 CubebUtils::AudioDeviceID aDeviceId
);
167 void EnsurePacketizer(AudioProcessingTrack
* aTrack
);
168 void EnsureAudioProcessing(AudioProcessingTrack
* aTrack
);
169 void ResetAudioProcessing(MediaTrackGraph
* aGraph
);
170 PrincipalHandle
GetCheckedPrincipal(const AudioSegment
& aSegment
);
171 // This implements the processing algoritm to apply to the input (e.g. a
172 // microphone). If all algorithms are disabled, this class in not used. This
173 // class only accepts audio chunks of 10ms. It has two inputs and one output:
174 // it is fed the speaker data and the microphone data. It outputs processed
176 UniquePtr
<webrtc::AudioProcessing
> mAudioProcessing
;
177 // Packetizer to be able to feed 10ms packets to the input side of
178 // mAudioProcessing. Not used if the processing is bypassed.
179 Maybe
<AudioPacketizer
<AudioDataValue
, float>> mPacketizerInput
;
180 // The current settings from about:config preferences and content-provided
182 MediaEnginePrefs mSettings
;
183 // Buffer for up to one 10ms packet of planar mixed audio output for the
184 // reverse-stream (speaker data) of mAudioProcessing AEC.
185 // Length is packet size * channel count, regardless of how many frames are
186 // buffered. Not used if the processing is bypassed.
187 AlignedFloatBuffer mOutputBuffer
;
188 // Number of channels into which mOutputBuffer is divided.
189 uint32_t mOutputBufferChannelCount
= 0;
190 // Number of frames buffered in mOutputBuffer for the reverse stream.
191 uint32_t mOutputBufferFrameCount
= 0;
192 // Stores the input audio, to be processed by the APM.
193 AlignedFloatBuffer mInputBuffer
;
194 // Stores the deinterleaved microphone audio
195 AlignedFloatBuffer mDeinterleavedBuffer
;
196 // Stores the mixed down input audio
197 AlignedFloatBuffer mInputDownmixBuffer
;
198 // Stores data waiting to be pulled.
199 AudioSegment mSegment
;
200 // Whether or not this MediaEngine is enabled. If it's not enabled, it
201 // operates in "pull" mode, and we append silence only, releasing the audio
204 // Whether or not we've ended and removed the AudioProcessingTrack.
206 // When processing is enabled, the number of packets received by this
207 // instance, to implement periodic logging.
208 uint64_t mPacketCount
;
209 // Temporary descriptor for a slice of an AudioChunk parameter passed to
210 // ProcessOutputData(). This is a member rather than on the stack so that
211 // any memory allocated for its mChannelData pointer array is not
212 // reallocated on each iteration.
213 AudioChunk mSubChunk
;
214 // A storage holding the interleaved audio data converted the AudioSegment.
215 // This will be used as an input parameter for PacketizeAndProcess. This
216 // should be removed once bug 1729041 is done.
217 AutoTArray
<AudioDataValue
,
218 SilentChannel::AUDIO_PROCESSING_FRAMES
* GUESS_AUDIO_CHANNELS
>
220 // Tracks the pending frames with paired principals piled up in packetizer.
221 std::deque
<std::pair
<TrackTime
, PrincipalHandle
>> mChunksInPacketizer
;
224 // MediaTrack subclass tailored for MediaEngineWebRTCMicrophoneSource.
225 class AudioProcessingTrack
: public DeviceInputConsumerTrack
{
226 // Only accessed on the graph thread.
227 RefPtr
<AudioInputProcessing
> mInputProcessing
;
229 explicit AudioProcessingTrack(TrackRate aSampleRate
)
230 : DeviceInputConsumerTrack(aSampleRate
) {}
232 ~AudioProcessingTrack() = default;
236 void Destroy() override
;
237 void SetInputProcessing(RefPtr
<AudioInputProcessing
> aInputProcessing
);
238 static AudioProcessingTrack
* Create(MediaTrackGraph
* aGraph
);
241 void DestroyImpl() override
;
242 void ProcessInput(GraphTime aFrom
, GraphTime aTo
, uint32_t aFlags
) override
;
243 uint32_t NumberOfChannels() const override
{
244 MOZ_DIAGNOSTIC_ASSERT(
246 "Must set mInputProcessing before exposing to content");
247 return mInputProcessing
->GetRequestedInputChannelCount();
249 // Pass the graph's mixed audio output to mInputProcessing for processing as
250 // the reverse stream.
251 void NotifyOutputData(MediaTrackGraph
* aGraph
, const AudioChunk
& aChunk
);
254 AudioProcessingTrack
* AsAudioProcessingTrack() override
{ return this; }
258 void SetInputProcessingImpl(RefPtr
<AudioInputProcessing
> aInputProcessing
);
261 class MediaEngineWebRTCAudioCaptureSource
: public MediaEngineSource
{
263 explicit MediaEngineWebRTCAudioCaptureSource(const MediaDevice
* aMediaDevice
);
264 static nsString
GetUUID();
265 static nsString
GetGroupId();
266 nsresult
Allocate(const dom::MediaTrackConstraints
& aConstraints
,
267 const MediaEnginePrefs
& aPrefs
, uint64_t aWindowID
,
268 const char** aOutBadConstraint
) override
{
269 // Nothing to do here, everything is managed in MediaManager.cpp
272 nsresult
Deallocate() override
{
273 // Nothing to do here, everything is managed in MediaManager.cpp
276 void SetTrack(const RefPtr
<MediaTrack
>& aTrack
,
277 const PrincipalHandle
& aPrincipal
) override
;
278 nsresult
Start() override
;
279 nsresult
Stop() override
;
280 nsresult
Reconfigure(const dom::MediaTrackConstraints
& aConstraints
,
281 const MediaEnginePrefs
& aPrefs
,
282 const char** aOutBadConstraint
) override
;
284 nsresult
TakePhoto(MediaEnginePhotoCallback
* aCallback
) override
{
285 return NS_ERROR_NOT_IMPLEMENTED
;
288 void GetSettings(dom::MediaTrackSettings
& aOutSettings
) const override
;
291 virtual ~MediaEngineWebRTCAudioCaptureSource() = default;
294 } // end namespace mozilla
296 #endif // MediaEngineWebRTCAudio_h