1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/audio/audio_input_controller.h"
8 #include "base/metrics/histogram_macros.h"
9 #include "base/strings/string_number_conversions.h"
10 #include "base/strings/stringprintf.h"
11 #include "base/threading/thread_restrictions.h"
12 #include "base/time/time.h"
13 #include "media/audio/audio_parameters.h"
14 #include "media/base/user_input_monitor.h"
16 using base::TimeDelta
;
20 const int kMaxInputChannels
= 3;
22 // TODO(henrika): remove usage of timers and add support for proper
23 // notification of when the input device is removed. This was originally added
24 // to resolve http://crbug.com/79936 for Windows platforms. This then caused
25 // breakage (very hard to repro bugs!) on other platforms: See
26 // http://crbug.com/226327 and http://crbug.com/230972.
27 // See also that the timer has been disabled on Mac now due to
29 const int kTimerResetIntervalSeconds
= 1;
30 // We have received reports that the timer can be too trigger happy on some
31 // Mac devices and the initial timer interval has therefore been increased
32 // from 1 second to 5 seconds.
33 const int kTimerInitialIntervalSeconds
= 5;
35 #if defined(AUDIO_POWER_MONITORING)
36 // Time in seconds between two successive measurements of audio power levels.
37 const int kPowerMonitorLogIntervalSeconds
= 15;
39 // A warning will be logged when the microphone audio volume is below this
41 const int kLowLevelMicrophoneLevelPercent
= 10;
43 // Logs if the user has enabled the microphone mute or not. This is normally
44 // done by marking a checkbox in an audio-settings UI which is unique for each
45 // platform. Elements in this enum should not be added, deleted or rearranged.
46 enum MicrophoneMuteResult
{
47 MICROPHONE_IS_MUTED
= 0,
48 MICROPHONE_IS_NOT_MUTED
= 1,
49 MICROPHONE_MUTE_MAX
= MICROPHONE_IS_NOT_MUTED
52 void LogMicrophoneMuteResult(MicrophoneMuteResult result
) {
53 UMA_HISTOGRAM_ENUMERATION("Media.MicrophoneMuted",
55 MICROPHONE_MUTE_MAX
+ 1);
58 // Helper method which calculates the average power of an audio bus. Unit is in
59 // dBFS, where 0 dBFS corresponds to all channels and samples equal to 1.0.
60 float AveragePower(const media::AudioBus
& buffer
) {
61 const int frames
= buffer
.frames();
62 const int channels
= buffer
.channels();
63 if (frames
<= 0 || channels
<= 0)
66 // Scan all channels and accumulate the sum of squares for all samples.
67 float sum_power
= 0.0f
;
68 for (int ch
= 0; ch
< channels
; ++ch
) {
69 const float* channel_data
= buffer
.channel(ch
);
70 for (int i
= 0; i
< frames
; i
++) {
71 const float sample
= channel_data
[i
];
72 sum_power
+= sample
* sample
;
76 // Update accumulated average results, with clamping for sanity.
77 const float average_power
=
78 std::max(0.0f
, std::min(1.0f
, sum_power
/ (frames
* channels
)));
80 // Convert average power level to dBFS units, and pin it down to zero if it
81 // is insignificantly small.
82 const float kInsignificantPower
= 1.0e-10f
; // -100 dBFS
83 const float power_dbfs
= average_power
< kInsignificantPower
?
84 -std::numeric_limits
<float>::infinity() : 10.0f
* log10f(average_power
);
88 #endif // AUDIO_POWER_MONITORING
92 // Used to log the result of capture startup.
93 // This was previously logged as a boolean with only the no callback and OK
94 // options. The enum order is kept to ensure backwards compatibility.
95 // Elements in this enum should not be deleted or rearranged; the only
96 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
97 // and update CAPTURE_STARTUP_RESULT_MAX.
98 enum CaptureStartupResult
{
99 CAPTURE_STARTUP_NO_DATA_CALLBACK
= 0,
100 CAPTURE_STARTUP_OK
= 1,
101 CAPTURE_STARTUP_CREATE_STREAM_FAILED
= 2,
102 CAPTURE_STARTUP_OPEN_STREAM_FAILED
= 3,
103 CAPTURE_STARTUP_RESULT_MAX
= CAPTURE_STARTUP_OPEN_STREAM_FAILED
106 void LogCaptureStartupResult(CaptureStartupResult result
) {
107 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
109 CAPTURE_STARTUP_RESULT_MAX
+ 1);
116 AudioInputController::Factory
* AudioInputController::factory_
= NULL
;
118 AudioInputController::AudioInputController(EventHandler
* handler
,
119 SyncWriter
* sync_writer
,
120 UserInputMonitor
* user_input_monitor
,
121 const bool agc_is_enabled
)
122 : creator_task_runner_(base::MessageLoopProxy::current()),
125 data_is_active_(false),
127 sync_writer_(sync_writer
),
129 user_input_monitor_(user_input_monitor
),
130 agc_is_enabled_(agc_is_enabled
),
131 #if defined(AUDIO_POWER_MONITORING)
132 power_measurement_is_enabled_(false),
133 log_silence_state_(false),
134 silence_state_(SILENCE_STATE_NO_MEASUREMENT
),
136 prev_key_down_count_(0) {
137 DCHECK(creator_task_runner_
.get());
140 AudioInputController::~AudioInputController() {
141 DCHECK_EQ(state_
, CLOSED
);
145 scoped_refptr
<AudioInputController
> AudioInputController::Create(
146 AudioManager
* audio_manager
,
147 EventHandler
* event_handler
,
148 const AudioParameters
& params
,
149 const std::string
& device_id
,
150 UserInputMonitor
* user_input_monitor
) {
151 DCHECK(audio_manager
);
153 if (!params
.IsValid() || (params
.channels() > kMaxInputChannels
))
157 return factory_
->Create(
158 audio_manager
, event_handler
, params
, user_input_monitor
);
160 scoped_refptr
<AudioInputController
> controller(
161 new AudioInputController(event_handler
, NULL
, user_input_monitor
, false));
163 controller
->task_runner_
= audio_manager
->GetTaskRunner();
165 // Create and open a new audio input stream from the existing
166 // audio-device thread.
167 if (!controller
->task_runner_
->PostTask(
169 base::Bind(&AudioInputController::DoCreate
,
171 base::Unretained(audio_manager
),
181 scoped_refptr
<AudioInputController
> AudioInputController::CreateLowLatency(
182 AudioManager
* audio_manager
,
183 EventHandler
* event_handler
,
184 const AudioParameters
& params
,
185 const std::string
& device_id
,
186 SyncWriter
* sync_writer
,
187 UserInputMonitor
* user_input_monitor
,
188 const bool agc_is_enabled
) {
189 DCHECK(audio_manager
);
192 if (!params
.IsValid() || (params
.channels() > kMaxInputChannels
))
195 // Create the AudioInputController object and ensure that it runs on
196 // the audio-manager thread.
197 scoped_refptr
<AudioInputController
> controller(new AudioInputController(
198 event_handler
, sync_writer
, user_input_monitor
, agc_is_enabled
));
199 controller
->task_runner_
= audio_manager
->GetTaskRunner();
201 // Create and open a new audio input stream from the existing
202 // audio-device thread. Use the provided audio-input device.
203 if (!controller
->task_runner_
->PostTask(
205 base::Bind(&AudioInputController::DoCreateForLowLatency
,
207 base::Unretained(audio_manager
),
217 scoped_refptr
<AudioInputController
> AudioInputController::CreateForStream(
218 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
,
219 EventHandler
* event_handler
,
220 AudioInputStream
* stream
,
221 SyncWriter
* sync_writer
,
222 UserInputMonitor
* user_input_monitor
) {
226 // Create the AudioInputController object and ensure that it runs on
227 // the audio-manager thread.
228 scoped_refptr
<AudioInputController
> controller(new AudioInputController(
229 event_handler
, sync_writer
, user_input_monitor
, false));
230 controller
->task_runner_
= task_runner
;
232 // TODO(miu): See TODO at top of file. Until that's resolved, we need to
233 // disable the error auto-detection here (since the audio mirroring
234 // implementation will reliably report error and close events). Note, of
235 // course, that we're assuming CreateForStream() has been called for the audio
236 // mirroring use case only.
237 if (!controller
->task_runner_
->PostTask(
239 base::Bind(&AudioInputController::DoCreateForStream
,
248 void AudioInputController::Record() {
249 task_runner_
->PostTask(FROM_HERE
, base::Bind(
250 &AudioInputController::DoRecord
, this));
253 void AudioInputController::Close(const base::Closure
& closed_task
) {
254 DCHECK(!closed_task
.is_null());
255 DCHECK(creator_task_runner_
->BelongsToCurrentThread());
257 task_runner_
->PostTaskAndReply(
258 FROM_HERE
, base::Bind(&AudioInputController::DoClose
, this), closed_task
);
261 void AudioInputController::SetVolume(double volume
) {
262 task_runner_
->PostTask(FROM_HERE
, base::Bind(
263 &AudioInputController::DoSetVolume
, this, volume
));
266 void AudioInputController::DoCreate(AudioManager
* audio_manager
,
267 const AudioParameters
& params
,
268 const std::string
& device_id
) {
269 DCHECK(task_runner_
->BelongsToCurrentThread());
270 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
272 handler_
->OnLog(this, "AIC::DoCreate");
274 #if defined(AUDIO_POWER_MONITORING)
275 // Disable power monitoring for streams that run without AGC enabled to
276 // avoid adding logs and UMA for non-WebRTC clients.
277 power_measurement_is_enabled_
= agc_is_enabled_
;
278 last_audio_level_log_time_
= base::TimeTicks::Now();
279 silence_state_
= SILENCE_STATE_NO_MEASUREMENT
;
282 // TODO(miu): See TODO at top of file. Until that's resolved, assume all
283 // platform audio input requires the |no_data_timer_| be used to auto-detect
284 // errors. In reality, probably only Windows needs to be treated as
286 DoCreateForStream(audio_manager
->MakeAudioInputStream(params
, device_id
));
289 void AudioInputController::DoCreateForLowLatency(AudioManager
* audio_manager
,
290 const AudioParameters
& params
,
291 const std::string
& device_id
) {
292 DCHECK(task_runner_
->BelongsToCurrentThread());
294 #if defined(AUDIO_POWER_MONITORING)
295 // We only log silence state UMA stats for low latency mode and if we use a
297 if (params
.format() != AudioParameters::AUDIO_FAKE
)
298 log_silence_state_
= true;
301 low_latency_create_time_
= base::TimeTicks::Now();
302 DoCreate(audio_manager
, params
, device_id
);
305 void AudioInputController::DoCreateForStream(
306 AudioInputStream
* stream_to_control
) {
307 DCHECK(task_runner_
->BelongsToCurrentThread());
310 stream_
= stream_to_control
;
314 handler_
->OnError(this, STREAM_CREATE_ERROR
);
315 LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED
);
319 if (stream_
&& !stream_
->Open()) {
323 handler_
->OnError(this, STREAM_OPEN_ERROR
);
324 LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED
);
328 DCHECK(!no_data_timer_
.get());
330 // Set AGC state using mode in |agc_is_enabled_| which can only be enabled in
331 // CreateLowLatency().
332 #if defined(AUDIO_POWER_MONITORING)
333 bool agc_is_supported
= false;
334 agc_is_supported
= stream_
->SetAutomaticGainControl(agc_is_enabled_
);
335 // Disable power measurements on platforms that does not support AGC at a
336 // lower level. AGC can fail on platforms where we don't support the
337 // functionality to modify the input volume slider. One such example is
339 power_measurement_is_enabled_
&= agc_is_supported
;
341 stream_
->SetAutomaticGainControl(agc_is_enabled_
);
344 // Create the data timer which will call FirstCheckForNoData(). The timer
345 // is started in DoRecord() and restarted in each DoCheckForNoData()
347 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
348 // from the timer must be ignored by the EventHandler.
349 // TODO(henrika): remove usage of timer when it has been verified on Canary
350 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
351 // everything that is tied to it. crbug.com/357569.
352 no_data_timer_
.reset(new base::Timer(
353 FROM_HERE
, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds
),
354 base::Bind(&AudioInputController::FirstCheckForNoData
,
355 base::Unretained(this)), false));
359 handler_
->OnCreated(this);
361 if (user_input_monitor_
) {
362 user_input_monitor_
->EnableKeyPressMonitoring();
363 prev_key_down_count_
= user_input_monitor_
->GetKeyPressCount();
367 void AudioInputController::DoRecord() {
368 DCHECK(task_runner_
->BelongsToCurrentThread());
369 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
371 if (state_
!= CREATED
)
375 base::AutoLock
auto_lock(lock_
);
380 handler_
->OnLog(this, "AIC::DoRecord");
382 if (no_data_timer_
) {
383 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
384 // a callback to FirstCheckForNoData() is made.
385 no_data_timer_
->Reset();
388 stream_
->Start(this);
390 handler_
->OnRecording(this);
393 void AudioInputController::DoClose() {
394 DCHECK(task_runner_
->BelongsToCurrentThread());
395 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
397 if (state_
== CLOSED
)
400 // If this is a low-latency stream, log the total duration (since DoCreate)
401 // and add it to a UMA histogram.
402 if (!low_latency_create_time_
.is_null()) {
403 base::TimeDelta duration
=
404 base::TimeTicks::Now() - low_latency_create_time_
;
405 UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDuration", duration
);
407 std::string log_string
=
408 base::StringPrintf("AIC::DoClose: stream duration=");
409 log_string
+= base::Int64ToString(duration
.InSeconds());
410 log_string
+= " seconds";
411 handler_
->OnLog(this, log_string
);
415 // Delete the timer on the same thread that created it.
416 no_data_timer_
.reset();
418 DoStopCloseAndClearStream();
419 SetDataIsActive(false);
421 if (SharedMemoryAndSyncSocketMode())
422 sync_writer_
->Close();
424 if (user_input_monitor_
)
425 user_input_monitor_
->DisableKeyPressMonitoring();
427 #if defined(AUDIO_POWER_MONITORING)
428 // Send UMA stats if enabled.
429 if (log_silence_state_
)
430 LogSilenceState(silence_state_
);
431 log_silence_state_
= false;
437 void AudioInputController::DoReportError() {
438 DCHECK(task_runner_
->BelongsToCurrentThread());
440 handler_
->OnError(this, STREAM_ERROR
);
443 void AudioInputController::DoSetVolume(double volume
) {
444 DCHECK(task_runner_
->BelongsToCurrentThread());
445 DCHECK_GE(volume
, 0);
446 DCHECK_LE(volume
, 1.0);
448 if (state_
!= CREATED
&& state_
!= RECORDING
)
451 // Only ask for the maximum volume at first call and use cached value
452 // for remaining function calls.
454 max_volume_
= stream_
->GetMaxVolume();
457 if (max_volume_
== 0.0) {
458 DLOG(WARNING
) << "Failed to access input volume control";
462 // Set the stream volume and scale to a range matched to the platform.
463 stream_
->SetVolume(max_volume_
* volume
);
466 void AudioInputController::FirstCheckForNoData() {
467 DCHECK(task_runner_
->BelongsToCurrentThread());
468 LogCaptureStartupResult(GetDataIsActive() ?
470 CAPTURE_STARTUP_NO_DATA_CALLBACK
);
472 handler_
->OnLog(this, GetDataIsActive() ?
473 "AIC::FirstCheckForNoData => data is active" :
474 "AIC::FirstCheckForNoData => data is NOT active");
479 void AudioInputController::DoCheckForNoData() {
480 DCHECK(task_runner_
->BelongsToCurrentThread());
482 if (!GetDataIsActive()) {
483 // The data-is-active marker will be false only if it has been more than
484 // one second since a data packet was recorded. This can happen if a
485 // capture device has been removed or disabled.
487 handler_
->OnError(this, NO_DATA_ERROR
);
490 // Mark data as non-active. The flag will be re-enabled in OnData() each
491 // time a data packet is received. Hence, under normal conditions, the
492 // flag will only be disabled during a very short period.
493 SetDataIsActive(false);
495 // Restart the timer to ensure that we check the flag again in
496 // |kTimerResetIntervalSeconds|.
497 no_data_timer_
->Start(
498 FROM_HERE
, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds
),
499 base::Bind(&AudioInputController::DoCheckForNoData
,
500 base::Unretained(this)));
503 void AudioInputController::OnData(AudioInputStream
* stream
,
504 const AudioBus
* source
,
505 uint32 hardware_delay_bytes
,
507 // Mark data as active to ensure that the periodic calls to
508 // DoCheckForNoData() does not report an error to the event handler.
509 SetDataIsActive(true);
512 base::AutoLock
auto_lock(lock_
);
513 if (state_
!= RECORDING
)
517 bool key_pressed
= false;
518 if (user_input_monitor_
) {
519 size_t current_count
= user_input_monitor_
->GetKeyPressCount();
520 key_pressed
= current_count
!= prev_key_down_count_
;
521 prev_key_down_count_
= current_count
;
522 DVLOG_IF(6, key_pressed
) << "Detected keypress.";
525 // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
526 // Used by all low-latency clients except WebSpeech.
527 if (SharedMemoryAndSyncSocketMode()) {
528 sync_writer_
->Write(source
, volume
, key_pressed
);
529 sync_writer_
->UpdateRecordedBytes(hardware_delay_bytes
);
531 #if defined(AUDIO_POWER_MONITORING)
532 // Only do power-level measurements if DoCreate() has been called. It will
533 // ensure that logging will mainly be done for WebRTC and WebSpeech
535 if (!power_measurement_is_enabled_
)
538 // Perform periodic audio (power) level measurements.
539 if ((base::TimeTicks::Now() - last_audio_level_log_time_
).InSeconds() >
540 kPowerMonitorLogIntervalSeconds
) {
541 // Calculate the average power of the signal, or the energy per sample.
542 const float average_power_dbfs
= AveragePower(*source
);
544 // Add current microphone volume to log and UMA histogram.
545 const int mic_volume_percent
= static_cast<int>(100.0 * volume
);
547 // Use event handler on the audio thread to relay a message to the ARIH
548 // in content which does the actual logging on the IO thread.
549 task_runner_
->PostTask(FROM_HERE
,
550 base::Bind(&AudioInputController::DoLogAudioLevels
,
553 mic_volume_percent
));
555 last_audio_level_log_time_
= base::TimeTicks::Now();
561 // TODO(henrika): Investigate if we can avoid the extra copy here.
562 // (see http://crbug.com/249316 for details). AFAIK, this scope is only
563 // active for WebSpeech clients.
564 scoped_ptr
<AudioBus
> audio_data
=
565 AudioBus::Create(source
->channels(), source
->frames());
566 source
->CopyTo(audio_data
.get());
568 // Ownership of the audio buffer will be with the callback until it is run,
569 // when ownership is passed to the callback function.
570 task_runner_
->PostTask(
573 &AudioInputController::DoOnData
, this, base::Passed(&audio_data
)));
576 void AudioInputController::DoOnData(scoped_ptr
<AudioBus
> data
) {
577 DCHECK(task_runner_
->BelongsToCurrentThread());
579 handler_
->OnData(this, data
.get());
582 void AudioInputController::DoLogAudioLevels(float level_dbfs
,
583 int microphone_volume_percent
) {
584 #if defined(AUDIO_POWER_MONITORING)
585 DCHECK(task_runner_
->BelongsToCurrentThread());
589 // Detect if the user has enabled hardware mute by pressing the mute
590 // button in audio settings for the selected microphone.
591 const bool microphone_is_muted
= stream_
->IsMuted();
592 if (microphone_is_muted
) {
593 LogMicrophoneMuteResult(MICROPHONE_IS_MUTED
);
594 handler_
->OnLog(this, "AIC::OnData: microphone is muted!");
595 // Return early if microphone is muted. No need to adding logs and UMA stats
596 // of audio levels if we know that the micropone is muted.
600 LogMicrophoneMuteResult(MICROPHONE_IS_NOT_MUTED
);
602 std::string log_string
= base::StringPrintf(
603 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs
);
604 static const float kSilenceThresholdDBFS
= -72.24719896f
;
605 if (level_dbfs
< kSilenceThresholdDBFS
)
606 log_string
+= " <=> low audio input level!";
607 handler_
->OnLog(this, log_string
);
609 UpdateSilenceState(level_dbfs
< kSilenceThresholdDBFS
);
611 UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent
);
612 log_string
= base::StringPrintf(
613 "AIC::OnData: microphone volume=%d%%", microphone_volume_percent
);
614 if (microphone_volume_percent
< kLowLevelMicrophoneLevelPercent
)
615 log_string
+= " <=> low microphone level!";
616 handler_
->OnLog(this, log_string
);
620 void AudioInputController::OnError(AudioInputStream
* stream
) {
621 // Handle error on the audio-manager thread.
622 task_runner_
->PostTask(FROM_HERE
, base::Bind(
623 &AudioInputController::DoReportError
, this));
626 void AudioInputController::DoStopCloseAndClearStream() {
627 DCHECK(task_runner_
->BelongsToCurrentThread());
629 // Allow calling unconditionally and bail if we don't have a stream to close.
630 if (stream_
!= NULL
) {
636 // The event handler should not be touched after the stream has been closed.
640 void AudioInputController::SetDataIsActive(bool enabled
) {
641 base::subtle::Release_Store(&data_is_active_
, enabled
);
644 bool AudioInputController::GetDataIsActive() {
645 return (base::subtle::Acquire_Load(&data_is_active_
) != false);
648 #if defined(AUDIO_POWER_MONITORING)
649 void AudioInputController::UpdateSilenceState(bool silence
) {
651 if (silence_state_
== SILENCE_STATE_NO_MEASUREMENT
) {
652 silence_state_
= SILENCE_STATE_ONLY_SILENCE
;
653 } else if (silence_state_
== SILENCE_STATE_ONLY_AUDIO
) {
654 silence_state_
= SILENCE_STATE_AUDIO_AND_SILENCE
;
656 DCHECK(silence_state_
== SILENCE_STATE_ONLY_SILENCE
||
657 silence_state_
== SILENCE_STATE_AUDIO_AND_SILENCE
);
660 if (silence_state_
== SILENCE_STATE_NO_MEASUREMENT
) {
661 silence_state_
= SILENCE_STATE_ONLY_AUDIO
;
662 } else if (silence_state_
== SILENCE_STATE_ONLY_SILENCE
) {
663 silence_state_
= SILENCE_STATE_AUDIO_AND_SILENCE
;
665 DCHECK(silence_state_
== SILENCE_STATE_ONLY_AUDIO
||
666 silence_state_
== SILENCE_STATE_AUDIO_AND_SILENCE
);
671 void AudioInputController::LogSilenceState(SilenceState value
) {
672 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
674 SILENCE_STATE_MAX
+ 1);