content/renderer/media/webrtc_audio_device_impl.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_device_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/win/windows_version.h"
  11 #include "content/renderer/media/media_stream_audio_processor.h"
  12 #include "content/renderer/media/webrtc_audio_capturer.h"
  13 #include "content/renderer/media/webrtc_audio_renderer.h"
  14 #include "content/renderer/render_thread_impl.h"
  15 #include "media/audio/audio_parameters.h"
  16 #include "media/audio/sample_rates.h"
  17
  18 using media::AudioParameters;
  19 using media::ChannelLayout;
  20
  21 namespace content {
  22
  23 WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
  24     : ref_count_(0),
  25       audio_transport_callback_(NULL),
  26       input_delay_ms_(0),
  27       output_delay_ms_(0),
  28       initialized_(false),
  29       playing_(false),
  30       recording_(false),
  31       microphone_volume_(0),
  32       is_audio_track_processing_enabled_(
  33           MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled()) {
  34   DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
  35 }
  36
  37 WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
  38   DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
  39   DCHECK(thread_checker_.CalledOnValidThread());
  40   Terminate();
  41 }
  42
  43 int32_t WebRtcAudioDeviceImpl::AddRef() {
  44   DCHECK(thread_checker_.CalledOnValidThread());
  45   return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
  46 }
  47
  48 int32_t WebRtcAudioDeviceImpl::Release() {
  49   DCHECK(thread_checker_.CalledOnValidThread());
  50   int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
  51   if (ret == 0) {
  52     delete this;
  53   }
  54   return ret;
  55 }
  56 int WebRtcAudioDeviceImpl::OnData(const int16* audio_data,
  57                                   int sample_rate,
  58                                   int number_of_channels,
  59                                   int number_of_frames,
  60                                   const std::vector<int>& channels,
  61                                   int audio_delay_milliseconds,
  62                                   int current_volume,
  63                                   bool need_audio_processing,
  64                                   bool key_pressed) {
  65   int total_delay_ms = 0;
  66   {
  67     base::AutoLock auto_lock(lock_);
  68     // Return immediately when not recording or |channels| is empty.
  69     // See crbug.com/274017: renderer crash dereferencing invalid channels[0].
  70     if (!recording_ || channels.empty())
  71       return 0;
  72
  73     // Store the reported audio delay locally.
  74     input_delay_ms_ = audio_delay_milliseconds;
  75     total_delay_ms = input_delay_ms_ + output_delay_ms_;
  76     DVLOG(2) << "total delay: " << input_delay_ms_ + output_delay_ms_;
  77   }
  78
  79   // Write audio frames in blocks of 10 milliseconds to the registered
  80   // webrtc::AudioTransport sink. Keep writing until our internal byte
  81   // buffer is empty.
  82   const int16* audio_buffer = audio_data;
  83   const int frames_per_10_ms = (sample_rate / 100);
  84   CHECK_EQ(number_of_frames % frames_per_10_ms, 0);
  85   int accumulated_audio_frames = 0;
  86   uint32_t new_volume = 0;
  87
  88   // The lock here is to protect a race in the resampler inside webrtc when
  89   // there are more than one input stream calling OnData(), which can happen
  90   // when the users setup two getUserMedia, one for the microphone, another
  91   // for WebAudio. Currently we don't have a better way to fix it except for
  92   // adding a lock here to sequence the call.
  93   // TODO(xians): Remove this workaround after we move the
  94   // webrtc::AudioProcessing module to Chrome. See http://crbug/264611 for
  95   // details.
  96   base::AutoLock auto_lock(capture_callback_lock_);
  97   while (accumulated_audio_frames < number_of_frames) {
  98     // Deliver 10ms of recorded 16-bit linear PCM audio.
  99     int new_mic_level = audio_transport_callback_->OnDataAvailable(
 100         &channels[0],
 101         channels.size(),
 102         audio_buffer,
 103         sample_rate,
 104         number_of_channels,
 105         frames_per_10_ms,
 106         total_delay_ms,
 107         current_volume,
 108         key_pressed,
 109         need_audio_processing);
 110
 111     accumulated_audio_frames += frames_per_10_ms;
 112     audio_buffer += frames_per_10_ms * number_of_channels;
 113
 114     // The latest non-zero new microphone level will be returned.
 115     if (new_mic_level)
 116       new_volume = new_mic_level;
 117   }
 118
 119   return new_volume;
 120 }
 121
 122 void WebRtcAudioDeviceImpl::OnSetFormat(
 123     const media::AudioParameters& params) {
 124   DVLOG(1) << "WebRtcAudioDeviceImpl::OnSetFormat()";
 125 }
 126
 127 void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
 128                                        int sample_rate,
 129                                        int audio_delay_milliseconds) {
 130   render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
 131
 132   {
 133     base::AutoLock auto_lock(lock_);
 134     DCHECK(audio_transport_callback_);
 135     // Store the reported audio delay locally.
 136     output_delay_ms_ = audio_delay_milliseconds;
 137   }
 138
 139   int frames_per_10_ms = (sample_rate / 100);
 140   int bytes_per_sample = sizeof(render_buffer_[0]);
 141   const int bytes_per_10_ms =
 142       audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
 143   DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
 144
 145   // Get audio frames in blocks of 10 milliseconds from the registered
 146   // webrtc::AudioTransport source. Keep reading until our internal buffer
 147   // is full.
 148   uint32_t num_audio_frames = 0;
 149   int accumulated_audio_frames = 0;
 150   int16* audio_data = &render_buffer_[0];
 151   while (accumulated_audio_frames < audio_bus->frames()) {
 152     // Get 10ms and append output to temporary byte buffer.
 153     int64_t elapsed_time_ms = -1;
 154     int64_t ntp_time_ms = -1;
 155     if (is_audio_track_processing_enabled_) {
 156       // When audio processing is enabled in the audio track, we use
 157       // PullRenderData() instead of NeedMorePlayData() to avoid passing the
 158       // render data to the APM in WebRTC as reference signal for echo
 159       // cancellation.
 160       static const int kBitsPerByte = 8;
 161       audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
 162                                                 sample_rate,
 163                                                 audio_bus->channels(),
 164                                                 frames_per_10_ms,
 165                                                 audio_data,
 166                                                 &elapsed_time_ms,
 167                                                 &ntp_time_ms);
 168       accumulated_audio_frames += frames_per_10_ms;
 169     } else {
 170       // TODO(xians): Remove the following code after the APM in WebRTC is
 171       // deprecated.
 172       audio_transport_callback_->NeedMorePlayData(frames_per_10_ms,
 173                                                   bytes_per_sample,
 174                                                   audio_bus->channels(),
 175                                                   sample_rate,
 176                                                   audio_data,
 177                                                   num_audio_frames,
 178                                                   &elapsed_time_ms,
 179                                                   &ntp_time_ms);
 180       accumulated_audio_frames += num_audio_frames;
 181     }
 182
 183     audio_data += bytes_per_10_ms;
 184   }
 185
 186   // De-interleave each channel and convert to 32-bit floating-point
 187   // with nominal range -1.0 -> +1.0 to match the callback format.
 188   audio_bus->FromInterleaved(&render_buffer_[0],
 189                              audio_bus->frames(),
 190                              bytes_per_sample);
 191
 192   // Pass the render data to the playout sinks.
 193   base::AutoLock auto_lock(lock_);
 194   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 195        it != playout_sinks_.end(); ++it) {
 196     (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
 197   }
 198 }
 199
 200 void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
 201   DCHECK(thread_checker_.CalledOnValidThread());
 202   DCHECK_EQ(renderer, renderer_);
 203   base::AutoLock auto_lock(lock_);
 204   // Notify the playout sink of the change.
 205   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 206        it != playout_sinks_.end(); ++it) {
 207     (*it)->OnPlayoutDataSourceChanged();
 208   }
 209
 210   renderer_ = NULL;
 211   playing_ = false;
 212 }
 213
 214 int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
 215     webrtc::AudioTransport* audio_callback) {
 216   DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
 217   DCHECK(thread_checker_.CalledOnValidThread());
 218   DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
 219   audio_transport_callback_ = audio_callback;
 220   return 0;
 221 }
 222
 223 int32_t WebRtcAudioDeviceImpl::Init() {
 224   DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
 225   DCHECK(thread_checker_.CalledOnValidThread());
 226
 227   // We need to return a success to continue the initialization of WebRtc VoE
 228   // because failure on the capturer_ initialization should not prevent WebRTC
 229   // from working. See issue http://crbug.com/144421 for details.
 230   initialized_ = true;
 231
 232   return 0;
 233 }
 234
 235 int32_t WebRtcAudioDeviceImpl::Terminate() {
 236   DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
 237   DCHECK(thread_checker_.CalledOnValidThread());
 238
 239   // Calling Terminate() multiple times in a row is OK.
 240   if (!initialized_)
 241     return 0;
 242
 243   StopRecording();
 244   StopPlayout();
 245
 246   DCHECK(!renderer_.get() || !renderer_->IsStarted())
 247       << "The shared audio renderer shouldn't be running";
 248
 249   DisableAecDump();
 250
 251   capturers_.clear();
 252
 253   initialized_ = false;
 254   return 0;
 255 }
 256
 257 bool WebRtcAudioDeviceImpl::Initialized() const {
 258   return initialized_;
 259 }
 260
 261 int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
 262   *available = initialized_;
 263   return 0;
 264 }
 265
 266 bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
 267   return initialized_;
 268 }
 269
 270 int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
 271   *available = (!capturers_.empty());
 272   return 0;
 273 }
 274
 275 bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
 276   DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
 277   DCHECK(thread_checker_.CalledOnValidThread());
 278   return (!capturers_.empty());
 279 }
 280
 281 int32_t WebRtcAudioDeviceImpl::StartPlayout() {
 282   DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
 283   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 284   {
 285     base::AutoLock auto_lock(lock_);
 286     if (!audio_transport_callback_)
 287       return 0;
 288   }
 289
 290   if (playing_) {
 291     // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
 292     // that the call is ignored the second time.
 293     return 0;
 294   }
 295
 296   playing_ = true;
 297   return 0;
 298 }
 299
 300 int32_t WebRtcAudioDeviceImpl::StopPlayout() {
 301   DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
 302   if (!playing_) {
 303     // webrtc::VoiceEngine assumes that it is OK to call Stop() just in case.
 304     return 0;
 305   }
 306
 307   playing_ = false;
 308   return 0;
 309 }
 310
 311 bool WebRtcAudioDeviceImpl::Playing() const {
 312   return playing_;
 313 }
 314
 315 int32_t WebRtcAudioDeviceImpl::StartRecording() {
 316   DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
 317   DCHECK(initialized_);
 318   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 319   if (!audio_transport_callback_) {
 320     return -1;
 321   }
 322
 323   {
 324     base::AutoLock auto_lock(lock_);
 325     if (recording_)
 326       return 0;
 327
 328     recording_ = true;
 329   }
 330
 331   return 0;
 332 }
 333
 334 int32_t WebRtcAudioDeviceImpl::StopRecording() {
 335   DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
 336   {
 337     base::AutoLock auto_lock(lock_);
 338     if (!recording_)
 339       return 0;
 340
 341     recording_ = false;
 342   }
 343
 344   return 0;
 345 }
 346
 347 bool WebRtcAudioDeviceImpl::Recording() const {
 348   base::AutoLock auto_lock(lock_);
 349   return recording_;
 350 }
 351
 352 int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
 353   DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
 354   DCHECK(initialized_);
 355
 356   // Only one microphone is supported at the moment, which is represented by
 357   // the default capturer.
 358   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 359   if (!capturer.get())
 360     return -1;
 361
 362   capturer->SetVolume(volume);
 363   return 0;
 364 }
 365
 366 // TODO(henrika): sort out calling thread once we start using this API.
 367 int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
 368   DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
 369   // We only support one microphone now, which is accessed via the default
 370   // capturer.
 371   DCHECK(initialized_);
 372   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 373   if (!capturer.get())
 374     return -1;
 375
 376   *volume = static_cast<uint32_t>(capturer->Volume());
 377
 378   return 0;
 379 }
 380
 381 int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
 382   DCHECK(initialized_);
 383   *max_volume = kMaxVolumeLevel;
 384   return 0;
 385 }
 386
 387 int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
 388   *min_volume = 0;
 389   return 0;
 390 }
 391
 392 int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
 393   DCHECK(initialized_);
 394   *available = renderer_ && renderer_->channels() == 2;
 395   return 0;
 396 }
 397
 398 int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
 399     bool* available) const {
 400   DCHECK(initialized_);
 401   // TODO(xians): These kind of hardware methods do not make much sense since we
 402   // support multiple sources. Remove or figure out new APIs for such methods.
 403   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 404   if (!capturer.get())
 405     return -1;
 406
 407   *available = (capturer->source_audio_parameters().channels() == 2);
 408   return 0;
 409 }
 410
 411 int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
 412   base::AutoLock auto_lock(lock_);
 413   *delay_ms = static_cast<uint16_t>(output_delay_ms_);
 414   return 0;
 415 }
 416
 417 int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
 418   base::AutoLock auto_lock(lock_);
 419   *delay_ms = static_cast<uint16_t>(input_delay_ms_);
 420   return 0;
 421 }
 422
 423 int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
 424     uint32_t* sample_rate) const {
 425   // We use the default capturer as the recording sample rate.
 426   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 427   if (!capturer.get())
 428     return -1;
 429
 430   *sample_rate = static_cast<uint32_t>(
 431       capturer->source_audio_parameters().sample_rate());
 432   return 0;
 433 }
 434
 435 int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
 436     uint32_t* sample_rate) const {
 437   *sample_rate = renderer_ ? renderer_->sample_rate() : 0;
 438   return 0;
 439 }
 440
 441 bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
 442   DCHECK(thread_checker_.CalledOnValidThread());
 443   DCHECK(renderer);
 444
 445   base::AutoLock auto_lock(lock_);
 446   if (renderer_.get())
 447     return false;
 448
 449   if (!renderer->Initialize(this))
 450     return false;
 451
 452   renderer_ = renderer;
 453   return true;
 454 }
 455
 456 void WebRtcAudioDeviceImpl::AddAudioCapturer(
 457     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 458   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 459   DCHECK(thread_checker_.CalledOnValidThread());
 460   DCHECK(capturer.get());
 461   DCHECK(!capturer->device_id().empty());
 462   {
 463     base::AutoLock auto_lock(lock_);
 464     DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
 465         capturers_.end());
 466     capturers_.push_back(capturer);
 467   }
 468
 469   // Start the Aec dump if the Aec dump has been enabled and has not been
 470   // started.
 471   if (aec_dump_file_.IsValid())
 472     MaybeStartAecDump();
 473 }
 474
 475 void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
 476     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 477   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 478   DCHECK(thread_checker_.CalledOnValidThread());
 479   DCHECK(capturer.get());
 480   base::AutoLock auto_lock(lock_);
 481   capturers_.remove(capturer);
 482 }
 483
 484 scoped_refptr<WebRtcAudioCapturer>
 485 WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
 486   base::AutoLock auto_lock(lock_);
 487   // Use the last |capturer| which is from the latest getUserMedia call as
 488   // the default capture device.
 489   return capturers_.empty() ? NULL : capturers_.back();
 490 }
 491
 492 void WebRtcAudioDeviceImpl::AddPlayoutSink(
 493     WebRtcPlayoutDataSource::Sink* sink) {
 494   DCHECK(thread_checker_.CalledOnValidThread());
 495   DCHECK(sink);
 496   base::AutoLock auto_lock(lock_);
 497   DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
 498       playout_sinks_.end());
 499   playout_sinks_.push_back(sink);
 500 }
 501
 502 void WebRtcAudioDeviceImpl::RemovePlayoutSink(
 503     WebRtcPlayoutDataSource::Sink* sink) {
 504   DCHECK(thread_checker_.CalledOnValidThread());
 505   DCHECK(sink);
 506   base::AutoLock auto_lock(lock_);
 507   playout_sinks_.remove(sink);
 508 }
 509
 510 bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
 511     int* session_id,
 512     int* output_sample_rate,
 513     int* output_frames_per_buffer) {
 514   DCHECK(thread_checker_.CalledOnValidThread());
 515   // If there is no capturer or there are more than one open capture devices,
 516   // return false.
 517   if (capturers_.empty() || capturers_.size() > 1)
 518     return false;
 519
 520   return GetDefaultCapturer()->GetPairedOutputParameters(
 521       session_id, output_sample_rate, output_frames_per_buffer);
 522 }
 523
 524 void WebRtcAudioDeviceImpl::EnableAecDump(base::File aec_dump_file) {
 525   DCHECK(thread_checker_.CalledOnValidThread());
 526   DCHECK(aec_dump_file.IsValid());
 527
 528   // Close the previous AEC dump file description if it has not been consumed.
 529   // This can happen if no getUserMedia has been made yet.
 530   // TODO(xians): DCHECK(!aec_dump_file_.IsValid()) after the browser
 531   // guarantees it won't call EnableAecDump() more than once in a row.
 532   if (aec_dump_file_.IsValid())
 533     aec_dump_file_.Close();
 534
 535   aec_dump_file_ = aec_dump_file.Pass();
 536   MaybeStartAecDump();
 537 }
 538
 539 void WebRtcAudioDeviceImpl::DisableAecDump() {
 540   DCHECK(thread_checker_.CalledOnValidThread());
 541   // Simply invalidate the |aec_dump_file_| if we have not pass the ownership
 542   // to WebRtc.
 543   if (aec_dump_file_.IsValid()) {
 544     aec_dump_file_.Close();
 545     return;
 546   }
 547
 548   // We might have call StartAecDump() on one of the capturer. Loop
 549   // through all the capturers and call StopAecDump() on each of them.
 550   for (CapturerList::const_iterator iter = capturers_.begin();
 551        iter != capturers_.end(); ++iter) {
 552     (*iter)->StopAecDump();
 553   }
 554 }
 555
 556 void WebRtcAudioDeviceImpl::MaybeStartAecDump() {
 557   DCHECK(thread_checker_.CalledOnValidThread());
 558   DCHECK(aec_dump_file_.IsValid());
 559
 560   // Start the Aec dump on the current default capturer.
 561   scoped_refptr<WebRtcAudioCapturer> default_capturer(GetDefaultCapturer());
 562   if (!default_capturer)
 563     return;
 564
 565   default_capturer->StartAecDump(aec_dump_file_.Pass());
 566 }
 567
 568 }  // namespace content