content/browser/media/media_internals.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/media_internals.h"
   6
   7 #include "base/metrics/histogram.h"
   8 #include "base/strings/string16.h"
   9 #include "base/strings/string_number_conversions.h"
  10 #include "base/strings/stringprintf.h"
  11 #include "content/public/browser/browser_thread.h"
  12 #include "content/public/browser/notification_observer.h"
  13 #include "content/public/browser/notification_registrar.h"
  14 #include "content/public/browser/notification_service.h"
  15 #include "content/public/browser/notification_types.h"
  16 #include "content/public/browser/render_process_host.h"
  17 #include "content/public/browser/web_ui.h"
  18 #include "media/audio/audio_parameters.h"
  19 #include "media/base/media_log_event.h"
  20 #include "media/filters/decrypting_video_decoder.h"
  21 #include "media/filters/gpu_video_decoder.h"
  22
  23 namespace {
  24
  25 static base::LazyInstance<content::MediaInternals>::Leaky g_media_internals =
  26     LAZY_INSTANCE_INITIALIZER;
  27
  28 base::string16 SerializeUpdate(const std::string& function,
  29                                const base::Value* value) {
  30   return content::WebUI::GetJavascriptCall(
  31       function, std::vector<const base::Value*>(1, value));
  32 }
  33
  34 std::string EffectsToString(int effects) {
  35   if (effects == media::AudioParameters::NO_EFFECTS)
  36     return "NO_EFFECTS";
  37
  38   struct {
  39     int flag;
  40     const char* name;
  41   } flags[] = {
  42     { media::AudioParameters::ECHO_CANCELLER, "ECHO_CANCELLER" },
  43     { media::AudioParameters::DUCKING, "DUCKING" },
  44     { media::AudioParameters::KEYBOARD_MIC, "KEYBOARD_MIC" },
  45     { media::AudioParameters::HOTWORD, "HOTWORD" },
  46   };
  47
  48   std::string ret;
  49   for (size_t i = 0; i < arraysize(flags); ++i) {
  50     if (effects & flags[i].flag) {
  51       if (!ret.empty())
  52         ret += " | ";
  53       ret += flags[i].name;
  54       effects &= ~flags[i].flag;
  55     }
  56   }
  57
  58   if (effects) {
  59     if (!ret.empty())
  60       ret += " | ";
  61     ret += base::IntToString(effects);
  62   }
  63
  64   return ret;
  65 }
  66
  67 std::string FormatToString(media::AudioParameters::Format format) {
  68   switch (format) {
  69     case media::AudioParameters::AUDIO_PCM_LINEAR:
  70       return "pcm_linear";
  71     case media::AudioParameters::AUDIO_PCM_LOW_LATENCY:
  72       return "pcm_low_latency";
  73     case media::AudioParameters::AUDIO_FAKE:
  74       return "fake";
  75     case media::AudioParameters::AUDIO_LAST_FORMAT:
  76       break;
  77   }
  78
  79   NOTREACHED();
  80   return "unknown";
  81 }
  82
  83 const char kAudioLogStatusKey[] = "status";
  84 const char kAudioLogUpdateFunction[] = "media.updateAudioComponent";
  85
  86 }  // namespace
  87
  88 namespace content {
  89
  90 class AudioLogImpl : public media::AudioLog {
  91  public:
  92   AudioLogImpl(int owner_id,
  93                media::AudioLogFactory::AudioComponent component,
  94                content::MediaInternals* media_internals);
  95   ~AudioLogImpl() override;
  96
  97   void OnCreated(int component_id,
  98                  const media::AudioParameters& params,
  99                  const std::string& device_id) override;
 100   void OnStarted(int component_id) override;
 101   void OnStopped(int component_id) override;
 102   void OnClosed(int component_id) override;
 103   void OnError(int component_id) override;
 104   void OnSetVolume(int component_id, double volume) override;
 105
 106  private:
 107   void SendSingleStringUpdate(int component_id,
 108                               const std::string& key,
 109                               const std::string& value);
 110   void StoreComponentMetadata(int component_id, base::DictionaryValue* dict);
 111   std::string FormatCacheKey(int component_id);
 112
 113   const int owner_id_;
 114   const media::AudioLogFactory::AudioComponent component_;
 115   content::MediaInternals* const media_internals_;
 116
 117   DISALLOW_COPY_AND_ASSIGN(AudioLogImpl);
 118 };
 119
 120 AudioLogImpl::AudioLogImpl(int owner_id,
 121                            media::AudioLogFactory::AudioComponent component,
 122                            content::MediaInternals* media_internals)
 123     : owner_id_(owner_id),
 124       component_(component),
 125       media_internals_(media_internals) {}
 126
 127 AudioLogImpl::~AudioLogImpl() {}
 128
 129 void AudioLogImpl::OnCreated(int component_id,
 130                              const media::AudioParameters& params,
 131                              const std::string& device_id) {
 132   base::DictionaryValue dict;
 133   StoreComponentMetadata(component_id, &dict);
 134
 135   dict.SetString(kAudioLogStatusKey, "created");
 136   dict.SetString("device_id", device_id);
 137   dict.SetString("device_type", FormatToString(params.format()));
 138   dict.SetInteger("frames_per_buffer", params.frames_per_buffer());
 139   dict.SetInteger("sample_rate", params.sample_rate());
 140   dict.SetInteger("channels", params.channels());
 141   dict.SetString("channel_layout",
 142                  ChannelLayoutToString(params.channel_layout()));
 143   dict.SetString("effects", EffectsToString(params.effects()));
 144
 145   media_internals_->SendUpdateAndCacheAudioStreamKey(
 146       FormatCacheKey(component_id), kAudioLogUpdateFunction, &dict);
 147 }
 148
 149 void AudioLogImpl::OnStarted(int component_id) {
 150   SendSingleStringUpdate(component_id, kAudioLogStatusKey, "started");
 151 }
 152
 153 void AudioLogImpl::OnStopped(int component_id) {
 154   SendSingleStringUpdate(component_id, kAudioLogStatusKey, "stopped");
 155 }
 156
 157 void AudioLogImpl::OnClosed(int component_id) {
 158   base::DictionaryValue dict;
 159   StoreComponentMetadata(component_id, &dict);
 160   dict.SetString(kAudioLogStatusKey, "closed");
 161   media_internals_->SendUpdateAndPurgeAudioStreamCache(
 162       FormatCacheKey(component_id), kAudioLogUpdateFunction, &dict);
 163 }
 164
 165 void AudioLogImpl::OnError(int component_id) {
 166   SendSingleStringUpdate(component_id, "error_occurred", "true");
 167 }
 168
 169 void AudioLogImpl::OnSetVolume(int component_id, double volume) {
 170   base::DictionaryValue dict;
 171   StoreComponentMetadata(component_id, &dict);
 172   dict.SetDouble("volume", volume);
 173   media_internals_->SendUpdateAndCacheAudioStreamKey(
 174       FormatCacheKey(component_id), kAudioLogUpdateFunction, &dict);
 175 }
 176
 177 std::string AudioLogImpl::FormatCacheKey(int component_id) {
 178   return base::StringPrintf("%d:%d:%d", owner_id_, component_, component_id);
 179 }
 180
 181 void AudioLogImpl::SendSingleStringUpdate(int component_id,
 182                                           const std::string& key,
 183                                           const std::string& value) {
 184   base::DictionaryValue dict;
 185   StoreComponentMetadata(component_id, &dict);
 186   dict.SetString(key, value);
 187   media_internals_->SendUpdateAndCacheAudioStreamKey(
 188       FormatCacheKey(component_id), kAudioLogUpdateFunction, &dict);
 189 }
 190
 191 void AudioLogImpl::StoreComponentMetadata(int component_id,
 192                                           base::DictionaryValue* dict) {
 193   dict->SetInteger("owner_id", owner_id_);
 194   dict->SetInteger("component_id", component_id);
 195   dict->SetInteger("component_type", component_);
 196 }
 197
 198 class MediaInternals::MediaInternalsUMAHandler : public NotificationObserver {
 199  public:
 200   MediaInternalsUMAHandler();
 201
 202   // NotificationObserver implementation.
 203   void Observe(int type,
 204                const NotificationSource& source,
 205                const NotificationDetails& details) override;
 206
 207   // Reports the pipeline status to UMA for every player
 208   // associated with the renderer process and then deletes the player state.
 209   void LogAndClearPlayersInRenderer(int render_process_id);
 210
 211   // Helper function to save the event payload to RendererPlayerMap.
 212   void SavePlayerState(const media::MediaLogEvent& event,
 213                        int render_process_id);
 214
 215  private:
 216   struct PipelineInfo {
 217     media::PipelineStatus last_pipeline_status;
 218     bool has_audio;
 219     bool has_video;
 220     bool video_dds;
 221     bool video_decoder_changed;
 222     std::string audio_codec_name;
 223     std::string video_codec_name;
 224     std::string video_decoder;
 225     PipelineInfo()
 226         : last_pipeline_status(media::PIPELINE_OK),
 227           has_audio(false),
 228           has_video(false),
 229           video_dds(false),
 230           video_decoder_changed(false) {}
 231   };
 232
 233   // Helper function to report PipelineStatus associated with a player to UMA.
 234   void ReportUMAForPipelineStatus(const PipelineInfo& player_info);
 235
 236   // Helper to generate PipelineStatus UMA name for AudioVideo streams.
 237   std::string GetUMANameForAVStream(const PipelineInfo& player_info);
 238
 239   // Key is playerid
 240   typedef std::map<int, PipelineInfo> PlayerInfoMap;
 241
 242   // Key is renderer id
 243   typedef std::map<int, PlayerInfoMap> RendererPlayerMap;
 244
 245   // Stores player information per renderer
 246   RendererPlayerMap renderer_info_;
 247
 248   NotificationRegistrar registrar_;
 249
 250   DISALLOW_COPY_AND_ASSIGN(MediaInternalsUMAHandler);
 251 };
 252
 253 MediaInternals::MediaInternalsUMAHandler::MediaInternalsUMAHandler() {
 254   registrar_.Add(this, NOTIFICATION_RENDERER_PROCESS_TERMINATED,
 255                  NotificationService::AllBrowserContextsAndSources());
 256 }
 257
 258 void MediaInternals::MediaInternalsUMAHandler::Observe(
 259     int type,
 260     const NotificationSource& source,
 261     const NotificationDetails& details) {
 262   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 263   DCHECK_EQ(type, NOTIFICATION_RENDERER_PROCESS_TERMINATED);
 264   RenderProcessHost* process = Source<RenderProcessHost>(source).ptr();
 265
 266   // Post the task to the IO thread to avoid race in updating renderer_info_ map
 267   // by both SavePlayerState & LogAndClearPlayersInRenderer from different
 268   // threads.
 269   // Using base::Unretained() on MediaInternalsUMAHandler is safe since
 270   // it is owned by MediaInternals and share the same lifetime
 271   BrowserThread::PostTask(
 272       BrowserThread::IO, FROM_HERE,
 273       base::Bind(&MediaInternalsUMAHandler::LogAndClearPlayersInRenderer,
 274                  base::Unretained(this), process->GetID()));
 275 }
 276
 277 void MediaInternals::MediaInternalsUMAHandler::SavePlayerState(
 278     const media::MediaLogEvent& event,
 279     int render_process_id) {
 280   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 281   PlayerInfoMap& player_info = renderer_info_[render_process_id];
 282   switch (event.type) {
 283     case media::MediaLogEvent::PIPELINE_ERROR: {
 284       int status;
 285       event.params.GetInteger("pipeline_error", &status);
 286       player_info[event.id].last_pipeline_status =
 287           static_cast<media::PipelineStatus>(status);
 288       break;
 289     }
 290     case media::MediaLogEvent::PROPERTY_CHANGE:
 291       if (event.params.HasKey("found_audio_stream")) {
 292         event.params.GetBoolean("found_audio_stream",
 293                                 &player_info[event.id].has_audio);
 294       }
 295       if (event.params.HasKey("found_video_stream")) {
 296         event.params.GetBoolean("found_video_stream",
 297                                 &player_info[event.id].has_video);
 298       }
 299       if (event.params.HasKey("audio_codec_name")) {
 300         event.params.GetString("audio_codec_name",
 301                                &player_info[event.id].audio_codec_name);
 302       }
 303       if (event.params.HasKey("video_codec_name")) {
 304         event.params.GetString("video_codec_name",
 305                                &player_info[event.id].video_codec_name);
 306       }
 307       if (event.params.HasKey("video_decoder")) {
 308         std::string previous_video_decoder(player_info[event.id].video_decoder);
 309         event.params.GetString("video_decoder",
 310                                &player_info[event.id].video_decoder);
 311         if (!previous_video_decoder.empty() &&
 312             previous_video_decoder != player_info[event.id].video_decoder) {
 313           player_info[event.id].video_decoder_changed = true;
 314         }
 315       }
 316       if (event.params.HasKey("video_dds")) {
 317         event.params.GetBoolean("video_dds", &player_info[event.id].video_dds);
 318       }
 319       break;
 320     default:
 321       break;
 322   }
 323   return;
 324 }
 325
 326 std::string MediaInternals::MediaInternalsUMAHandler::GetUMANameForAVStream(
 327     const PipelineInfo& player_info) {
 328   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 329   static const char kPipelineUmaPrefix[] = "Media.PipelineStatus.AudioVideo.";
 330   std::string uma_name = kPipelineUmaPrefix;
 331   if (player_info.video_codec_name == "vp8") {
 332     uma_name += "VP8.";
 333   } else if (player_info.video_codec_name == "vp9") {
 334     uma_name += "VP9.";
 335   } else if (player_info.video_codec_name == "h264") {
 336     uma_name += "H264.";
 337   } else {
 338     return uma_name + "Other";
 339   }
 340
 341   if (player_info.video_decoder ==
 342       media::DecryptingVideoDecoder::kDecoderName) {
 343     return uma_name + "DVD";
 344   }
 345
 346   if (player_info.video_dds) {
 347     uma_name += "DDS.";
 348   }
 349
 350   if (player_info.video_decoder == media::GpuVideoDecoder::kDecoderName) {
 351     uma_name += "HW";
 352   } else {
 353     uma_name += "SW";
 354   }
 355   return uma_name;
 356 }
 357
 358 void MediaInternals::MediaInternalsUMAHandler::ReportUMAForPipelineStatus(
 359     const PipelineInfo& player_info) {
 360   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 361   if (player_info.has_video && player_info.has_audio) {
 362     base::LinearHistogram::FactoryGet(
 363         GetUMANameForAVStream(player_info), 1, media::PIPELINE_STATUS_MAX,
 364         media::PIPELINE_STATUS_MAX + 1,
 365         base::HistogramBase::kUmaTargetedHistogramFlag)
 366         ->Add(player_info.last_pipeline_status);
 367   } else if (player_info.has_audio) {
 368     UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.AudioOnly",
 369                               player_info.last_pipeline_status,
 370                               media::PIPELINE_STATUS_MAX + 1);
 371   } else if (player_info.has_video) {
 372     UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.VideoOnly",
 373                               player_info.last_pipeline_status,
 374                               media::PIPELINE_STATUS_MAX + 1);
 375   } else {
 376     UMA_HISTOGRAM_ENUMERATION("Media.PipelineStatus.Unsupported",
 377                               player_info.last_pipeline_status,
 378                               media::PIPELINE_STATUS_MAX + 1);
 379   }
 380   // Report whether video decoder fallback happened, but only if a video decoder
 381   // was reported.
 382   if (!player_info.video_decoder.empty()) {
 383     UMA_HISTOGRAM_BOOLEAN("Media.VideoDecoderFallback",
 384                           player_info.video_decoder_changed);
 385   }
 386 }
 387
 388 void MediaInternals::MediaInternalsUMAHandler::LogAndClearPlayersInRenderer(
 389     int render_process_id) {
 390   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 391   auto players_it = renderer_info_.find(render_process_id);
 392   if (players_it == renderer_info_.end())
 393     return;
 394   auto it = players_it->second.begin();
 395   while (it != players_it->second.end()) {
 396     ReportUMAForPipelineStatus(it->second);
 397     players_it->second.erase(it++);
 398   }
 399 }
 400
 401 MediaInternals* MediaInternals::GetInstance() {
 402   return g_media_internals.Pointer();
 403 }
 404
 405 MediaInternals::MediaInternals()
 406     : owner_ids_(), uma_handler_(new MediaInternalsUMAHandler()) {
 407 }
 408
 409 MediaInternals::~MediaInternals() {}
 410
 411 void MediaInternals::OnMediaEvents(
 412     int render_process_id, const std::vector<media::MediaLogEvent>& events) {
 413   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 414   // Notify observers that |event| has occurred.
 415   for (auto event = events.begin(); event != events.end(); ++event) {
 416     base::DictionaryValue dict;
 417     dict.SetInteger("renderer", render_process_id);
 418     dict.SetInteger("player", event->id);
 419     dict.SetString("type", media::MediaLog::EventTypeToString(event->type));
 420
 421     // TODO(dalecurtis): This is technically not correct.  TimeTicks "can't" be
 422     // converted to to a human readable time format.  See base/time/time.h.
 423     const double ticks = event->time.ToInternalValue();
 424     const double ticks_millis = ticks / base::Time::kMicrosecondsPerMillisecond;
 425     dict.SetDouble("ticksMillis", ticks_millis);
 426
 427     // Convert PipelineStatus to human readable string
 428     if (event->type == media::MediaLogEvent::PIPELINE_ERROR) {
 429       int status;
 430       event->params.GetInteger("pipeline_error", &status);
 431       media::PipelineStatus error = static_cast<media::PipelineStatus>(status);
 432       dict.SetString("params.pipeline_error",
 433                      media::MediaLog::PipelineStatusToString(error));
 434     } else {
 435       dict.Set("params", event->params.DeepCopy());
 436     }
 437
 438     SendUpdate(SerializeUpdate("media.onMediaEvent", &dict));
 439     uma_handler_->SavePlayerState(*event, render_process_id);
 440   }
 441 }
 442
 443 void MediaInternals::AddUpdateCallback(const UpdateCallback& callback) {
 444   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 445   update_callbacks_.push_back(callback);
 446 }
 447
 448 void MediaInternals::RemoveUpdateCallback(const UpdateCallback& callback) {
 449   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 450   for (size_t i = 0; i < update_callbacks_.size(); ++i) {
 451     if (update_callbacks_[i].Equals(callback)) {
 452       update_callbacks_.erase(update_callbacks_.begin() + i);
 453       return;
 454     }
 455   }
 456   NOTREACHED();
 457 }
 458
 459 void MediaInternals::SendAudioStreamData() {
 460   base::string16 audio_stream_update;
 461   {
 462     base::AutoLock auto_lock(lock_);
 463     audio_stream_update = SerializeUpdate(
 464         "media.onReceiveAudioStreamData", &audio_streams_cached_data_);
 465   }
 466   SendUpdate(audio_stream_update);
 467 }
 468
 469 void MediaInternals::SendVideoCaptureDeviceCapabilities() {
 470   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 471   SendUpdate(SerializeUpdate("media.onReceiveVideoCaptureCapabilities",
 472                              &video_capture_capabilities_cached_data_));
 473 }
 474
 475 void MediaInternals::UpdateVideoCaptureDeviceCapabilities(
 476     const media::VideoCaptureDeviceInfos& video_capture_device_infos) {
 477   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 478   video_capture_capabilities_cached_data_.Clear();
 479
 480   for (const auto& video_capture_device_info : video_capture_device_infos) {
 481     base::ListValue* format_list = new base::ListValue();
 482     for (const auto& format : video_capture_device_info.supported_formats)
 483       format_list->AppendString(format.ToString());
 484
 485     base::DictionaryValue* device_dict = new base::DictionaryValue();
 486     device_dict->SetString("id", video_capture_device_info.name.id());
 487     device_dict->SetString(
 488         "name", video_capture_device_info.name.GetNameAndModel());
 489     device_dict->Set("formats", format_list);
 490 #if defined(OS_WIN) || defined(OS_MACOSX)
 491     device_dict->SetString(
 492         "captureApi",
 493         video_capture_device_info.name.GetCaptureApiTypeString());
 494 #endif
 495     video_capture_capabilities_cached_data_.Append(device_dict);
 496   }
 497
 498   if (update_callbacks_.size() > 0)
 499     SendVideoCaptureDeviceCapabilities();
 500 }
 501
 502 scoped_ptr<media::AudioLog> MediaInternals::CreateAudioLog(
 503     AudioComponent component) {
 504   base::AutoLock auto_lock(lock_);
 505   return scoped_ptr<media::AudioLog>(new AudioLogImpl(
 506       owner_ids_[component]++, component, this));
 507 }
 508
 509 void MediaInternals::SendUpdate(const base::string16& update) {
 510   // SendUpdate() may be called from any thread, but must run on the IO thread.
 511   // TODO(dalecurtis): This is pretty silly since the update callbacks simply
 512   // forward the calls to the UI thread.  We should avoid the extra hop.
 513   if (!BrowserThread::CurrentlyOn(BrowserThread::IO)) {
 514     BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
 515         &MediaInternals::SendUpdate, base::Unretained(this), update));
 516     return;
 517   }
 518
 519   for (size_t i = 0; i < update_callbacks_.size(); i++)
 520     update_callbacks_[i].Run(update);
 521 }
 522
 523 void MediaInternals::SendUpdateAndCacheAudioStreamKey(
 524     const std::string& cache_key,
 525     const std::string& function,
 526     const base::DictionaryValue* value) {
 527   SendUpdate(SerializeUpdate(function, value));
 528
 529   base::AutoLock auto_lock(lock_);
 530   if (!audio_streams_cached_data_.HasKey(cache_key)) {
 531     audio_streams_cached_data_.Set(cache_key, value->DeepCopy());
 532     return;
 533   }
 534
 535   base::DictionaryValue* existing_dict = NULL;
 536   CHECK(audio_streams_cached_data_.GetDictionary(cache_key, &existing_dict));
 537   existing_dict->MergeDictionary(value);
 538 }
 539
 540 void MediaInternals::SendUpdateAndPurgeAudioStreamCache(
 541     const std::string& cache_key,
 542     const std::string& function,
 543     const base::DictionaryValue* value) {
 544   SendUpdate(SerializeUpdate(function, value));
 545
 546   base::AutoLock auto_lock(lock_);
 547   scoped_ptr<base::Value> out_value;
 548   CHECK(audio_streams_cached_data_.Remove(cache_key, &out_value));
 549 }
 550
 551 }  // namespace content