content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/debug/trace_event.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "media/base/video_util.h"
  16
  17 namespace content {
  18
  19 namespace {
  20
  21 // Amount of frame intervals to wait before considering the source as muted, for
  22 // the first frame and under normal conditions, respectively. First frame might
  23 // take longer to arrive due to source startup.
  24 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  25 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  26
  27 // Min delta time between two frames allowed without being dropped if a max
  28 // frame rate is specified.
  29 const int kMinTimeInMsBetweenFrames = 5;
  30
  31 // Empty method used for keeping a reference to the original media::VideoFrame
  32 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  33 // The reference to |frame| is kept in the closure that calls this method.
  34 void ReleaseOriginalFrame(
  35     const scoped_refptr<media::VideoFrame>& frame) {
  36 }
  37
  38 void ResetCallbackOnMainRenderThread(
  39     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  40   // |callback| will be deleted when this exits.
  41 }
  42
  43 }  // anonymous namespace
  44
  45 // VideoFrameResolutionAdapter is created on and lives on
  46 // on the IO-thread. It does the resolution adaptation and delivers frames to
  47 // all registered tracks on the IO-thread.
  48 // All method calls must be on the IO-thread.
  49 class VideoTrackAdapter::VideoFrameResolutionAdapter
  50     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  51  public:
  52   VideoFrameResolutionAdapter(
  53       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  54       const gfx::Size& max_size,
  55       double min_aspect_ratio,
  56       double max_aspect_ratio,
  57       double max_frame_rate);
  58
  59   // Add |callback| to receive video frames on the IO-thread.
  60   // |callback| will however be released on the main render thread.
  61   void AddCallback(const MediaStreamVideoTrack* track,
  62                    const VideoCaptureDeliverFrameCB& callback);
  63
  64   // Removes |callback| associated with |track| from receiving video frames if
  65   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  66   // has not been added. The |callback| is released on the main render thread.
  67   void RemoveCallback(const MediaStreamVideoTrack* track);
  68
  69   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  70                     const media::VideoCaptureFormat& format,
  71                     const base::TimeTicks& estimated_capture_time);
  72
  73   // Returns true if all arguments match with the output of this adapter.
  74   bool ConstraintsMatch(const gfx::Size& max_size,
  75                         double min_aspect_ratio,
  76                         double max_aspect_ratio,
  77                         double max_frame_rate) const;
  78
  79   bool IsEmpty() const;
  80
  81  private:
  82   virtual ~VideoFrameResolutionAdapter();
  83   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  84
  85   virtual void DoDeliverFrame(
  86       const scoped_refptr<media::VideoFrame>& frame,
  87       const media::VideoCaptureFormat& format,
  88       const base::TimeTicks& estimated_capture_time);
  89
  90   // Returns |true| if the input frame rate is higher that the requested max
  91   // frame rate and |frame| should be dropped.
  92   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame);
  93
  94   // Bound to the IO-thread.
  95   base::ThreadChecker io_thread_checker_;
  96
  97   // The task runner where we will release VideoCaptureDeliverFrameCB
  98   // registered in AddCallback.
  99   scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 100
 101   gfx::Size max_frame_size_;
 102   double min_aspect_ratio_;
 103   double max_aspect_ratio_;
 104
 105   double frame_rate_;
 106   base::TimeDelta last_time_stamp_;
 107   double max_frame_rate_;
 108   double keep_frame_counter_;
 109
 110   typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
 111       VideoIdCallbackPair;
 112   std::vector<VideoIdCallbackPair> callbacks_;
 113
 114   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 115 };
 116
 117 VideoTrackAdapter::
 118 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 119     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 120     const gfx::Size& max_size,
 121     double min_aspect_ratio,
 122     double max_aspect_ratio,
 123     double max_frame_rate)
 124     : renderer_task_runner_(render_message_loop),
 125       max_frame_size_(max_size),
 126       min_aspect_ratio_(min_aspect_ratio),
 127       max_aspect_ratio_(max_aspect_ratio),
 128       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 129       max_frame_rate_(max_frame_rate),
 130       keep_frame_counter_(0.0f) {
 131   DCHECK(renderer_task_runner_.get());
 132   DCHECK(io_thread_checker_.CalledOnValidThread());
 133   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 134   CHECK_NE(0, max_aspect_ratio_);
 135   DVLOG(3) << "VideoFrameResolutionAdapter("
 136           << "{ max_width =" << max_frame_size_.width() << "}, "
 137           << "{ max_height =" << max_frame_size_.height() << "}, "
 138           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 139           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 140           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 141 }
 142
 143 VideoTrackAdapter::
 144 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 145   DCHECK(io_thread_checker_.CalledOnValidThread());
 146   DCHECK(callbacks_.empty());
 147 }
 148
 149 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 150     const scoped_refptr<media::VideoFrame>& frame,
 151     const media::VideoCaptureFormat& format,
 152     const base::TimeTicks& estimated_capture_time) {
 153   DCHECK(io_thread_checker_.CalledOnValidThread());
 154
 155   if (MaybeDropFrame(frame))
 156     return;
 157
 158   // TODO(perkj): Allow cropping / scaling of textures once
 159   // http://crbug/362521 is fixed.
 160   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
 161     DoDeliverFrame(frame, format, estimated_capture_time);
 162     return;
 163   }
 164   scoped_refptr<media::VideoFrame> video_frame(frame);
 165   double input_ratio =
 166       static_cast<double>(frame->natural_size().width()) /
 167       frame->natural_size().height();
 168
 169   // If |frame| has larger width or height than requested, or the aspect ratio
 170   // does not match the requested, we want to create a wrapped version of this
 171   // frame with a size that fulfills the constraints.
 172   if (frame->natural_size().width() > max_frame_size_.width() ||
 173       frame->natural_size().height() > max_frame_size_.height() ||
 174       input_ratio > max_aspect_ratio_ ||
 175       input_ratio < min_aspect_ratio_) {
 176     int desired_width = std::min(max_frame_size_.width(),
 177                                  frame->natural_size().width());
 178     int desired_height = std::min(max_frame_size_.height(),
 179                                   frame->natural_size().height());
 180
 181     double resulting_ratio =
 182         static_cast<double>(desired_width) / desired_height;
 183     double requested_ratio = resulting_ratio;
 184
 185     if (requested_ratio > max_aspect_ratio_)
 186       requested_ratio = max_aspect_ratio_;
 187     else if (requested_ratio < min_aspect_ratio_)
 188       requested_ratio = min_aspect_ratio_;
 189
 190     if (resulting_ratio < requested_ratio) {
 191       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 192                                         requested_ratio);
 193       // Make sure we scale to an even height to avoid rounding errors
 194       desired_height = (desired_height + 1) & ~1;
 195     } else if (resulting_ratio > requested_ratio) {
 196       desired_width = static_cast<int>((desired_width * requested_ratio) /
 197                                        resulting_ratio);
 198       // Make sure we scale to an even width to avoid rounding errors.
 199       desired_width = (desired_width + 1) & ~1;
 200     }
 201
 202     gfx::Size desired_size(desired_width, desired_height);
 203
 204     // Get the largest centered rectangle with the same aspect ratio of
 205     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 206     // This will be the rect we need to crop the original frame to.
 207     // From this rect, the original frame can be scaled down to |desired_size|.
 208     gfx::Rect region_in_frame =
 209         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 210
 211     video_frame = media::VideoFrame::WrapVideoFrame(
 212         frame,
 213         region_in_frame,
 214         desired_size,
 215         base::Bind(&ReleaseOriginalFrame, frame));
 216
 217     DVLOG(3) << "desired size  " << desired_size.ToString()
 218              << " output natural size "
 219              << video_frame->natural_size().ToString()
 220              << " output visible rect  "
 221              << video_frame->visible_rect().ToString();
 222   }
 223   DoDeliverFrame(video_frame, format, estimated_capture_time);
 224 }
 225
 226 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 227     const scoped_refptr<media::VideoFrame>& frame) {
 228   if (max_frame_rate_ == 0.0f)
 229     return false;
 230
 231   base::TimeDelta delta = frame->timestamp() - last_time_stamp_;
 232   if (delta.InMilliseconds() < kMinTimeInMsBetweenFrames) {
 233     // We have seen video frames being delivered from camera devices back to
 234     // back. The simple AR filter for frame rate calculation is too short to
 235     // handle that. http://crbug/394315
 236     // TODO(perkj): Can we come up with a way to fix the times stamps and the
 237     // timing when frames are delivered so all frames can be used?
 238     // The time stamps are generated by Chrome and not the actual device.
 239     // Most likely the back to back problem is caused by software and not the
 240     // actual camera.
 241     DVLOG(3) << "Drop frame since delta time since previous frame is "
 242              << delta.InMilliseconds() << "ms.";
 243     return true;
 244   }
 245   last_time_stamp_ = frame->timestamp();
 246   if (delta == last_time_stamp_) // First received frame.
 247     return false;
 248   // Calculate the frame rate using a simple AR filter.
 249   // Use a simple filter with 0.1 weight of the current sample.
 250   frame_rate_ = 100 / delta.InMillisecondsF() + 0.9 * frame_rate_;
 251
 252   // Prefer to not drop frames.
 253   if (max_frame_rate_ + 0.5f > frame_rate_)
 254     return false;  // Keep this frame.
 255
 256   // The input frame rate is higher than requested.
 257   // Decide if we should keep this frame or drop it.
 258   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 259   if (keep_frame_counter_ >= 1) {
 260     keep_frame_counter_ -= 1;
 261     // Keep the frame.
 262     return false;
 263   }
 264   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 265   return true;
 266 }
 267
 268 void VideoTrackAdapter::
 269 VideoFrameResolutionAdapter::DoDeliverFrame(
 270     const scoped_refptr<media::VideoFrame>& frame,
 271     const media::VideoCaptureFormat& format,
 272     const base::TimeTicks& estimated_capture_time) {
 273   DCHECK(io_thread_checker_.CalledOnValidThread());
 274   for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
 275        it != callbacks_.end(); ++it) {
 276     it->second.Run(frame, format, estimated_capture_time);
 277   }
 278 }
 279
 280 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 281     const MediaStreamVideoTrack* track,
 282     const VideoCaptureDeliverFrameCB& callback) {
 283   DCHECK(io_thread_checker_.CalledOnValidThread());
 284   callbacks_.push_back(std::make_pair(track, callback));
 285 }
 286
 287 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 288     const MediaStreamVideoTrack* track) {
 289   DCHECK(io_thread_checker_.CalledOnValidThread());
 290   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 291   for (; it != callbacks_.end(); ++it) {
 292     if (it->first == track) {
 293       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 294       // render thread since it was added on the main render thread in
 295       // VideoTrackAdapter::AddTrack.
 296       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 297           new VideoCaptureDeliverFrameCB(it->second));
 298       callbacks_.erase(it);
 299       renderer_task_runner_->PostTask(
 300           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 301                                 base::Passed(&callback)));
 302
 303       return;
 304     }
 305   }
 306 }
 307
 308 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 309     const gfx::Size& max_size,
 310     double min_aspect_ratio,
 311     double max_aspect_ratio,
 312     double max_frame_rate) const {
 313   DCHECK(io_thread_checker_.CalledOnValidThread());
 314   return max_frame_size_ == max_size &&
 315       min_aspect_ratio_ == min_aspect_ratio &&
 316       max_aspect_ratio_ == max_aspect_ratio &&
 317       max_frame_rate_ == max_frame_rate;
 318 }
 319
 320 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 321   DCHECK(io_thread_checker_.CalledOnValidThread());
 322   return callbacks_.empty();
 323 }
 324
 325 VideoTrackAdapter::VideoTrackAdapter(
 326     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
 327     : io_message_loop_(io_message_loop),
 328       renderer_task_runner_(base::MessageLoopProxy::current()),
 329       frame_counter_(0),
 330       source_frame_rate_(0.0f) {
 331   DCHECK(io_message_loop_.get());
 332 }
 333
 334 VideoTrackAdapter::~VideoTrackAdapter() {
 335   DCHECK(adapters_.empty());
 336   UMA_HISTOGRAM_BOOLEAN("Media.VideoTrackAdapter.FramesReceived",
 337                         frame_counter_ > 0);
 338 }
 339
 340 void VideoTrackAdapter::AddTrack(
 341     const MediaStreamVideoTrack* track,
 342     VideoCaptureDeliverFrameCB frame_callback,
 343     int max_width,
 344     int max_height,
 345     double min_aspect_ratio,
 346     double max_aspect_ratio,
 347     double max_frame_rate,
 348     double source_frame_rate,
 349     const OnMutedCallback& on_muted_state_callback) {
 350   DCHECK(thread_checker_.CalledOnValidThread());
 351   // Track monitoring should be scheduled before AddTrackOnIO() so it can find
 352   // |adapters_| empty.
 353   io_message_loop_->PostTask(
 354       FROM_HERE,
 355       base::Bind(&VideoTrackAdapter::StartTrackMonitoringOnIO,
 356                  this, on_muted_state_callback, source_frame_rate));
 357   io_message_loop_->PostTask(
 358       FROM_HERE,
 359       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
 360                  this, track, frame_callback, gfx::Size(max_width, max_height),
 361                  min_aspect_ratio, max_aspect_ratio, max_frame_rate));
 362 }
 363
 364 void VideoTrackAdapter::AddTrackOnIO(
 365     const MediaStreamVideoTrack* track,
 366     VideoCaptureDeliverFrameCB frame_callback,
 367     const gfx::Size& max_frame_size,
 368     double min_aspect_ratio,
 369     double max_aspect_ratio,
 370     double max_frame_rate) {
 371   DCHECK(io_message_loop_->BelongsToCurrentThread());
 372   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 373   for (FrameAdapters::const_iterator it = adapters_.begin();
 374        it != adapters_.end(); ++it) {
 375     if ((*it)->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 376                                 max_aspect_ratio, max_frame_rate)) {
 377       adapter = it->get();
 378       break;
 379     }
 380   }
 381   if (!adapter.get()) {
 382     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 383                                               max_frame_size,
 384                                               min_aspect_ratio,
 385                                               max_aspect_ratio,
 386                                               max_frame_rate);
 387     adapters_.push_back(adapter);
 388   }
 389
 390   adapter->AddCallback(track, frame_callback);
 391 }
 392
 393 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 394   DCHECK(thread_checker_.CalledOnValidThread());
 395   io_message_loop_->PostTask(
 396       FROM_HERE,
 397       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 398 }
 399
 400 void VideoTrackAdapter::StartTrackMonitoringOnIO(
 401     const OnMutedCallback& on_muted_state_callback,
 402     double source_frame_rate) {
 403   DCHECK(io_message_loop_->BelongsToCurrentThread());
 404   // Only trigger monitoring for the first Track.
 405   if (!adapters_.empty())
 406     return;
 407   // If the source does not know the frame rate, set one by default.
 408   if (source_frame_rate == 0.0f)
 409     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 410   source_frame_rate_ = source_frame_rate;
 411   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 412       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 413   io_message_loop_->PostDelayedTask(FROM_HERE,
 414        base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 415                   on_muted_state_callback, frame_counter_),
 416        base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 417                                      source_frame_rate_));
 418 }
 419
 420 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 421   DCHECK(io_message_loop_->BelongsToCurrentThread());
 422   for (FrameAdapters::iterator it = adapters_.begin();
 423        it != adapters_.end(); ++it) {
 424     (*it)->RemoveCallback(track);
 425     if ((*it)->IsEmpty()) {
 426       adapters_.erase(it);
 427       break;
 428     }
 429   }
 430 }
 431
 432 void VideoTrackAdapter::DeliverFrameOnIO(
 433     const scoped_refptr<media::VideoFrame>& frame,
 434     const media::VideoCaptureFormat& format,
 435     const base::TimeTicks& estimated_capture_time) {
 436   DCHECK(io_message_loop_->BelongsToCurrentThread());
 437   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 438   ++frame_counter_;
 439   for (FrameAdapters::iterator it = adapters_.begin();
 440        it != adapters_.end(); ++it) {
 441     (*it)->DeliverFrame(frame, format, estimated_capture_time);
 442   }
 443 }
 444
 445 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 446     const OnMutedCallback& set_muted_state_callback,
 447     uint64 old_frame_counter_snapshot) {
 448   DCHECK(io_message_loop_->BelongsToCurrentThread());
 449   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 450       << "No frames have passed, setting source as Muted.";
 451   set_muted_state_callback.Run(old_frame_counter_snapshot == frame_counter_);
 452
 453   // Rearm the monitoring while there are active Tracks, i.e. as long as the
 454   // owner MediaStreamSource is active.
 455   io_message_loop_->PostDelayedTask(FROM_HERE,
 456       base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 457           set_muted_state_callback, frame_counter_),
 458       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 459                                     source_frame_rate_));
 460 }
 461
 462 }  // namespace content