content/renderer/media/webrtc/media_stream_remote_video_source.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc/media_stream_remote_video_source.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/debug/trace_event.h"
  10 #include "base/location.h"
  11 #include "base/message_loop/message_loop_proxy.h"
  12 #include "base/threading/thread_checker.h"
  13 #include "content/renderer/media/native_handle_impl.h"
  14 #include "media/base/bind_to_current_loop.h"
  15 #include "media/base/video_frame.h"
  16 #include "media/base/video_frame_pool.h"
  17 #include "media/base/video_util.h"
  18 #include "third_party/libjingle/source/talk/media/base/videoframe.h"
  19
  20 namespace content {
  21
  22 // Internal class used for receiving frames from the webrtc track on a
  23 // libjingle thread and forward it to the IO-thread.
  24 class MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate
  25     : public base::RefCountedThreadSafe<RemoteVideoSourceDelegate>,
  26       public webrtc::VideoRendererInterface {
  27  public:
  28   RemoteVideoSourceDelegate(
  29       const scoped_refptr<base::MessageLoopProxy>& io_message_loop,
  30       const VideoCaptureDeliverFrameCB& new_frame_callback);
  31
  32  protected:
  33   friend class base::RefCountedThreadSafe<RemoteVideoSourceDelegate>;
  34   ~RemoteVideoSourceDelegate() override;
  35
  36   // Implements webrtc::VideoRendererInterface used for receiving video frames
  37   // from the PeerConnection video track. May be called on a libjingle internal
  38   // thread.
  39   void SetSize(int width, int height) override;
  40   void RenderFrame(const cricket::VideoFrame* frame) override;
  41
  42   void DoRenderFrameOnIOThread(scoped_refptr<media::VideoFrame> video_frame,
  43                                const media::VideoCaptureFormat& format);
  44  private:
  45   // Bound to the render thread.
  46   base::ThreadChecker thread_checker_;
  47
  48   scoped_refptr<base::MessageLoopProxy> io_message_loop_;
  49   // |frame_pool_| is only accessed on whatever
  50   // thread webrtc::VideoRendererInterface::RenderFrame is called on.
  51   media::VideoFramePool frame_pool_;
  52
  53   // |frame_callback_| is accessed on the IO thread.
  54   VideoCaptureDeliverFrameCB frame_callback_;
  55 };
  56
  57 MediaStreamRemoteVideoSource::
  58 RemoteVideoSourceDelegate::RemoteVideoSourceDelegate(
  59     const scoped_refptr<base::MessageLoopProxy>& io_message_loop,
  60     const VideoCaptureDeliverFrameCB& new_frame_callback)
  61     : io_message_loop_(io_message_loop),
  62       frame_callback_(new_frame_callback) {
  63 }
  64
  65 MediaStreamRemoteVideoSource::
  66 RemoteVideoSourceDelegate::~RemoteVideoSourceDelegate() {
  67 }
  68
  69 void MediaStreamRemoteVideoSource::
  70 RemoteVideoSourceDelegate::SetSize(int width, int height) {
  71 }
  72
  73 void MediaStreamRemoteVideoSource::
  74 RemoteVideoSourceDelegate::RenderFrame(
  75     const cricket::VideoFrame* frame) {
  76   TRACE_EVENT0("webrtc", "RemoteVideoSourceDelegate::RenderFrame");
  77   base::TimeDelta timestamp = base::TimeDelta::FromMicroseconds(
  78       frame->GetElapsedTime() / rtc::kNumNanosecsPerMicrosec);
  79
  80   scoped_refptr<media::VideoFrame> video_frame;
  81   if (frame->GetNativeHandle() != NULL) {
  82     NativeHandleImpl* handle =
  83         static_cast<NativeHandleImpl*>(frame->GetNativeHandle());
  84     video_frame = static_cast<media::VideoFrame*>(handle->GetHandle());
  85     video_frame->set_timestamp(timestamp);
  86   } else {
  87     gfx::Size size(frame->GetWidth(), frame->GetHeight());
  88     video_frame = frame_pool_.CreateFrame(
  89         media::VideoFrame::YV12, size, gfx::Rect(size), size, timestamp);
  90
  91     // Non-square pixels are unsupported.
  92     DCHECK_EQ(frame->GetPixelWidth(), 1u);
  93     DCHECK_EQ(frame->GetPixelHeight(), 1u);
  94
  95     int y_rows = frame->GetHeight();
  96     int uv_rows = frame->GetChromaHeight();
  97     CopyYPlane(
  98         frame->GetYPlane(), frame->GetYPitch(), y_rows, video_frame.get());
  99     CopyUPlane(
 100         frame->GetUPlane(), frame->GetUPitch(), uv_rows, video_frame.get());
 101     CopyVPlane(
 102         frame->GetVPlane(), frame->GetVPitch(), uv_rows, video_frame.get());
 103   }
 104
 105   media::VideoPixelFormat pixel_format =
 106       (video_frame->format() == media::VideoFrame::YV12) ?
 107           media::PIXEL_FORMAT_YV12 : media::PIXEL_FORMAT_TEXTURE;
 108
 109   media::VideoCaptureFormat format(
 110       gfx::Size(video_frame->natural_size().width(),
 111                 video_frame->natural_size().height()),
 112                 MediaStreamVideoSource::kUnknownFrameRate,
 113                 pixel_format);
 114
 115   io_message_loop_->PostTask(
 116       FROM_HERE,
 117       base::Bind(&RemoteVideoSourceDelegate::DoRenderFrameOnIOThread,
 118                  this, video_frame, format));
 119 }
 120
 121 void MediaStreamRemoteVideoSource::
 122 RemoteVideoSourceDelegate::DoRenderFrameOnIOThread(
 123     scoped_refptr<media::VideoFrame> video_frame,
 124     const media::VideoCaptureFormat& format) {
 125   DCHECK(io_message_loop_->BelongsToCurrentThread());
 126   TRACE_EVENT0("webrtc", "RemoteVideoSourceDelegate::DoRenderFrameOnIOThread");
 127   // TODO(hclam): Give the estimated capture time.
 128   frame_callback_.Run(video_frame, format, base::TimeTicks());
 129 }
 130
 131 MediaStreamRemoteVideoSource::Observer::Observer(
 132     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 133     webrtc::VideoTrackInterface* track)
 134     : main_thread_(main_thread),
 135 #if DCHECK_IS_ON
 136       source_set_(false),
 137 #endif
 138       track_(track),
 139       state_(track->state()) {
 140   track->RegisterObserver(this);
 141 }
 142
 143 const scoped_refptr<webrtc::VideoTrackInterface>&
 144 MediaStreamRemoteVideoSource::Observer::track() {
 145   DCHECK(main_thread_->BelongsToCurrentThread());
 146   return track_;
 147 }
 148
 149 webrtc::MediaStreamTrackInterface::TrackState
 150 MediaStreamRemoteVideoSource::Observer::state() const {
 151   DCHECK(main_thread_->BelongsToCurrentThread());
 152   return state_;
 153 }
 154
 155 MediaStreamRemoteVideoSource::Observer::~Observer() {
 156   DCHECK(main_thread_->BelongsToCurrentThread());
 157   track_->UnregisterObserver(this);
 158 }
 159
 160 void MediaStreamRemoteVideoSource::Observer::SetSource(
 161     const base::WeakPtr<MediaStreamRemoteVideoSource>& source) {
 162   DCHECK(main_thread_->BelongsToCurrentThread());
 163   DCHECK(!source_);
 164   source_ = source;
 165 #if DCHECK_IS_ON
 166   // |source_set_| means that there is a MediaStreamRemoteVideoSource that
 167   // should handle all state changes. Since an Observer is always created when
 168   // a remote MediaStream changes in RemoteMediaStreamImpl::Observer::OnChanged,
 169   // it can happen that an Observer is created but SetSource is never called.
 170   source_set_ = true;
 171 #endif
 172 }
 173
 174 void MediaStreamRemoteVideoSource::Observer::OnChanged() {
 175   webrtc::MediaStreamTrackInterface::TrackState state = track_->state();
 176   main_thread_->PostTask(FROM_HERE,
 177       base::Bind(&MediaStreamRemoteVideoSource::Observer::OnChangedImpl,
 178           this, state));
 179 }
 180
 181 void MediaStreamRemoteVideoSource::Observer::OnChangedImpl(
 182     webrtc::MediaStreamTrackInterface::TrackState state) {
 183   DCHECK(main_thread_->BelongsToCurrentThread());
 184 #if DCHECK_IS_ON
 185   DCHECK(source_ || !source_set_) << "Dropping a state change event. " << state;
 186 #endif
 187   if (source_ && state != state_)
 188     source_->OnChanged(state);
 189   state_ = state;
 190 }
 191
 192 MediaStreamRemoteVideoSource::MediaStreamRemoteVideoSource(
 193     const scoped_refptr<MediaStreamRemoteVideoSource::Observer>& observer)
 194     : observer_(observer), weak_factory_(this) {
 195   observer->SetSource(weak_factory_.GetWeakPtr());
 196 }
 197
 198 MediaStreamRemoteVideoSource::~MediaStreamRemoteVideoSource() {
 199   DCHECK(CalledOnValidThread());
 200   observer_ = nullptr;
 201 }
 202
 203 void MediaStreamRemoteVideoSource::GetCurrentSupportedFormats(
 204     int max_requested_width,
 205     int max_requested_height,
 206     double max_requested_frame_rate,
 207     const VideoCaptureDeviceFormatsCB& callback) {
 208   DCHECK(CalledOnValidThread());
 209   media::VideoCaptureFormats formats;
 210   // Since the remote end is free to change the resolution at any point in time
 211   // the supported formats are unknown.
 212   callback.Run(formats);
 213 }
 214
 215 void MediaStreamRemoteVideoSource::StartSourceImpl(
 216     const media::VideoCaptureFormat& format,
 217     const VideoCaptureDeliverFrameCB& frame_callback) {
 218   DCHECK(CalledOnValidThread());
 219   DCHECK(!delegate_.get());
 220   delegate_ = new RemoteVideoSourceDelegate(io_message_loop(), frame_callback);
 221   observer_->track()->AddRenderer(delegate_.get());
 222   OnStartDone(MEDIA_DEVICE_OK);
 223 }
 224
 225 void MediaStreamRemoteVideoSource::StopSourceImpl() {
 226   DCHECK(CalledOnValidThread());
 227   DCHECK(state() != MediaStreamVideoSource::ENDED);
 228   observer_->track()->RemoveRenderer(delegate_.get());
 229 }
 230
 231 webrtc::VideoRendererInterface*
 232 MediaStreamRemoteVideoSource::RenderInterfaceForTest() {
 233   return delegate_.get();
 234 }
 235
 236 void MediaStreamRemoteVideoSource::OnChanged(
 237     webrtc::MediaStreamTrackInterface::TrackState state) {
 238   DCHECK(CalledOnValidThread());
 239   switch (state) {
 240     case webrtc::MediaStreamTrackInterface::kInitializing:
 241       // Ignore the kInitializing state since there is no match in
 242       // WebMediaStreamSource::ReadyState.
 243       break;
 244     case webrtc::MediaStreamTrackInterface::kLive:
 245       SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
 246       break;
 247     case webrtc::MediaStreamTrackInterface::kEnded:
 248       SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
 249       break;
 250     default:
 251       NOTREACHED();
 252       break;
 253   }
 254 }
 255
 256 }  // namespace content