1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/cast/sender/external_video_encoder.h"
8 #include "base/logging.h"
9 #include "base/memory/scoped_vector.h"
10 #include "base/memory/shared_memory.h"
11 #include "base/message_loop/message_loop.h"
12 #include "base/metrics/histogram.h"
13 #include "media/base/video_frame.h"
14 #include "media/base/video_util.h"
15 #include "media/cast/cast_defines.h"
16 #include "media/cast/logging/logging_defines.h"
17 #include "media/cast/net/cast_transport_config.h"
18 #include "media/video/video_encode_accelerator.h"
22 class LocalVideoEncodeAcceleratorClient
;
27 static const size_t kOutputBufferCount
= 3;
29 void LogFrameEncodedEvent(
30 const scoped_refptr
<media::cast::CastEnvironment
>& cast_environment
,
31 base::TimeTicks event_time
,
32 media::cast::RtpTimestamp rtp_timestamp
,
34 cast_environment
->Logging()->InsertFrameEvent(
35 event_time
, media::cast::FRAME_ENCODED
, media::cast::VIDEO_EVENT
,
36 rtp_timestamp
, frame_id
);
43 // Container for the associated data of a video frame being processed.
44 struct EncodedFrameReturnData
{
45 EncodedFrameReturnData(base::TimeTicks c_time
,
46 VideoEncoder::FrameEncodedCallback callback
) {
47 capture_time
= c_time
;
48 frame_encoded_callback
= callback
;
50 base::TimeTicks capture_time
;
51 VideoEncoder::FrameEncodedCallback frame_encoded_callback
;
54 // The ExternalVideoEncoder class can be deleted directly by cast, while
55 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
56 // down the VideoEncodeAccelerator.
57 class LocalVideoEncodeAcceleratorClient
58 : public VideoEncodeAccelerator::Client
,
59 public base::RefCountedThreadSafe
<LocalVideoEncodeAcceleratorClient
> {
61 // Create an instance of this class and post a task to create
62 // video_encode_accelerator_. A ref to |this| will be kept, awaiting reply
63 // via ProxyCreateVideoEncodeAccelerator, which will provide us with the
64 // encoder task runner and vea instance. We cannot be destroyed until we
65 // receive the reply, otherwise the VEA object created may leak.
66 static scoped_refptr
<LocalVideoEncodeAcceleratorClient
> Create(
67 scoped_refptr
<CastEnvironment
> cast_environment
,
68 const CreateVideoEncodeAcceleratorCallback
& create_vea_cb
,
69 const CreateVideoEncodeMemoryCallback
& create_video_encode_mem_cb
,
70 const base::WeakPtr
<ExternalVideoEncoder
>& weak_owner
) {
71 scoped_refptr
<LocalVideoEncodeAcceleratorClient
> client(
72 new LocalVideoEncodeAcceleratorClient(
73 cast_environment
, create_video_encode_mem_cb
, weak_owner
));
75 // This will keep a ref to |client|, if weak_owner is destroyed before
76 // ProxyCreateVideoEncodeAccelerator is called, we will stay alive until
77 // we can properly destroy the VEA.
78 create_vea_cb
.Run(base::Bind(
79 &LocalVideoEncodeAcceleratorClient::OnCreateVideoEncodeAcceleratorProxy
,
85 // Initialize the real HW encoder.
86 void Initialize(const VideoSenderConfig
& video_config
) {
87 DCHECK(encoder_task_runner_
.get());
88 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
90 VideoCodecProfile output_profile
= media::VIDEO_CODEC_PROFILE_UNKNOWN
;
91 switch (video_config
.codec
) {
93 output_profile
= media::VP8PROFILE_ANY
;
95 case CODEC_VIDEO_H264
:
96 output_profile
= media::H264PROFILE_MAIN
;
98 case CODEC_VIDEO_FAKE
:
99 NOTREACHED() << "Fake software video encoder cannot be external";
102 NOTREACHED() << "Video codec not specified or not supported";
105 max_frame_rate_
= video_config
.max_frame_rate
;
107 bool result
= video_encode_accelerator_
->Initialize(
108 media::VideoFrame::I420
,
109 gfx::Size(video_config
.width
, video_config
.height
),
111 video_config
.start_bitrate
,
114 UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
117 NotifyError(VideoEncodeAccelerator::kInvalidArgumentError
);
121 // Wait until shared memory is allocated to indicate that encoder is
125 // Destroy the VEA on the correct thread.
127 DCHECK(encoder_task_runner_
.get());
128 if (!video_encode_accelerator_
)
131 if (encoder_task_runner_
->RunsTasksOnCurrentThread()) {
132 video_encode_accelerator_
.reset();
134 // We do this instead of just reposting to encoder_task_runner_, because
135 // we are called from the destructor.
136 encoder_task_runner_
->PostTask(
138 base::Bind(&DestroyVideoEncodeAcceleratorOnEncoderThread
,
139 base::Passed(&video_encode_accelerator_
)));
143 void SetBitRate(uint32 bit_rate
) {
144 DCHECK(encoder_task_runner_
.get());
145 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
147 video_encode_accelerator_
->RequestEncodingParametersChange(bit_rate
,
151 void EncodeVideoFrame(
152 const scoped_refptr
<media::VideoFrame
>& video_frame
,
153 const base::TimeTicks
& capture_time
,
154 bool key_frame_requested
,
155 const VideoEncoder::FrameEncodedCallback
& frame_encoded_callback
) {
156 DCHECK(encoder_task_runner_
.get());
157 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
159 encoded_frame_data_storage_
.push_back(
160 EncodedFrameReturnData(capture_time
, frame_encoded_callback
));
162 // BitstreamBufferReady will be called once the encoder is done.
163 video_encode_accelerator_
->Encode(video_frame
, key_frame_requested
);
167 virtual void NotifyError(VideoEncodeAccelerator::Error error
) OVERRIDE
{
168 DCHECK(encoder_task_runner_
.get());
169 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
170 VLOG(1) << "ExternalVideoEncoder NotifyError: " << error
;
172 cast_environment_
->PostTask(
173 CastEnvironment::MAIN
,
175 base::Bind(&ExternalVideoEncoder::EncoderError
, weak_owner_
));
178 // Called to allocate the input and output buffers.
179 virtual void RequireBitstreamBuffers(unsigned int input_count
,
180 const gfx::Size
& input_coded_size
,
181 size_t output_buffer_size
) OVERRIDE
{
182 DCHECK(encoder_task_runner_
.get());
183 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
184 DCHECK(video_encode_accelerator_
);
186 for (size_t j
= 0; j
< kOutputBufferCount
; ++j
) {
187 create_video_encode_memory_cb_
.Run(
189 base::Bind(&LocalVideoEncodeAcceleratorClient::OnCreateSharedMemory
,
194 // Encoder has encoded a frame and it's available in one of out output
196 virtual void BitstreamBufferReady(int32 bitstream_buffer_id
,
198 bool key_frame
) OVERRIDE
{
199 DCHECK(encoder_task_runner_
.get());
200 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
201 if (bitstream_buffer_id
< 0 ||
202 bitstream_buffer_id
>= static_cast<int32
>(output_buffers_
.size())) {
204 VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
205 << bitstream_buffer_id
;
206 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError
);
209 base::SharedMemory
* output_buffer
= output_buffers_
[bitstream_buffer_id
];
210 if (payload_size
> output_buffer
->mapped_size()) {
212 VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
214 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError
);
218 key_frame_encountered_
= true;
219 if (!key_frame_encountered_
) {
220 // Do not send video until we have encountered the first key frame.
221 // Save the bitstream buffer in |stream_header_| to be sent later along
222 // with the first key frame.
223 stream_header_
.append(static_cast<const char*>(output_buffer
->memory()),
225 } else if (!encoded_frame_data_storage_
.empty()) {
226 scoped_ptr
<EncodedFrame
> encoded_frame(
228 encoded_frame
->dependency
= key_frame
? EncodedFrame::KEY
:
229 EncodedFrame::DEPENDENT
;
230 encoded_frame
->frame_id
= ++last_encoded_frame_id_
;
232 encoded_frame
->referenced_frame_id
= encoded_frame
->frame_id
;
234 encoded_frame
->referenced_frame_id
= encoded_frame
->frame_id
- 1;
235 encoded_frame
->reference_time
=
236 encoded_frame_data_storage_
.front().capture_time
;
237 encoded_frame
->rtp_timestamp
=
238 GetVideoRtpTimestamp(encoded_frame
->reference_time
);
239 if (!stream_header_
.empty()) {
240 encoded_frame
->data
= stream_header_
;
241 stream_header_
.clear();
243 encoded_frame
->data
.append(
244 static_cast<const char*>(output_buffer
->memory()), payload_size
);
246 cast_environment_
->PostTask(
247 CastEnvironment::MAIN
,
249 base::Bind(&LogFrameEncodedEvent
,
251 cast_environment_
->Clock()->NowTicks(),
252 encoded_frame
->rtp_timestamp
,
253 encoded_frame
->frame_id
));
255 cast_environment_
->PostTask(
256 CastEnvironment::MAIN
,
258 base::Bind(encoded_frame_data_storage_
.front().frame_encoded_callback
,
259 base::Passed(&encoded_frame
)));
261 encoded_frame_data_storage_
.pop_front();
263 VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
266 // We need to re-add the output buffer to the encoder after we are done
268 video_encode_accelerator_
->UseOutputBitstreamBuffer(media::BitstreamBuffer(
270 output_buffers_
[bitstream_buffer_id
]->handle(),
271 output_buffers_
[bitstream_buffer_id
]->mapped_size()));
275 LocalVideoEncodeAcceleratorClient(
276 scoped_refptr
<CastEnvironment
> cast_environment
,
277 const CreateVideoEncodeMemoryCallback
& create_video_encode_mem_cb
,
278 const base::WeakPtr
<ExternalVideoEncoder
>& weak_owner
)
279 : cast_environment_(cast_environment
),
280 create_video_encode_memory_cb_(create_video_encode_mem_cb
),
281 weak_owner_(weak_owner
),
282 last_encoded_frame_id_(kStartFrameId
),
283 key_frame_encountered_(false) {}
285 // Trampoline VEA creation callback to OnCreateVideoEncodeAccelerator()
286 // on encoder_task_runner. Normally we would just repost the same method to
287 // it, and would not need a separate proxy method, but we can't
288 // ThreadTaskRunnerHandle::Get() in unittests just yet.
289 void OnCreateVideoEncodeAcceleratorProxy(
290 scoped_refptr
<base::SingleThreadTaskRunner
> encoder_task_runner
,
291 scoped_ptr
<media::VideoEncodeAccelerator
> vea
) {
292 encoder_task_runner
->PostTask(
294 base::Bind(&media::cast::LocalVideoEncodeAcceleratorClient::
295 OnCreateVideoEncodeAccelerator
,
298 base::Passed(&vea
)));
301 void OnCreateVideoEncodeAccelerator(
302 scoped_refptr
<base::SingleThreadTaskRunner
> encoder_task_runner
,
303 scoped_ptr
<media::VideoEncodeAccelerator
> vea
) {
304 encoder_task_runner_
= encoder_task_runner
;
305 video_encode_accelerator_
.reset(vea
.release());
307 cast_environment_
->PostTask(
308 CastEnvironment::MAIN
,
310 base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator
,
312 encoder_task_runner_
));
315 // Note: This method can be called on any thread.
316 void OnCreateSharedMemory(scoped_ptr
<base::SharedMemory
> memory
) {
317 encoder_task_runner_
->PostTask(
319 base::Bind(&LocalVideoEncodeAcceleratorClient::ReceivedSharedMemory
,
321 base::Passed(&memory
)));
324 void ReceivedSharedMemory(scoped_ptr
<base::SharedMemory
> memory
) {
325 DCHECK(encoder_task_runner_
.get());
326 DCHECK(encoder_task_runner_
->RunsTasksOnCurrentThread());
328 output_buffers_
.push_back(memory
.release());
330 // Wait until all requested buffers are received.
331 if (output_buffers_
.size() < kOutputBufferCount
)
334 // Immediately provide all output buffers to the VEA.
335 for (size_t i
= 0; i
< output_buffers_
.size(); ++i
) {
336 video_encode_accelerator_
->UseOutputBitstreamBuffer(
337 media::BitstreamBuffer(static_cast<int32
>(i
),
338 output_buffers_
[i
]->handle(),
339 output_buffers_
[i
]->mapped_size()));
342 cast_environment_
->PostTask(
343 CastEnvironment::MAIN
,
345 base::Bind(&ExternalVideoEncoder::EncoderInitialized
, weak_owner_
));
348 static void DestroyVideoEncodeAcceleratorOnEncoderThread(
349 scoped_ptr
<media::VideoEncodeAccelerator
> vea
) {
350 // VEA::~VEA specialization takes care of calling Destroy() on the VEA impl.
353 friend class base::RefCountedThreadSafe
<LocalVideoEncodeAcceleratorClient
>;
355 virtual ~LocalVideoEncodeAcceleratorClient() {
357 DCHECK(!video_encode_accelerator_
);
360 const scoped_refptr
<CastEnvironment
> cast_environment_
;
361 scoped_refptr
<base::SingleThreadTaskRunner
> encoder_task_runner_
;
362 scoped_ptr
<media::VideoEncodeAccelerator
> video_encode_accelerator_
;
363 const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_
;
364 const base::WeakPtr
<ExternalVideoEncoder
> weak_owner_
;
366 uint32 last_encoded_frame_id_
;
367 bool key_frame_encountered_
;
368 std::string stream_header_
;
370 // Shared memory buffers for output with the VideoAccelerator.
371 ScopedVector
<base::SharedMemory
> output_buffers_
;
374 std::list
<EncodedFrameReturnData
> encoded_frame_data_storage_
;
376 DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient
);
379 ExternalVideoEncoder::ExternalVideoEncoder(
380 scoped_refptr
<CastEnvironment
> cast_environment
,
381 const VideoSenderConfig
& video_config
,
382 const CreateVideoEncodeAcceleratorCallback
& create_vea_cb
,
383 const CreateVideoEncodeMemoryCallback
& create_video_encode_mem_cb
)
384 : video_config_(video_config
),
385 cast_environment_(cast_environment
),
386 encoder_active_(false),
387 key_frame_requested_(false),
388 weak_factory_(this) {
389 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
391 video_accelerator_client_
=
392 LocalVideoEncodeAcceleratorClient::Create(cast_environment_
,
394 create_video_encode_mem_cb
,
395 weak_factory_
.GetWeakPtr());
396 DCHECK(video_accelerator_client_
.get());
399 ExternalVideoEncoder::~ExternalVideoEncoder() {
402 void ExternalVideoEncoder::EncoderInitialized() {
403 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
404 encoder_active_
= true;
407 void ExternalVideoEncoder::EncoderError() {
408 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
409 encoder_active_
= false;
412 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
413 scoped_refptr
<base::SingleThreadTaskRunner
> encoder_task_runner
) {
414 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
415 encoder_task_runner_
= encoder_task_runner
;
417 encoder_task_runner_
->PostTask(
419 base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize
,
420 video_accelerator_client_
,
424 bool ExternalVideoEncoder::EncodeVideoFrame(
425 const scoped_refptr
<media::VideoFrame
>& video_frame
,
426 const base::TimeTicks
& capture_time
,
427 const FrameEncodedCallback
& frame_encoded_callback
) {
428 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
430 if (!encoder_active_
)
433 encoder_task_runner_
->PostTask(
435 base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame
,
436 video_accelerator_client_
,
439 key_frame_requested_
,
440 frame_encoded_callback
));
442 key_frame_requested_
= false;
446 // Inform the encoder about the new target bit rate.
447 void ExternalVideoEncoder::SetBitRate(int new_bit_rate
) {
448 if (!encoder_active_
) {
449 // If we receive SetBitRate() before VEA creation callback is invoked,
450 // cache the new bit rate in the encoder config and use the new settings
451 // to initialize VEA.
452 video_config_
.start_bitrate
= new_bit_rate
;
456 encoder_task_runner_
->PostTask(
458 base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate
,
459 video_accelerator_client_
,
463 // Inform the encoder to encode the next frame as a key frame.
464 void ExternalVideoEncoder::GenerateKeyFrame() {
465 DCHECK(cast_environment_
->CurrentlyOn(CastEnvironment::MAIN
));
466 key_frame_requested_
= true;
469 // Inform the encoder to only reference frames older or equal to frame_id;
470 void ExternalVideoEncoder::LatestFrameIdToReference(uint32
/*frame_id*/) {
471 // Do nothing not supported.