media/video/ffmpeg_video_decode_engine.cc

   1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/video/ffmpeg_video_decode_engine.h"
   6
   7 #include "base/command_line.h"
   8 #include "base/string_number_conversions.h"
   9 #include "base/task.h"
  10 #include "media/base/buffers.h"
  11 #include "media/base/callback.h"
  12 #include "media/base/limits.h"
  13 #include "media/base/media_switches.h"
  14 #include "media/base/pipeline.h"
  15 #include "media/ffmpeg/ffmpeg_common.h"
  16 #include "media/filters/ffmpeg_demuxer.h"
  17
  18 namespace media {
  19
  20 FFmpegVideoDecodeEngine::FFmpegVideoDecodeEngine()
  21     : codec_context_(NULL),
  22       event_handler_(NULL),
  23       frame_rate_numerator_(0),
  24       frame_rate_denominator_(0),
  25       pending_input_buffers_(0),
  26       pending_output_buffers_(0),
  27       output_eos_reached_(false),
  28       flush_pending_(false) {
  29 }
  30
  31 FFmpegVideoDecodeEngine::~FFmpegVideoDecodeEngine() {
  32   if (codec_context_) {
  33     av_free(codec_context_->extradata);
  34     avcodec_close(codec_context_);
  35     av_free(codec_context_);
  36   }
  37 }
  38
  39 void FFmpegVideoDecodeEngine::Initialize(
  40     MessageLoop* message_loop,
  41     VideoDecodeEngine::EventHandler* event_handler,
  42     VideoDecodeContext* context,
  43     const VideoDecoderConfig& config) {
  44   // Always try to use three threads for video decoding.  There is little reason
  45   // not to since current day CPUs tend to be multi-core and we measured
  46   // performance benefits on older machines such as P4s with hyperthreading.
  47   //
  48   // Handling decoding on separate threads also frees up the pipeline thread to
  49   // continue processing. Although it'd be nice to have the option of a single
  50   // decoding thread, FFmpeg treats having one thread the same as having zero
  51   // threads (i.e., avcodec_decode_video() will execute on the calling thread).
  52   // Yet another reason for having two threads :)
  53   static const int kDecodeThreads = 2;
  54   static const int kMaxDecodeThreads = 16;
  55
  56   // Initialize AVCodecContext structure.
  57   codec_context_ = avcodec_alloc_context();
  58
  59   // TODO(scherkus): should video format get passed in via VideoDecoderConfig?
  60   codec_context_->pix_fmt = PIX_FMT_YUV420P;
  61   codec_context_->codec_type = AVMEDIA_TYPE_VIDEO;
  62   codec_context_->codec_id = VideoCodecToCodecID(config.codec());
  63   codec_context_->coded_width = config.width();
  64   codec_context_->coded_height = config.height();
  65
  66   frame_rate_numerator_ = config.frame_rate_numerator();
  67   frame_rate_denominator_ = config.frame_rate_denominator();
  68
  69   if (config.extra_data() != NULL) {
  70     codec_context_->extradata_size = config.extra_data_size();
  71     codec_context_->extradata = reinterpret_cast<uint8_t*>(
  72         av_malloc(config.extra_data_size() + FF_INPUT_BUFFER_PADDING_SIZE));
  73     memcpy(codec_context_->extradata, config.extra_data(),
  74            config.extra_data_size());
  75     memset(codec_context_->extradata + config.extra_data_size(), '\0',
  76            FF_INPUT_BUFFER_PADDING_SIZE);
  77   }
  78
  79   // Enable motion vector search (potentially slow), strong deblocking filter
  80   // for damaged macroblocks, and set our error detection sensitivity.
  81   codec_context_->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
  82   codec_context_->error_recognition = FF_ER_CAREFUL;
  83
  84   AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
  85
  86   // TODO(fbarchard): Improve thread logic based on size / codec.
  87   // TODO(fbarchard): Fix bug affecting video-cookie.html
  88   int decode_threads = (codec_context_->codec_id == CODEC_ID_THEORA) ?
  89       1 : kDecodeThreads;
  90
  91   const CommandLine* cmd_line = CommandLine::ForCurrentProcess();
  92   std::string threads(cmd_line->GetSwitchValueASCII(switches::kVideoThreads));
  93   if ((!threads.empty() &&
  94       !base::StringToInt(threads, &decode_threads)) ||
  95       decode_threads < 0 || decode_threads > kMaxDecodeThreads) {
  96     decode_threads = kDecodeThreads;
  97   }
  98
  99   // We don't allocate AVFrame on the stack since different versions of FFmpeg
 100   // may change the size of AVFrame, causing stack corruption.  The solution is
 101   // to let FFmpeg allocate the structure via avcodec_alloc_frame().
 102   av_frame_.reset(avcodec_alloc_frame());
 103   VideoCodecInfo info;
 104   info.success = false;
 105   info.provides_buffers = true;
 106   info.stream_info.surface_format = GetSurfaceFormat();
 107   info.stream_info.surface_width = config.surface_width();
 108   info.stream_info.surface_height = config.surface_height();
 109
 110   // If we do not have enough buffers, we will report error too.
 111   frame_queue_available_.clear();
 112
 113   // Create output buffer pool when direct rendering is not used.
 114   for (size_t i = 0; i < Limits::kMaxVideoFrames; ++i) {
 115     scoped_refptr<VideoFrame> video_frame =
 116         VideoFrame::CreateFrame(VideoFrame::YV12,
 117                                 config.width(),
 118                                 config.height(),
 119                                 kNoTimestamp,
 120                                 kNoTimestamp);
 121     frame_queue_available_.push_back(video_frame);
 122   }
 123   codec_context_->thread_count = decode_threads;
 124   if (codec &&
 125       avcodec_open(codec_context_, codec) >= 0 &&
 126       av_frame_.get()) {
 127     info.success = true;
 128   }
 129   event_handler_ = event_handler;
 130   event_handler_->OnInitializeComplete(info);
 131 }
 132
 133 // TODO(scherkus): Move this function to a utility class and unit test.
 134 static void CopyPlane(size_t plane,
 135                       scoped_refptr<VideoFrame> video_frame,
 136                       const AVFrame* frame,
 137                       size_t source_height) {
 138   DCHECK_EQ(video_frame->width() % 2, 0u);
 139   const uint8* source = frame->data[plane];
 140   const size_t source_stride = frame->linesize[plane];
 141   uint8* dest = video_frame->data(plane);
 142   const size_t dest_stride = video_frame->stride(plane);
 143
 144   // Calculate amounts to copy and clamp to minium frame dimensions.
 145   size_t bytes_per_line = video_frame->width();
 146   size_t copy_lines = std::min(video_frame->height(), source_height);
 147   if (plane != VideoFrame::kYPlane) {
 148     bytes_per_line /= 2;
 149     if (video_frame->format() == VideoFrame::YV12) {
 150       copy_lines = (copy_lines + 1) / 2;
 151     }
 152   }
 153   bytes_per_line = std::min(bytes_per_line, source_stride);
 154
 155   // Copy!
 156   for (size_t i = 0; i < copy_lines; ++i) {
 157     memcpy(dest, source, bytes_per_line);
 158     source += source_stride;
 159     dest += dest_stride;
 160   }
 161 }
 162
 163 void FFmpegVideoDecodeEngine::ConsumeVideoSample(
 164     scoped_refptr<Buffer> buffer) {
 165   pending_input_buffers_--;
 166   if (flush_pending_) {
 167     TryToFinishPendingFlush();
 168   } else {
 169     // Otherwise try to decode this buffer.
 170     DecodeFrame(buffer);
 171   }
 172 }
 173
 174 void FFmpegVideoDecodeEngine::ProduceVideoFrame(
 175     scoped_refptr<VideoFrame> frame) {
 176   // We should never receive NULL frame or EOS frame.
 177   DCHECK(frame.get() && !frame->IsEndOfStream());
 178
 179   // Increment pending output buffer count.
 180   pending_output_buffers_++;
 181
 182   // Return this frame to available pool after display.
 183   frame_queue_available_.push_back(frame);
 184
 185   if (flush_pending_) {
 186     TryToFinishPendingFlush();
 187   } else if (!output_eos_reached_) {
 188     // If we already deliver EOS to renderer, we stop reading new input.
 189     ReadInput();
 190   }
 191 }
 192
 193 // Try to decode frame when both input and output are ready.
 194 void FFmpegVideoDecodeEngine::DecodeFrame(scoped_refptr<Buffer> buffer) {
 195   scoped_refptr<VideoFrame> video_frame;
 196
 197   // Create a packet for input data.
 198   // Due to FFmpeg API changes we no longer have const read-only pointers.
 199   AVPacket packet;
 200   av_init_packet(&packet);
 201   packet.data = const_cast<uint8*>(buffer->GetData());
 202   packet.size = buffer->GetDataSize();
 203
 204   PipelineStatistics statistics;
 205   statistics.video_bytes_decoded = buffer->GetDataSize();
 206
 207   // Let FFmpeg handle presentation timestamp reordering.
 208   codec_context_->reordered_opaque = buffer->GetTimestamp().InMicroseconds();
 209
 210   // This is for codecs not using get_buffer to initialize
 211   // |av_frame_->reordered_opaque|
 212   av_frame_->reordered_opaque = codec_context_->reordered_opaque;
 213
 214   int frame_decoded = 0;
 215   int result = avcodec_decode_video2(codec_context_,
 216                                      av_frame_.get(),
 217                                      &frame_decoded,
 218                                      &packet);
 219
 220   // Log the problem if we can't decode a video frame and exit early.
 221   if (result < 0) {
 222     LOG(ERROR) << "Error decoding a video frame with timestamp: "
 223                << buffer->GetTimestamp().InMicroseconds() << " us, duration: "
 224                << buffer->GetDuration().InMicroseconds() << " us, packet size: "
 225                << buffer->GetDataSize() << " bytes";
 226     event_handler_->OnError();
 227     return;
 228   }
 229
 230   // If frame_decoded == 0, then no frame was produced.
 231   // In this case, if we already begin to flush codec with empty
 232   // input packet at the end of input stream, the first time we
 233   // encounter frame_decoded == 0 signal output frame had been
 234   // drained, we mark the flag. Otherwise we read from demuxer again.
 235   if (frame_decoded == 0) {
 236     if (buffer->IsEndOfStream()) {  // We had started flushing.
 237       event_handler_->ConsumeVideoFrame(video_frame, statistics);
 238       output_eos_reached_ = true;
 239     } else {
 240       ReadInput();
 241     }
 242     return;
 243   }
 244
 245   // TODO(fbarchard): Work around for FFmpeg http://crbug.com/27675
 246   // The decoder is in a bad state and not decoding correctly.
 247   // Checking for NULL avoids a crash in CopyPlane().
 248   if (!av_frame_->data[VideoFrame::kYPlane] ||
 249       !av_frame_->data[VideoFrame::kUPlane] ||
 250       !av_frame_->data[VideoFrame::kVPlane]) {
 251     event_handler_->OnError();
 252     return;
 253   }
 254
 255   // Determine timestamp and calculate the duration based on the repeat picture
 256   // count.  According to FFmpeg docs, the total duration can be calculated as
 257   // follows:
 258   //   fps = 1 / time_base
 259   //
 260   //   duration = (1 / fps) + (repeat_pict) / (2 * fps)
 261   //            = (2 + repeat_pict) / (2 * fps)
 262   //            = (2 + repeat_pict) / (2 * (1 / time_base))
 263   DCHECK_LE(av_frame_->repeat_pict, 2);  // Sanity check.
 264   AVRational doubled_time_base;
 265   doubled_time_base.num = frame_rate_denominator_;
 266   doubled_time_base.den = frame_rate_numerator_ * 2;
 267
 268   base::TimeDelta timestamp =
 269       base::TimeDelta::FromMicroseconds(av_frame_->reordered_opaque);
 270   base::TimeDelta duration =
 271       ConvertFromTimeBase(doubled_time_base, 2 + av_frame_->repeat_pict);
 272
 273   // Available frame is guaranteed, because we issue as much reads as
 274   // available frame, except the case of |frame_decoded| == 0, which
 275   // implies decoder order delay, and force us to read more inputs.
 276   DCHECK(frame_queue_available_.size());
 277   video_frame = frame_queue_available_.front();
 278   frame_queue_available_.pop_front();
 279
 280   // Copy the frame data since FFmpeg reuses internal buffers for AVFrame
 281   // output, meaning the data is only valid until the next
 282   // avcodec_decode_video() call.
 283   size_t height = codec_context_->height;
 284   CopyPlane(VideoFrame::kYPlane, video_frame.get(), av_frame_.get(), height);
 285   CopyPlane(VideoFrame::kUPlane, video_frame.get(), av_frame_.get(), height);
 286   CopyPlane(VideoFrame::kVPlane, video_frame.get(), av_frame_.get(), height);
 287
 288   video_frame->SetTimestamp(timestamp);
 289   video_frame->SetDuration(duration);
 290
 291   pending_output_buffers_--;
 292   event_handler_->ConsumeVideoFrame(video_frame, statistics);
 293 }
 294
 295 void FFmpegVideoDecodeEngine::Uninitialize() {
 296   event_handler_->OnUninitializeComplete();
 297 }
 298
 299 void FFmpegVideoDecodeEngine::Flush() {
 300   avcodec_flush_buffers(codec_context_);
 301   flush_pending_ = true;
 302   TryToFinishPendingFlush();
 303 }
 304
 305 void FFmpegVideoDecodeEngine::TryToFinishPendingFlush() {
 306   DCHECK(flush_pending_);
 307
 308   // We consider ourself flushed when there is no pending input buffers
 309   // and output buffers, which implies that all buffers had been returned
 310   // to its owner.
 311   if (!pending_input_buffers_ && !pending_output_buffers_) {
 312     // Try to finish flushing and notify pipeline.
 313     flush_pending_ = false;
 314     event_handler_->OnFlushComplete();
 315   }
 316 }
 317
 318 void FFmpegVideoDecodeEngine::Seek() {
 319   // After a seek, output stream no longer considered as EOS.
 320   output_eos_reached_ = false;
 321
 322   // The buffer provider is assumed to perform pre-roll operation.
 323   for (unsigned int i = 0; i < Limits::kMaxVideoFrames; ++i)
 324     ReadInput();
 325
 326   event_handler_->OnSeekComplete();
 327 }
 328
 329 void FFmpegVideoDecodeEngine::ReadInput() {
 330   DCHECK_EQ(output_eos_reached_, false);
 331   pending_input_buffers_++;
 332   event_handler_->ProduceVideoSample(NULL);
 333 }
 334
 335 VideoFrame::Format FFmpegVideoDecodeEngine::GetSurfaceFormat() const {
 336   // J (Motion JPEG) versions of YUV are full range 0..255.
 337   // Regular (MPEG) YUV is 16..240.
 338   // For now we will ignore the distinction and treat them the same.
 339   switch (codec_context_->pix_fmt) {
 340     case PIX_FMT_YUV420P:
 341     case PIX_FMT_YUVJ420P:
 342       return VideoFrame::YV12;
 343     case PIX_FMT_YUV422P:
 344     case PIX_FMT_YUVJ422P:
 345       return VideoFrame::YV16;
 346     default:
 347       // TODO(scherkus): More formats here?
 348       break;
 349   }
 350   return VideoFrame::INVALID;
 351 }
 352
 353 }  // namespace media
 354
 355 // Disable refcounting for this object because this object only lives
 356 // on the video decoder thread and there's no need to refcount it.
 357 DISABLE_RUNNABLE_METHOD_REFCOUNT(media::FFmpegVideoDecodeEngine);