remoting/codec/video_encoder_vpx.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/codec/video_encoder_vpx.h"
   6
   7 #include "base/bind.h"
   8 #include "base/command_line.h"
   9 #include "base/logging.h"
  10 #include "base/sys_info.h"
  11 #include "remoting/base/util.h"
  12 #include "remoting/proto/video.pb.h"
  13 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
  14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
  17
  18 extern "C" {
  19 #define VPX_CODEC_DISABLE_COMPAT 1
  20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
  21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  22 }
  23
  24 namespace remoting {
  25
  26 namespace {
  27
  28 // Name of command-line flag to enable VP9 to use I444 by default.
  29 const char kEnableI444SwitchName[] = "enable-i444";
  30
  31 // Number of bytes in an RGBx pixel.
  32 const int kBytesPerRgbPixel = 4;
  33
  34 // Defines the dimension of a macro block. This is used to compute the active
  35 // map for the encoder.
  36 const int kMacroBlockSize = 16;
  37
  38 // Magic encoder profile numbers for I420 and I444 input formats.
  39 const int kVp9I420ProfileNumber = 0;
  40 const int kVp9I444ProfileNumber = 1;
  41
  42 void SetCommonCodecParameters(const webrtc::DesktopSize& size,
  43                               vpx_codec_enc_cfg_t* config) {
  44   // Use millisecond granularity time base.
  45   config->g_timebase.num = 1;
  46   config->g_timebase.den = 1000;
  47
  48   // Adjust default target bit-rate to account for actual desktop size.
  49   config->rc_target_bitrate = size.width() * size.height() *
  50       config->rc_target_bitrate / config->g_w / config->g_h;
  51
  52   config->g_w = size.width();
  53   config->g_h = size.height();
  54   config->g_pass = VPX_RC_ONE_PASS;
  55
  56   // Start emitting packets immediately.
  57   config->g_lag_in_frames = 0;
  58
  59   // Using 2 threads gives a great boost in performance for most systems with
  60   // adequate processing power. NB: Going to multiple threads on low end
  61   // windows systems can really hurt performance.
  62   // http://crbug.com/99179
  63   config->g_threads = (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
  64 }
  65
  66 ScopedVpxCodec CreateVP8Codec(const webrtc::DesktopSize& size) {
  67   ScopedVpxCodec codec(new vpx_codec_ctx_t);
  68
  69   // Configure the encoder.
  70   vpx_codec_enc_cfg_t config;
  71   const vpx_codec_iface_t* algo = vpx_codec_vp8_cx();
  72   CHECK(algo);
  73   vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0);
  74   if (ret != VPX_CODEC_OK)
  75     return ScopedVpxCodec();
  76
  77   SetCommonCodecParameters(size, &config);
  78
  79   // Value of 2 means using the real time profile. This is basically a
  80   // redundant option since we explicitly select real time mode when doing
  81   // encoding.
  82   config.g_profile = 2;
  83
  84   // Clamping the quantizer constrains the worst-case quality and CPU usage.
  85   config.rc_min_quantizer = 20;
  86   config.rc_max_quantizer = 30;
  87
  88   if (vpx_codec_enc_init(codec.get(), algo, &config, 0))
  89     return ScopedVpxCodec();
  90
  91   // Value of 16 will have the smallest CPU load. This turns off subpixel
  92   // motion search.
  93   if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, 16))
  94     return ScopedVpxCodec();
  95
  96   // Use the lowest level of noise sensitivity so as to spend less time
  97   // on motion estimation and inter-prediction mode.
  98   if (vpx_codec_control(codec.get(), VP8E_SET_NOISE_SENSITIVITY, 0))
  99     return ScopedVpxCodec();
 100
 101   return codec.Pass();
 102 }
 103
 104 ScopedVpxCodec CreateVP9Codec(const webrtc::DesktopSize& size,
 105                               bool lossless_color,
 106                               bool lossless_encode) {
 107   ScopedVpxCodec codec(new vpx_codec_ctx_t);
 108
 109   // Configure the encoder.
 110   vpx_codec_enc_cfg_t config;
 111   const vpx_codec_iface_t* algo = vpx_codec_vp9_cx();
 112   CHECK(algo);
 113   vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0);
 114   if (ret != VPX_CODEC_OK)
 115     return ScopedVpxCodec();
 116
 117   SetCommonCodecParameters(size, &config);
 118
 119   // Configure VP9 for I420 or I444 source frames.
 120   config.g_profile =
 121       lossless_color ? kVp9I444ProfileNumber : kVp9I420ProfileNumber;
 122
 123   if (lossless_encode) {
 124     // Disable quantization entirely, putting the encoder in "lossless" mode.
 125     config.rc_min_quantizer = 0;
 126     config.rc_max_quantizer = 0;
 127   } else {
 128     // Lossy encode using the same settings as for VP8.
 129     config.rc_min_quantizer = 20;
 130     config.rc_max_quantizer = 30;
 131   }
 132
 133   if (vpx_codec_enc_init(codec.get(), algo, &config, 0))
 134     return ScopedVpxCodec();
 135
 136   // Request the lowest-CPU encode feature-set that VP9 supports.
 137   // Note that this is configured via the same parameter as for VP8.
 138   if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, 5))
 139     return ScopedVpxCodec();
 140
 141   // Use the lowest level of noise sensitivity so as to spend less time
 142   // on motion estimation and inter-prediction mode.
 143   // Note that this is configured via the same parameter as for VP8.
 144   if (vpx_codec_control(codec.get(), VP8E_SET_NOISE_SENSITIVITY, 0))
 145     return ScopedVpxCodec();
 146
 147   return codec.Pass();
 148 }
 149
 150 void CreateImage(bool use_i444,
 151                  const webrtc::DesktopSize& size,
 152                  scoped_ptr<vpx_image_t>* out_image,
 153                  scoped_ptr<uint8[]>* out_image_buffer) {
 154   DCHECK(!size.is_empty());
 155
 156   scoped_ptr<vpx_image_t> image(new vpx_image_t());
 157   memset(image.get(), 0, sizeof(vpx_image_t));
 158
 159   // libvpx seems to require both to be assigned.
 160   image->d_w = size.width();
 161   image->w = size.width();
 162   image->d_h = size.height();
 163   image->h = size.height();
 164
 165   // libvpx should derive chroma shifts from|fmt| but currently has a bug:
 166   // https://code.google.com/p/webm/issues/detail?id=627
 167   if (use_i444) {
 168     image->fmt = VPX_IMG_FMT_I444;
 169     image->x_chroma_shift = 0;
 170     image->y_chroma_shift = 0;
 171   } else { // I420
 172     image->fmt = VPX_IMG_FMT_YV12;
 173     image->x_chroma_shift = 1;
 174     image->y_chroma_shift = 1;
 175   }
 176
 177   // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
 178   // the Y, U and V planes' strides to multiples of 16 bytes.
 179   const int y_stride = ((image->w - 1) & ~15) + 16;
 180   const int uv_unaligned_stride = y_stride >> image->x_chroma_shift;
 181   const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16;
 182
 183   // libvpx accesses the source image in macro blocks, and will over-read
 184   // if the image is not padded out to the next macroblock: crbug.com/119633.
 185   // Pad the Y, U and V planes' height out to compensate.
 186   // Assuming macroblocks are 16x16, aligning the planes' strides above also
 187   // macroblock aligned them.
 188   DCHECK_EQ(16, kMacroBlockSize);
 189   const int y_rows = ((image->h - 1) & ~(kMacroBlockSize-1)) + kMacroBlockSize;
 190   const int uv_rows = y_rows >> image->y_chroma_shift;
 191
 192   // Allocate a YUV buffer large enough for the aligned data & padding.
 193   const int buffer_size = y_stride * y_rows + 2*uv_stride * uv_rows;
 194   scoped_ptr<uint8[]> image_buffer(new uint8[buffer_size]);
 195
 196   // Reset image value to 128 so we just need to fill in the y plane.
 197   memset(image_buffer.get(), 128, buffer_size);
 198
 199   // Fill in the information for |image_|.
 200   unsigned char* uchar_buffer =
 201       reinterpret_cast<unsigned char*>(image_buffer.get());
 202   image->planes[0] = uchar_buffer;
 203   image->planes[1] = image->planes[0] + y_stride * y_rows;
 204   image->planes[2] = image->planes[1] + uv_stride * uv_rows;
 205   image->stride[0] = y_stride;
 206   image->stride[1] = uv_stride;
 207   image->stride[2] = uv_stride;
 208
 209   *out_image = image.Pass();
 210   *out_image_buffer = image_buffer.Pass();
 211 }
 212
 213 } // namespace
 214
 215 // static
 216 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP8() {
 217   return scoped_ptr<VideoEncoderVpx>(new VideoEncoderVpx(false));
 218 }
 219
 220 // static
 221 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP9() {
 222   return scoped_ptr<VideoEncoderVpx>(new VideoEncoderVpx(true));
 223 }
 224
 225 VideoEncoderVpx::~VideoEncoderVpx() {}
 226
 227 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless) {
 228   if (use_vp9_ && (want_lossless != lossless_encode_)) {
 229     lossless_encode_ = want_lossless;
 230     codec_.reset(); // Force encoder re-initialization.
 231   }
 232 }
 233
 234 void VideoEncoderVpx::SetLosslessColor(bool want_lossless) {
 235   if (use_vp9_ && (want_lossless != lossless_color_)) {
 236     lossless_color_ = want_lossless;
 237     codec_.reset(); // Force encoder re-initialization.
 238   }
 239 }
 240
 241 scoped_ptr<VideoPacket> VideoEncoderVpx::Encode(
 242     const webrtc::DesktopFrame& frame) {
 243   DCHECK_LE(32, frame.size().width());
 244   DCHECK_LE(32, frame.size().height());
 245
 246   base::TimeTicks encode_start_time = base::TimeTicks::Now();
 247
 248   if (!codec_ ||
 249       !frame.size().equals(webrtc::DesktopSize(image_->w, image_->h))) {
 250     bool ret = Initialize(frame.size());
 251     // TODO(hclam): Handle error better.
 252     CHECK(ret) << "Initialization of encoder failed";
 253
 254     // Set now as the base for timestamp calculation.
 255     timestamp_base_ = encode_start_time;
 256   }
 257
 258   // Convert the updated capture data ready for encode.
 259   webrtc::DesktopRegion updated_region;
 260   PrepareImage(frame, &updated_region);
 261
 262   // Update active map based on updated region.
 263   PrepareActiveMap(updated_region);
 264
 265   // Apply active map to the encoder.
 266   vpx_active_map_t act_map;
 267   act_map.rows = active_map_height_;
 268   act_map.cols = active_map_width_;
 269   act_map.active_map = active_map_.get();
 270   if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) {
 271     LOG(ERROR) << "Unable to apply active map";
 272   }
 273
 274   // Do the actual encoding.
 275   int timestamp = (encode_start_time - timestamp_base_).InMilliseconds();
 276   vpx_codec_err_t ret = vpx_codec_encode(
 277       codec_.get(), image_.get(), timestamp, 1, 0, VPX_DL_REALTIME);
 278   DCHECK_EQ(ret, VPX_CODEC_OK)
 279       << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n"
 280       << "Details: " << vpx_codec_error(codec_.get()) << "\n"
 281       << vpx_codec_error_detail(codec_.get());
 282
 283   // Read the encoded data.
 284   vpx_codec_iter_t iter = NULL;
 285   bool got_data = false;
 286
 287   // TODO(hclam): Make sure we get exactly one frame from the packet.
 288   // TODO(hclam): We should provide the output buffer to avoid one copy.
 289   scoped_ptr<VideoPacket> packet(new VideoPacket());
 290
 291   while (!got_data) {
 292     const vpx_codec_cx_pkt_t* vpx_packet =
 293         vpx_codec_get_cx_data(codec_.get(), &iter);
 294     if (!vpx_packet)
 295       continue;
 296
 297     switch (vpx_packet->kind) {
 298       case VPX_CODEC_CX_FRAME_PKT:
 299         got_data = true;
 300         packet->set_data(vpx_packet->data.frame.buf, vpx_packet->data.frame.sz);
 301         break;
 302       default:
 303         break;
 304     }
 305   }
 306
 307   // Construct the VideoPacket message.
 308   packet->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8);
 309   packet->mutable_format()->set_screen_width(frame.size().width());
 310   packet->mutable_format()->set_screen_height(frame.size().height());
 311   packet->set_capture_time_ms(frame.capture_time_ms());
 312   packet->set_encode_time_ms(
 313       (base::TimeTicks::Now() - encode_start_time).InMillisecondsRoundedUp());
 314   if (!frame.dpi().is_zero()) {
 315     packet->mutable_format()->set_x_dpi(frame.dpi().x());
 316     packet->mutable_format()->set_y_dpi(frame.dpi().y());
 317   }
 318   for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
 319        r.Advance()) {
 320     Rect* rect = packet->add_dirty_rects();
 321     rect->set_x(r.rect().left());
 322     rect->set_y(r.rect().top());
 323     rect->set_width(r.rect().width());
 324     rect->set_height(r.rect().height());
 325   }
 326
 327   return packet.Pass();
 328 }
 329
 330 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9)
 331     : use_vp9_(use_vp9),
 332       lossless_encode_(false),
 333       lossless_color_(false),
 334       active_map_width_(0),
 335       active_map_height_(0) {
 336   if (use_vp9_) {
 337     // Use lossless encoding mode by default.
 338     SetLosslessEncode(true);
 339
 340     // Use I444 colour space, by default, if specified on the command-line.
 341     if (CommandLine::ForCurrentProcess()->HasSwitch(kEnableI444SwitchName)) {
 342       SetLosslessColor(true);
 343     }
 344   }
 345 }
 346
 347 bool VideoEncoderVpx::Initialize(const webrtc::DesktopSize& size) {
 348   DCHECK(use_vp9_ || !lossless_color_);
 349   DCHECK(use_vp9_ || !lossless_encode_);
 350
 351   codec_.reset();
 352
 353   // (Re)Create the VPX image structure and pixel buffer.
 354   CreateImage(lossless_color_, size, &image_, &image_buffer_);
 355
 356   // Initialize active map.
 357   active_map_width_ = (image_->w + kMacroBlockSize - 1) / kMacroBlockSize;
 358   active_map_height_ = (image_->h + kMacroBlockSize - 1) / kMacroBlockSize;
 359   active_map_.reset(new uint8[active_map_width_ * active_map_height_]);
 360
 361   // (Re)Initialize the codec.
 362   if (use_vp9_) {
 363     codec_ = CreateVP9Codec(size, lossless_color_, lossless_encode_);
 364   } else {
 365     codec_ = CreateVP8Codec(size);
 366   }
 367
 368   return codec_;
 369 }
 370
 371 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame& frame,
 372                                    webrtc::DesktopRegion* updated_region) {
 373   if (frame.updated_region().is_empty()) {
 374     updated_region->Clear();
 375     return;
 376   }
 377
 378   // Align the region to macroblocks, to avoid encoding artefacts.
 379   // This also ensures that all rectangles have even-aligned top-left, which
 380   // is required for ConvertRGBToYUVWithRect() to work.
 381   std::vector<webrtc::DesktopRect> aligned_rects;
 382   for (webrtc::DesktopRegion::Iterator r(frame.updated_region());
 383        !r.IsAtEnd(); r.Advance()) {
 384     const webrtc::DesktopRect& rect = r.rect();
 385     aligned_rects.push_back(AlignRect(webrtc::DesktopRect::MakeLTRB(
 386         rect.left(), rect.top(), rect.right(), rect.bottom())));
 387   }
 388   DCHECK(!aligned_rects.empty());
 389   updated_region->Clear();
 390   updated_region->AddRects(&aligned_rects[0], aligned_rects.size());
 391
 392   // Clip back to the screen dimensions, in case they're not macroblock aligned.
 393   // The conversion routines don't require even width & height, so this is safe
 394   // even if the source dimensions are not even.
 395   updated_region->IntersectWith(
 396       webrtc::DesktopRect::MakeWH(image_->w, image_->h));
 397
 398   // Convert the updated region to YUV ready for encoding.
 399   const uint8* rgb_data = frame.data();
 400   const int rgb_stride = frame.stride();
 401   const int y_stride = image_->stride[0];
 402   DCHECK_EQ(image_->stride[1], image_->stride[2]);
 403   const int uv_stride = image_->stride[1];
 404   uint8* y_data = image_->planes[0];
 405   uint8* u_data = image_->planes[1];
 406   uint8* v_data = image_->planes[2];
 407
 408   switch (image_->fmt) {
 409     case VPX_IMG_FMT_I444:
 410       for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
 411            r.Advance()) {
 412         const webrtc::DesktopRect& rect = r.rect();
 413         int rgb_offset = rgb_stride * rect.top() +
 414                          rect.left() * kBytesPerRgbPixel;
 415         int yuv_offset = uv_stride * rect.top() + rect.left();
 416         libyuv::ARGBToI444(rgb_data + rgb_offset, rgb_stride,
 417                            y_data + yuv_offset, y_stride,
 418                            u_data + yuv_offset, uv_stride,
 419                            v_data + yuv_offset, uv_stride,
 420                            rect.width(), rect.height());
 421       }
 422       break;
 423     case VPX_IMG_FMT_YV12:
 424       for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
 425            r.Advance()) {
 426         const webrtc::DesktopRect& rect = r.rect();
 427         int rgb_offset = rgb_stride * rect.top() +
 428                          rect.left() * kBytesPerRgbPixel;
 429         int y_offset = y_stride * rect.top() + rect.left();
 430         int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
 431         libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride,
 432                            y_data + y_offset, y_stride,
 433                            u_data + uv_offset, uv_stride,
 434                            v_data + uv_offset, uv_stride,
 435                            rect.width(), rect.height());
 436       }
 437       break;
 438     default:
 439       NOTREACHED();
 440       break;
 441   }
 442 }
 443
 444 void VideoEncoderVpx::PrepareActiveMap(
 445     const webrtc::DesktopRegion& updated_region) {
 446   // Clear active map first.
 447   memset(active_map_.get(), 0, active_map_width_ * active_map_height_);
 448
 449   // Mark updated areas active.
 450   for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
 451        r.Advance()) {
 452     const webrtc::DesktopRect& rect = r.rect();
 453     int left = rect.left() / kMacroBlockSize;
 454     int right = (rect.right() - 1) / kMacroBlockSize;
 455     int top = rect.top() / kMacroBlockSize;
 456     int bottom = (rect.bottom() - 1) / kMacroBlockSize;
 457     DCHECK_LT(right, active_map_width_);
 458     DCHECK_LT(bottom, active_map_height_);
 459
 460     uint8* map = active_map_.get() + top * active_map_width_;
 461     for (int y = top; y <= bottom; ++y) {
 462       for (int x = left; x <= right; ++x)
 463         map[x] = 1;
 464       map += active_map_width_;
 465     }
 466   }
 467 }
 468
 469 }  // namespace remoting