cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/math_util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  31                                uint32 sync_point)
  32       : gl_(gl), sync_point_(sync_point) {}
  33   ~SyncPointClientImpl() override {}
  34   uint32 InsertSyncPoint() override {
  35     if (sync_point_)
  36       return sync_point_;
  37     return gl_->InsertSyncPointCHROMIUM();
  38   }
  39   void WaitSyncPoint(uint32 sync_point) override {
  40     if (!sync_point)
  41       return;
  42     gl_->WaitSyncPointCHROMIUM(sync_point);
  43     if (sync_point_) {
  44       gl_->WaitSyncPointCHROMIUM(sync_point_);
  45       sync_point_ = 0;
  46     }
  47   }
  48
  49  private:
  50   gpu::gles2::GLES2Interface* gl_;
  51   uint32 sync_point_;
  52 };
  53
  54 }  // namespace
  55
  56 VideoResourceUpdater::PlaneResource::PlaneResource(
  57     unsigned int resource_id,
  58     const gfx::Size& resource_size,
  59     ResourceFormat resource_format,
  60     gpu::Mailbox mailbox)
  61     : resource_id(resource_id),
  62       resource_size(resource_size),
  63       resource_format(resource_format),
  64       mailbox(mailbox),
  65       ref_count(0),
  66       frame_ptr(nullptr),
  67       plane_index(0u) {
  68 }
  69
  70 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  71     const PlaneResource& plane_resource,
  72     const media::VideoFrame* video_frame,
  73     size_t plane_index) {
  74   return plane_resource.frame_ptr == video_frame &&
  75          plane_resource.plane_index == plane_index &&
  76          plane_resource.timestamp == video_frame->timestamp();
  77 }
  78
  79 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  80     const media::VideoFrame* video_frame,
  81     size_t plane_index,
  82     PlaneResource* plane_resource) {
  83   plane_resource->frame_ptr = video_frame;
  84   plane_resource->plane_index = plane_index;
  85   plane_resource->timestamp = video_frame->timestamp();
  86 }
  87
  88 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  89
  90 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  91
  92 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  93                                            ResourceProvider* resource_provider)
  94     : context_provider_(context_provider),
  95       resource_provider_(resource_provider) {
  96 }
  97
  98 VideoResourceUpdater::~VideoResourceUpdater() {
  99   for (const PlaneResource& plane_resource : all_resources_)
 100     resource_provider_->DeleteResource(plane_resource.resource_id);
 101 }
 102
 103 VideoResourceUpdater::ResourceList::iterator
 104 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 105                                        ResourceFormat format,
 106                                        bool has_mailbox) {
 107   // TODO(danakj): Abstract out hw/sw resource create/delete from
 108   // ResourceProvider and stop using ResourceProvider in this class.
 109   const ResourceId resource_id = resource_provider_->CreateResource(
 110       plane_size, GL_CLAMP_TO_EDGE, ResourceProvider::TEXTURE_HINT_IMMUTABLE,
 111       format);
 112   if (resource_id == 0)
 113     return all_resources_.end();
 114
 115   gpu::Mailbox mailbox;
 116   if (has_mailbox) {
 117     DCHECK(context_provider_);
 118
 119     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 120
 121     gl->GenMailboxCHROMIUM(mailbox.name);
 122     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 123     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 124                                      mailbox.name);
 125   }
 126   all_resources_.push_front(
 127       PlaneResource(resource_id, plane_size, format, mailbox));
 128   return all_resources_.begin();
 129 }
 130
 131 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 132   DCHECK_EQ(resource_it->ref_count, 0);
 133   resource_provider_->DeleteResource(resource_it->resource_id);
 134   all_resources_.erase(resource_it);
 135 }
 136
 137 VideoFrameExternalResources VideoResourceUpdater::
 138     CreateExternalResourcesFromVideoFrame(
 139         const scoped_refptr<media::VideoFrame>& video_frame) {
 140   if (video_frame->format() == media::VideoFrame::UNKNOWN)
 141     return VideoFrameExternalResources();
 142
 143   if (video_frame->storage_type() == media::VideoFrame::STORAGE_TEXTURE)
 144     return CreateForHardwarePlanes(video_frame);
 145   else
 146     return CreateForSoftwarePlanes(video_frame);
 147 }
 148
 149 // For frames that we receive in software format, determine the dimensions of
 150 // each plane in the frame.
 151 static gfx::Size SoftwarePlaneDimension(
 152     const scoped_refptr<media::VideoFrame>& input_frame,
 153     bool software_compositor,
 154     size_t plane_index) {
 155   if (!software_compositor) {
 156     return media::VideoFrame::PlaneSize(
 157         input_frame->format(), plane_index, input_frame->coded_size());
 158   }
 159   return input_frame->coded_size();
 160 }
 161
 162 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 163     const scoped_refptr<media::VideoFrame>& video_frame) {
 164   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 165   const media::VideoFrame::Format input_frame_format = video_frame->format();
 166
 167 #if defined(VIDEO_HOLE)
 168   if (video_frame->storage_type() == media::VideoFrame::STORAGE_HOLE) {
 169     VideoFrameExternalResources external_resources;
 170     external_resources.type = VideoFrameExternalResources::HOLE;
 171     return external_resources;
 172   }
 173 #endif  // defined(VIDEO_HOLE)
 174
 175   // Only YUV software video frames are supported.
 176   if (!media::VideoFrame::IsYuvPlanar(input_frame_format)) {
 177     NOTREACHED() << media::VideoFrame::FormatToString(input_frame_format);
 178     return VideoFrameExternalResources();
 179   }
 180
 181   const bool software_compositor = context_provider_ == NULL;
 182
 183   ResourceFormat output_resource_format =
 184       resource_provider_->yuv_resource_format();
 185   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 186
 187   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 188   // conversion here. That involves an extra copy of each frame to a bitmap.
 189   // Obviously, this is suboptimal and should be addressed once ubercompositor
 190   // starts shaping up.
 191   if (software_compositor) {
 192     output_resource_format = kRGBResourceFormat;
 193     output_plane_count = 1;
 194   }
 195
 196   // Drop recycled resources that are the wrong format.
 197   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 198     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 199       DeleteResource(it++);
 200     else
 201       ++it;
 202   }
 203
 204   const int max_resource_size = resource_provider_->max_texture_size();
 205   std::vector<ResourceList::iterator> plane_resources;
 206   for (size_t i = 0; i < output_plane_count; ++i) {
 207     gfx::Size output_plane_resource_size =
 208         SoftwarePlaneDimension(video_frame, software_compositor, i);
 209     if (output_plane_resource_size.IsEmpty() ||
 210         output_plane_resource_size.width() > max_resource_size ||
 211         output_plane_resource_size.height() > max_resource_size) {
 212       break;
 213     }
 214
 215     // Try recycle a previously-allocated resource.
 216     ResourceList::iterator resource_it = all_resources_.end();
 217     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 218       if (it->resource_size == output_plane_resource_size &&
 219           it->resource_format == output_resource_format) {
 220         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 221           // Bingo, we found a resource that already contains the data we are
 222           // planning to put in it. It's safe to reuse it even if
 223           // resource_provider_ holds some references to it, because those
 224           // references are read-only.
 225           resource_it = it;
 226           break;
 227         }
 228
 229         // This extra check is needed because resources backed by SharedMemory
 230         // are not ref-counted, unlike mailboxes. Full discussion in
 231         // codereview.chromium.org/145273021.
 232         const bool in_use =
 233             software_compositor &&
 234             resource_provider_->InUseByConsumer(it->resource_id);
 235         if (it->ref_count == 0 && !in_use) {
 236           // We found a resource with the correct size that we can overwrite.
 237           resource_it = it;
 238         }
 239       }
 240     }
 241
 242     // Check if we need to allocate a new resource.
 243     if (resource_it == all_resources_.end()) {
 244       resource_it =
 245           AllocateResource(output_plane_resource_size, output_resource_format,
 246                            !software_compositor);
 247     }
 248     if (resource_it == all_resources_.end())
 249       break;
 250
 251     ++resource_it->ref_count;
 252     plane_resources.push_back(resource_it);
 253   }
 254
 255   if (plane_resources.size() != output_plane_count) {
 256     // Allocation failed, nothing will be returned so restore reference counts.
 257     for (ResourceList::iterator resource_it : plane_resources)
 258       --resource_it->ref_count;
 259     return VideoFrameExternalResources();
 260   }
 261
 262   VideoFrameExternalResources external_resources;
 263
 264   if (software_compositor) {
 265     DCHECK_EQ(plane_resources.size(), 1u);
 266     PlaneResource& plane_resource = *plane_resources[0];
 267     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 268     DCHECK(plane_resource.mailbox.IsZero());
 269
 270     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 271       // We need to transfer data from |video_frame| to the plane resource.
 272       if (!video_renderer_)
 273         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 274
 275       ResourceProvider::ScopedWriteLockSoftware lock(
 276           resource_provider_, plane_resource.resource_id);
 277       SkCanvas canvas(lock.sk_bitmap());
 278       // This is software path, so canvas and video_frame are always backed
 279       // by software.
 280       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 281       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 282     }
 283
 284     external_resources.software_resources.push_back(plane_resource.resource_id);
 285     external_resources.software_release_callback =
 286         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 287     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 288     return external_resources;
 289   }
 290
 291   for (size_t i = 0; i < plane_resources.size(); ++i) {
 292     PlaneResource& plane_resource = *plane_resources[i];
 293     // Update each plane's resource id with its content.
 294     DCHECK_EQ(plane_resource.resource_format,
 295               resource_provider_->yuv_resource_format());
 296
 297     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 298       // We need to transfer data from |video_frame| to the plane resource.
 299       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 300
 301       // The |resource_size_pixels| is the size of the resource we want to
 302       // upload to.
 303       gfx::Size resource_size_pixels = plane_resource.resource_size;
 304       // The |video_stride_pixels| is the width of the video frame we are
 305       // uploading (including non-frame data to fill in the stride).
 306       size_t video_stride_pixels = video_frame->stride(i);
 307
 308       size_t bytes_per_pixel = BitsPerPixel(plane_resource.resource_format) / 8;
 309       // Use 4-byte row alignment (OpenGL default) for upload performance.
 310       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 311       size_t upload_image_stride = MathUtil::RoundUp<size_t>(
 312           bytes_per_pixel * resource_size_pixels.width(), 4u);
 313
 314       const uint8_t* pixels;
 315       if (upload_image_stride == video_stride_pixels * bytes_per_pixel) {
 316         pixels = video_frame->data(i);
 317       } else {
 318         // Avoid malloc for each frame/plane if possible.
 319         size_t needed_size =
 320             upload_image_stride * resource_size_pixels.height();
 321         if (upload_pixels_.size() < needed_size)
 322           upload_pixels_.resize(needed_size);
 323         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 324           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 325           const uint8_t* src = video_frame->data(i) +
 326                                bytes_per_pixel * video_stride_pixels * row;
 327           memcpy(dst, src, resource_size_pixels.width() * bytes_per_pixel);
 328         }
 329         pixels = &upload_pixels_[0];
 330       }
 331
 332       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 333                                          resource_size_pixels);
 334       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 335     }
 336
 337     external_resources.mailboxes.push_back(
 338         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 339     external_resources.release_callbacks.push_back(
 340         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 341   }
 342
 343   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 344   return external_resources;
 345 }
 346
 347 // static
 348 void VideoResourceUpdater::ReturnTexture(
 349     base::WeakPtr<VideoResourceUpdater> updater,
 350     const scoped_refptr<media::VideoFrame>& video_frame,
 351     uint32 sync_point,
 352     bool lost_resource,
 353     BlockingTaskRunner* main_thread_task_runner) {
 354   // TODO(dshwang) this case should be forwarded to the decoder as lost
 355   // resource.
 356   if (lost_resource || !updater.get())
 357     return;
 358   // Update the release sync point in |video_frame| with |sync_point|
 359   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 360   // |video_frame|'s previous sync point using the current GL context.
 361   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 362                              sync_point);
 363   video_frame->UpdateReleaseSyncPoint(&client);
 364 }
 365
 366 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 367     const scoped_refptr<media::VideoFrame>& video_frame) {
 368   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 369   DCHECK_EQ(video_frame->storage_type(), media::VideoFrame::STORAGE_TEXTURE);
 370   if (!context_provider_)
 371     return VideoFrameExternalResources();
 372
 373   const size_t textures = media::VideoFrame::NumPlanes(video_frame->format());
 374   DCHECK_GE(textures, 1u);
 375   VideoFrameExternalResources external_resources;
 376   switch (video_frame->format()) {
 377     case media::VideoFrame::ARGB:
 378     case media::VideoFrame::XRGB:
 379       DCHECK_EQ(1u, textures);
 380       switch (video_frame->mailbox_holder(0).texture_target) {
 381         case GL_TEXTURE_2D:
 382           external_resources.type =
 383               (video_frame->format() == media::VideoFrame::XRGB)
 384                   ? VideoFrameExternalResources::RGB_RESOURCE
 385                   : VideoFrameExternalResources::RGBA_RESOURCE;
 386           break;
 387         case GL_TEXTURE_EXTERNAL_OES:
 388           external_resources.type =
 389               VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 390           break;
 391         case GL_TEXTURE_RECTANGLE_ARB:
 392           external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 393           break;
 394         default:
 395           NOTREACHED();
 396           return VideoFrameExternalResources();
 397       }
 398       break;
 399     case media::VideoFrame::I420:
 400       external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 401       break;
 402 #if defined(OS_MACOSX) || defined(OS_CHROMEOS)
 403     case media::VideoFrame::NV12:
 404 #endif
 405     case media::VideoFrame::YV12:
 406     case media::VideoFrame::YV16:
 407     case media::VideoFrame::YV24:
 408     case media::VideoFrame::YV12A:
 409     case media::VideoFrame::UNKNOWN:
 410       DLOG(ERROR) << "Unsupported Texture format"
 411                   << media::VideoFrame::FormatToString(video_frame->format());
 412       return external_resources;
 413   }
 414   DCHECK_NE(VideoFrameExternalResources::NONE, external_resources.type);
 415
 416   for (size_t i = 0; i < textures; ++i) {
 417     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 418     external_resources.mailboxes.push_back(
 419         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 420                        mailbox_holder.sync_point));
 421     external_resources.mailboxes.back().set_allow_overlay(
 422         video_frame->allow_overlay());
 423     external_resources.release_callbacks.push_back(
 424         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 425   }
 426   return external_resources;
 427 }
 428
 429 // static
 430 void VideoResourceUpdater::RecycleResource(
 431     base::WeakPtr<VideoResourceUpdater> updater,
 432     ResourceId resource_id,
 433     uint32 sync_point,
 434     bool lost_resource,
 435     BlockingTaskRunner* main_thread_task_runner) {
 436   if (!updater.get()) {
 437     // Resource was already deleted.
 438     return;
 439   }
 440
 441   const ResourceList::iterator resource_it = std::find_if(
 442       updater->all_resources_.begin(), updater->all_resources_.end(),
 443       [resource_id](const PlaneResource& plane_resource) {
 444         return plane_resource.resource_id == resource_id;
 445       });
 446   if (resource_it == updater->all_resources_.end())
 447     return;
 448
 449   ContextProvider* context_provider = updater->context_provider_;
 450   if (context_provider && sync_point) {
 451     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 452   }
 453
 454   if (lost_resource) {
 455     resource_it->ref_count = 0;
 456     updater->DeleteResource(resource_it);
 457     return;
 458   }
 459
 460   --resource_it->ref_count;
 461   DCHECK_GE(resource_it->ref_count, 0);
 462 }
 463
 464 }  // namespace cc