media/base/video_frame.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
   6 #define MEDIA_BASE_VIDEO_FRAME_H_
   7
   8 #include <vector>
   9
  10 #include "base/callback.h"
  11 #include "base/md5.h"
  12 #include "base/memory/shared_memory.h"
  13 #include "base/synchronization/lock.h"
  14 #include "media/base/buffers.h"
  15 #include "ui/gfx/rect.h"
  16 #include "ui/gfx/size.h"
  17
  18 #if defined(OS_MACOSX)
  19 #include <CoreVideo/CVPixelBuffer.h>
  20 #include "base/mac/scoped_cftyperef.h"
  21 #endif
  22
  23 class SkBitmap;
  24
  25 namespace gpu {
  26 struct MailboxHolder;
  27 }  // namespace gpu
  28
  29 namespace media {
  30
  31 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  32  public:
  33   enum {
  34     kFrameSizeAlignment = 16,
  35     kFrameSizePadding = 16,
  36     kFrameAddressAlignment = 32
  37   };
  38
  39   enum {
  40     kMaxPlanes = 4,
  41
  42     kYPlane = 0,
  43     kUPlane = 1,
  44     kUVPlane = kUPlane,
  45     kVPlane = 2,
  46     kAPlane = 3,
  47   };
  48
  49   // Surface formats roughly based on FOURCC labels, see:
  50   // http://www.fourcc.org/rgb.php
  51   // http://www.fourcc.org/yuv.php
  52   // Logged to UMA, so never reuse values.
  53   enum Format {
  54     UNKNOWN = 0,  // Unknown format value.
  55     YV12 = 1,  // 12bpp YVU planar 1x1 Y, 2x2 VU samples
  56     YV16 = 2,  // 16bpp YVU planar 1x1 Y, 2x1 VU samples
  57     I420 = 3,  // 12bpp YVU planar 1x1 Y, 2x2 UV samples.
  58     YV12A = 4,  // 20bpp YUVA planar 1x1 Y, 2x2 VU, 1x1 A samples.
  59 #if defined(VIDEO_HOLE)
  60     HOLE = 5,  // Hole frame.
  61 #endif  // defined(VIDEO_HOLE)
  62     NATIVE_TEXTURE = 6,  // Native texture.  Pixel-format agnostic.
  63     YV12J = 7,  // JPEG color range version of YV12
  64     NV12 = 8,  // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane.
  65     YV24 = 9,  // 24bpp YUV planar, no subsampling.
  66     FORMAT_MAX = YV24,  // Must always be equal to largest entry logged.
  67   };
  68
  69   // Returns the name of a Format as a string.
  70   static std::string FormatToString(Format format);
  71
  72   // Creates a new frame in system memory with given parameters. Buffers for
  73   // the frame are allocated but not initialized.
  74   static scoped_refptr<VideoFrame> CreateFrame(
  75       Format format,
  76       const gfx::Size& coded_size,
  77       const gfx::Rect& visible_rect,
  78       const gfx::Size& natural_size,
  79       base::TimeDelta timestamp);
  80
  81   // Call prior to CreateFrame to ensure validity of frame configuration. Called
  82   // automatically by VideoDecoderConfig::IsValidConfig().
  83   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
  84   static bool IsValidConfig(Format format, const gfx::Size& coded_size,
  85                             const gfx::Rect& visible_rect,
  86                             const gfx::Size& natural_size);
  87
  88   // CB to write pixels from the texture backing this frame into the
  89   // |const SkBitmap&| parameter.
  90   typedef base::Callback<void(const SkBitmap&)> ReadPixelsCB;
  91
  92   // CB to be called on the mailbox backing this frame when the frame is
  93   // destroyed.
  94   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
  95
  96   // Wraps a native texture of the given parameters with a VideoFrame.  The
  97   // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|,
  98   // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the
  99   // argument when the VideoFrame is to be destroyed.
 100   // |read_pixels_cb| may be used to do (slow!) readbacks from the
 101   // texture to main memory.
 102   static scoped_refptr<VideoFrame> WrapNativeTexture(
 103       scoped_ptr<gpu::MailboxHolder> mailbox_holder,
 104       const ReleaseMailboxCB& mailbox_holder_release_cb,
 105       const gfx::Size& coded_size,
 106       const gfx::Rect& visible_rect,
 107       const gfx::Size& natural_size,
 108       base::TimeDelta timestamp,
 109       const ReadPixelsCB& read_pixels_cb);
 110
 111   // Read pixels from the native texture backing |*this| and write
 112   // them to |pixels| as BGRA.  |pixels| must point to a buffer at
 113   // least as large as 4 * visible_rect().size().GetArea().
 114   void ReadPixelsFromNativeTexture(const SkBitmap& pixels);
 115
 116   // Wraps packed image data residing in a memory buffer with a VideoFrame.
 117   // The image data resides in |data| and is assumed to be packed tightly in a
 118   // buffer of logical dimensions |coded_size| with the appropriate bit depth
 119   // and plane count as given by |format|.  The shared memory handle of the
 120   // backing allocation, if present, can be passed in with |handle|.  When the
 121   // frame is destroyed, |no_longer_needed_cb.Run()| will be called.
 122   // Returns NULL on failure.
 123   static scoped_refptr<VideoFrame> WrapExternalPackedMemory(
 124       Format format,
 125       const gfx::Size& coded_size,
 126       const gfx::Rect& visible_rect,
 127       const gfx::Size& natural_size,
 128       uint8* data,
 129       size_t data_size,
 130       base::SharedMemoryHandle handle,
 131       base::TimeDelta timestamp,
 132       const base::Closure& no_longer_needed_cb);
 133
 134 #if defined(OS_POSIX)
 135   // Wraps provided dmabufs
 136   // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a
 137   // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame
 138   // retains a reference to them, and are automatically close()d on destruction,
 139   // dropping the reference. The caller may safely close() its reference after
 140   // calling WrapExternalDmabufs().
 141   // The image data is only accessible via dmabuf fds, which are usually passed
 142   // directly to a hardware device and/or to another process, or can also be
 143   // mapped via mmap() for CPU access.
 144   // When the frame is destroyed, |no_longer_needed_cb.Run()| will be called.
 145   // Returns NULL on failure.
 146   static scoped_refptr<VideoFrame> WrapExternalDmabufs(
 147       Format format,
 148       const gfx::Size& coded_size,
 149       const gfx::Rect& visible_rect,
 150       const gfx::Size& natural_size,
 151       const std::vector<int> dmabuf_fds,
 152       base::TimeDelta timestamp,
 153       const base::Closure& no_longer_needed_cb);
 154 #endif
 155
 156 #if defined(OS_MACOSX)
 157   // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
 158   // retained for the lifetime of the VideoFrame and released upon destruction.
 159   // The image data is only accessible via the pixel buffer, which could be
 160   // backed by an IOSurface from another process. All the attributes of the
 161   // VideoFrame are derived from the pixel buffer, with the exception of the
 162   // timestamp. If information is missing or is incompatible (for example, a
 163   // pixel format that has no VideoFrame match), NULL is returned.
 164   // http://crbug.com/401308
 165   static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
 166       CVPixelBufferRef cv_pixel_buffer,
 167       base::TimeDelta timestamp);
 168 #endif
 169
 170   // Wraps external YUV data of the given parameters with a VideoFrame.
 171   // The returned VideoFrame does not own the data passed in. When the frame
 172   // is destroyed |no_longer_needed_cb.Run()| will be called.
 173   // TODO(sheu): merge this into WrapExternalSharedMemory().
 174   // http://crbug.com/270217
 175   static scoped_refptr<VideoFrame> WrapExternalYuvData(
 176       Format format,
 177       const gfx::Size& coded_size,
 178       const gfx::Rect& visible_rect,
 179       const gfx::Size& natural_size,
 180       int32 y_stride,
 181       int32 u_stride,
 182       int32 v_stride,
 183       uint8* y_data,
 184       uint8* u_data,
 185       uint8* v_data,
 186       base::TimeDelta timestamp,
 187       const base::Closure& no_longer_needed_cb);
 188
 189   // Wraps |frame| and calls |no_longer_needed_cb| when the wrapper VideoFrame
 190   // gets destroyed. |visible_rect| must be a sub rect within
 191   // frame->visible_rect().
 192   static scoped_refptr<VideoFrame> WrapVideoFrame(
 193       const scoped_refptr<VideoFrame>& frame,
 194       const gfx::Rect& visible_rect,
 195       const gfx::Size& natural_size,
 196       const base::Closure& no_longer_needed_cb);
 197
 198   // Creates a frame which indicates end-of-stream.
 199   static scoped_refptr<VideoFrame> CreateEOSFrame();
 200
 201   // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
 202   static scoped_refptr<VideoFrame> CreateColorFrame(
 203       const gfx::Size& size,
 204       uint8 y, uint8 u, uint8 v,
 205       base::TimeDelta timestamp);
 206
 207   // Allocates YV12 frame based on |size|, and sets its data to the YUV
 208   // equivalent of RGB(0,0,0).
 209   static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
 210
 211   // Allocates YV12A frame based on |size|, and sets its data to the YUVA
 212   // equivalent of RGBA(0,0,0,0).
 213   static scoped_refptr<VideoFrame> CreateTransparentFrame(
 214       const gfx::Size& size);
 215
 216 #if defined(VIDEO_HOLE)
 217   // Allocates a hole frame.
 218   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 219 #endif  // defined(VIDEO_HOLE)
 220
 221   static size_t NumPlanes(Format format);
 222
 223   // Returns the required allocation size for a (tightly packed) frame of the
 224   // given coded size and format.
 225   static size_t AllocationSize(Format format, const gfx::Size& coded_size);
 226
 227   // Returns the plane size for a plane of the given coded size and format.
 228   static gfx::Size PlaneSize(Format format,
 229                              size_t plane,
 230                              const gfx::Size& coded_size);
 231
 232   // Returns the required allocation size for a (tightly packed) plane of the
 233   // given coded size and format.
 234   static size_t PlaneAllocationSize(Format format,
 235                                     size_t plane,
 236                                     const gfx::Size& coded_size);
 237
 238   // Returns horizontal bits per pixel for given |plane| and |format|.
 239   static int PlaneHorizontalBitsPerPixel(Format format, size_t plane);
 240
 241   // Returns the number of bytes per row for the given plane, format, and width.
 242   // The width may be aligned to format requirements.
 243   static int RowBytes(size_t plane, Format format, int width);
 244
 245   Format format() const { return format_; }
 246
 247   const gfx::Size& coded_size() const { return coded_size_; }
 248   const gfx::Rect& visible_rect() const { return visible_rect_; }
 249   const gfx::Size& natural_size() const { return natural_size_; }
 250
 251   int stride(size_t plane) const;
 252
 253   // Returns the number of bytes per row and number of rows for a given plane.
 254   //
 255   // As opposed to stride(), row_bytes() refers to the bytes representing
 256   // frame data scanlines (coded_size.width() pixels, without stride padding).
 257   int row_bytes(size_t plane) const;
 258   int rows(size_t plane) const;
 259
 260   // Returns pointer to the buffer for a given plane. The memory is owned by
 261   // VideoFrame object and must not be freed by the caller.
 262   uint8* data(size_t plane) const;
 263
 264   // Returns the mailbox holder of the native texture wrapped by this frame.
 265   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
 266   // mailbox, the caller must wait for the included sync point.
 267   const gpu::MailboxHolder* mailbox_holder() const;
 268
 269   // Returns the shared-memory handle, if present
 270   base::SharedMemoryHandle shared_memory_handle() const;
 271
 272 #if defined(OS_POSIX)
 273   // Returns backing dmabuf file descriptor for given |plane|, if present.
 274   int dmabuf_fd(size_t plane) const;
 275 #endif
 276
 277 #if defined(OS_MACOSX)
 278   // Returns the backing CVPixelBuffer, if present.
 279   CVPixelBufferRef cv_pixel_buffer() const;
 280 #endif
 281
 282   // Returns true if this VideoFrame represents the end of the stream.
 283   bool end_of_stream() const { return end_of_stream_; }
 284
 285   base::TimeDelta timestamp() const {
 286     return timestamp_;
 287   }
 288   void set_timestamp(const base::TimeDelta& timestamp) {
 289     timestamp_ = timestamp;
 290   }
 291
 292   class SyncPointClient {
 293    public:
 294     SyncPointClient() {}
 295     virtual uint32 InsertSyncPoint() = 0;
 296     virtual void WaitSyncPoint(uint32 sync_point) = 0;
 297
 298    protected:
 299     virtual ~SyncPointClient() {}
 300
 301     DISALLOW_COPY_AND_ASSIGN(SyncPointClient);
 302   };
 303   // It uses |client| to insert a new sync point and potentially waits on a
 304   // older sync point. The final sync point will be used to release this
 305   // VideoFrame.
 306   // This method is thread safe. Both blink and compositor threads can call it.
 307   void UpdateReleaseSyncPoint(SyncPointClient* client);
 308
 309   // Used to keep a running hash of seen frames.  Expects an initialized MD5
 310   // context.  Calls MD5Update with the context and the contents of the frame.
 311   void HashFrameForTesting(base::MD5Context* context);
 312
 313  private:
 314   friend class base::RefCountedThreadSafe<VideoFrame>;
 315
 316   // Returns true if |plane| is a valid plane number for the given format. This
 317   // can be used to DCHECK() plane parameters.
 318   static bool IsValidPlane(size_t plane, VideoFrame::Format format);
 319
 320   // Clients must use the static CreateFrame() method to create a new frame.
 321   VideoFrame(Format format,
 322              const gfx::Size& coded_size,
 323              const gfx::Rect& visible_rect,
 324              const gfx::Size& natural_size,
 325              scoped_ptr<gpu::MailboxHolder> mailbox_holder,
 326              base::TimeDelta timestamp,
 327              bool end_of_stream);
 328   virtual ~VideoFrame();
 329
 330   void AllocateYUV();
 331
 332   // Frame format.
 333   const Format format_;
 334
 335   // Width and height of the video frame, in pixels. This must include pixel
 336   // data for the whole image; i.e. for YUV formats with subsampled chroma
 337   // planes, in the case that the visible portion of the image does not line up
 338   // on a sample boundary, |coded_size_| must be rounded up appropriately and
 339   // the pixel data provided for the odd pixels.
 340   const gfx::Size coded_size_;
 341
 342   // Width, height, and offsets of the visible portion of the video frame. Must
 343   // be a subrect of |coded_size_|. Can be odd with respect to the sample
 344   // boundaries, e.g. for formats with subsampled chroma.
 345   const gfx::Rect visible_rect_;
 346
 347   // Width and height of the visible portion of the video frame
 348   // (|visible_rect_.size()|) with aspect ratio taken into account.
 349   const gfx::Size natural_size_;
 350
 351   // Array of strides for each plane, typically greater or equal to the width
 352   // of the surface divided by the horizontal sampling period.  Note that
 353   // strides can be negative.
 354   int32 strides_[kMaxPlanes];
 355
 356   // Array of data pointers to each plane.
 357   uint8* data_[kMaxPlanes];
 358
 359   // Native texture mailbox, if this is a NATIVE_TEXTURE frame.
 360   const scoped_ptr<gpu::MailboxHolder> mailbox_holder_;
 361   ReleaseMailboxCB mailbox_holder_release_cb_;
 362   ReadPixelsCB read_pixels_cb_;
 363
 364   // Shared memory handle, if this frame was allocated from shared memory.
 365   base::SharedMemoryHandle shared_memory_handle_;
 366
 367 #if defined(OS_POSIX)
 368   // Dmabufs for each plane, if this frame is wrapping memory
 369   // acquired via dmabuf.
 370   base::ScopedFD dmabuf_fds_[kMaxPlanes];
 371 #endif
 372
 373 #if defined(OS_MACOSX)
 374   // CVPixelBuffer, if this frame is wrapping one.
 375   base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
 376 #endif
 377
 378   base::Closure no_longer_needed_cb_;
 379
 380   base::TimeDelta timestamp_;
 381
 382   base::Lock release_sync_point_lock_;
 383   uint32 release_sync_point_;
 384
 385   const bool end_of_stream_;
 386
 387   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 388 };
 389
 390 }  // namespace media
 391
 392 #endif  // MEDIA_BASE_VIDEO_FRAME_H_