gfx/layers/ImageContainer.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef GFX_IMAGECONTAINER_H
   8 #define GFX_IMAGECONTAINER_H
   9
  10 #include <stdint.h>      // for int32_t, uint32_t, uint8_t, uint64_t
  11 #include "ImageTypes.h"  // for ImageFormat, etc
  12 #include "mozilla/AlreadyAddRefed.h"
  13 #include "mozilla/Assertions.h"      // for MOZ_ASSERT_HELPER2
  14 #include "mozilla/Mutex.h"           // for Mutex
  15 #include "mozilla/RecursiveMutex.h"  // for RecursiveMutex, etc
  16 #include "mozilla/ThreadSafeWeakPtr.h"
  17 #include "mozilla/TimeStamp.h"  // for TimeStamp
  18 #include "mozilla/gfx/Point.h"  // For IntSize
  19 #include "mozilla/gfx/Rect.h"
  20 #include "mozilla/gfx/Types.h"           // For ColorDepth
  21 #include "mozilla/layers/LayersTypes.h"  // for LayersBackend, etc
  22 #include "mozilla/layers/CompositorTypes.h"
  23 #include "mozilla/mozalloc.h"  // for operator delete, etc
  24 #include "nsDebug.h"           // for NS_ASSERTION
  25 #include "nsISupportsImpl.h"   // for Image::Release, etc
  26 #include "nsTArray.h"          // for nsTArray
  27 #include "nsThreadUtils.h"     // for NS_IsMainThread
  28 #include "mozilla/Atomics.h"
  29 #include "mozilla/gfx/2D.h"
  30 #include "mozilla/EnumeratedArray.h"
  31 #include "mozilla/UniquePtr.h"
  32 #include "MediaInfo.h"
  33 #include "nsTHashMap.h"
  34
  35 #ifdef XP_WIN
  36 struct ID3D10Texture2D;
  37 struct ID3D10Device;
  38 struct ID3D10ShaderResourceView;
  39 #endif
  40
  41 typedef void* HANDLE;
  42
  43 namespace mozilla {
  44
  45 namespace layers {
  46
  47 class GPUVideoImage;
  48 class ImageClient;
  49 class ImageCompositeNotification;
  50 class ImageContainer;
  51 class ImageContainerChild;
  52 class SharedPlanarYCbCrImage;
  53 class SurfaceDescriptor;
  54 class PlanarYCbCrImage;
  55 class TextureClient;
  56 class TextureClientRecycleAllocator;
  57 class KnowsCompositor;
  58 class NVImage;
  59 class MemoryOrShmem;
  60 #ifdef XP_WIN
  61 class D3D11RecycleAllocator;
  62 class D3D11YCbCrRecycleAllocator;
  63 #endif
  64 #ifdef XP_MACOSX
  65 class MacIOSurfaceRecycleAllocator;
  66 #endif
  67 class SurfaceDescriptorBuffer;
  68
  69 struct ImageBackendData {
  70   virtual ~ImageBackendData() = default;
  71
  72  protected:
  73   ImageBackendData() = default;
  74 };
  75
  76 /* Forward declarations for Image derivatives. */
  77 class GLImage;
  78 class SharedRGBImage;
  79 #ifdef MOZ_WIDGET_ANDROID
  80 class SurfaceTextureImage;
  81 #elif defined(XP_MACOSX)
  82 class MacIOSurfaceImage;
  83 #elif MOZ_WIDGET_GTK
  84 class DMABUFSurfaceImage;
  85 #endif
  86
  87 /**
  88  * A class representing a buffer of pixel data. The data can be in one
  89  * of various formats including YCbCr.
  90  *
  91  * Create an image using an ImageContainer. Fill the image with data, and
  92  * then call ImageContainer::SetImage to display it. An image must not be
  93  * modified after calling SetImage. Image implementations do not need to
  94  * perform locking; when filling an Image, the Image client is responsible
  95  * for ensuring only one thread accesses the Image at a time, and after
  96  * SetImage the image is immutable.
  97  *
  98  * When resampling an Image, only pixels within the buffer should be
  99  * sampled. For example, cairo images should be sampled in EXTEND_PAD mode.
 100  */
 101 class Image {
 102   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image)
 103
 104  public:
 105   ImageFormat GetFormat() const { return mFormat; }
 106   void* GetImplData() const { return mImplData; }
 107
 108   virtual gfx::IntSize GetSize() const = 0;
 109   virtual gfx::IntPoint GetOrigin() const { return gfx::IntPoint(0, 0); }
 110   virtual gfx::IntRect GetPictureRect() const {
 111     return gfx::IntRect(GetOrigin().x, GetOrigin().y, GetSize().width,
 112                         GetSize().height);
 113   }
 114   virtual gfx::ColorDepth GetColorDepth() const {
 115     return gfx::ColorDepth::COLOR_8;
 116   }
 117
 118   ImageBackendData* GetBackendData(LayersBackend aBackend) {
 119     return mBackendData[aBackend].get();
 120   }
 121   void SetBackendData(LayersBackend aBackend, ImageBackendData* aData) {
 122     mBackendData[aBackend] = mozilla::WrapUnique(aData);
 123   }
 124
 125   int32_t GetSerial() const { return mSerial; }
 126
 127   bool IsDRM() const { return mIsDRM; }
 128   virtual void SetIsDRM(bool aIsDRM) { mIsDRM = aIsDRM; }
 129
 130   virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() = 0;
 131
 132   virtual bool IsValid() const { return true; }
 133
 134   /**
 135    * For use with the TextureForwarder only (so that the later can
 136    * synchronize the TextureClient with the TextureHost).
 137    */
 138   virtual TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) {
 139     return nullptr;
 140   }
 141
 142   /* Access to derived classes. */
 143   virtual GLImage* AsGLImage() { return nullptr; }
 144   virtual GPUVideoImage* AsGPUVideoImage() { return nullptr; }
 145 #ifdef MOZ_WIDGET_ANDROID
 146   virtual SurfaceTextureImage* AsSurfaceTextureImage() { return nullptr; }
 147 #endif
 148 #ifdef XP_MACOSX
 149   virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; }
 150 #endif
 151   virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; }
 152 #ifdef MOZ_WIDGET_GTK
 153   virtual DMABUFSurfaceImage* AsDMABUFSurfaceImage() { return nullptr; }
 154 #endif
 155
 156   virtual NVImage* AsNVImage() { return nullptr; }
 157
 158   virtual Maybe<SurfaceDescriptor> GetDesc();
 159
 160  protected:
 161   Maybe<SurfaceDescriptor> GetDescFromTexClient(
 162       TextureClient* tcOverride = nullptr);
 163
 164   Image(void* aImplData, ImageFormat aFormat)
 165       : mImplData(aImplData),
 166         mSerial(++sSerialCounter),
 167         mFormat(aFormat),
 168         mIsDRM(false) {}
 169
 170   // Protected destructor, to discourage deletion outside of Release():
 171   virtual ~Image() = default;
 172
 173   mozilla::EnumeratedArray<mozilla::layers::LayersBackend,
 174                            mozilla::layers::LayersBackend::LAYERS_LAST,
 175                            UniquePtr<ImageBackendData>>
 176       mBackendData;
 177
 178   void* mImplData;
 179   int32_t mSerial;
 180   ImageFormat mFormat;
 181   bool mIsDRM;
 182
 183   static mozilla::Atomic<int32_t> sSerialCounter;
 184 };
 185
 186 /**
 187  * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
 188  * want to recycle from one image to the next.It's a separate object from
 189  * ImageContainer because images need to store a strong ref to their RecycleBin
 190  * and we must avoid creating a reference loop between an ImageContainer and
 191  * its active image.
 192  */
 193 class BufferRecycleBin final {
 194   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferRecycleBin)
 195
 196   // typedef mozilla::gl::GLContext GLContext;
 197
 198  public:
 199   BufferRecycleBin();
 200
 201   void RecycleBuffer(mozilla::UniquePtr<uint8_t[]> aBuffer, uint32_t aSize);
 202   // Returns a recycled buffer of the right size, or allocates a new buffer.
 203   mozilla::UniquePtr<uint8_t[]> GetBuffer(uint32_t aSize);
 204   virtual void ClearRecycledBuffers();
 205
 206  private:
 207   typedef mozilla::Mutex Mutex;
 208
 209   // Private destructor, to discourage deletion outside of Release():
 210   ~BufferRecycleBin() = default;
 211
 212   // This protects mRecycledBuffers, mRecycledBufferSize, mRecycledTextures
 213   // and mRecycledTextureSizes
 214   Mutex mLock;
 215
 216   // We should probably do something to prune this list on a timer so we don't
 217   // eat excess memory while video is paused...
 218   nsTArray<mozilla::UniquePtr<uint8_t[]>> mRecycledBuffers
 219       MOZ_GUARDED_BY(mLock);
 220   // This is only valid if mRecycledBuffers is non-empty
 221   uint32_t mRecycledBufferSize MOZ_GUARDED_BY(mLock);
 222 };
 223
 224 /**
 225  * A class that manages Image creation for a LayerManager. The only reason
 226  * we need a separate class here is that LayerManagers aren't threadsafe
 227  * (because layers can only be used on the main thread) and we want to
 228  * be able to create images from any thread, to facilitate video playback
 229  * without involving the main thread, for example.
 230  * Different layer managers can implement child classes of this making it
 231  * possible to create layer manager specific images.
 232  * This class is not meant to be used directly but rather can be set on an
 233  * image container. This is usually done by the layer system internally and
 234  * not explicitly by users. For PlanarYCbCr or Cairo images the default
 235  * implementation will creates images whose data lives in system memory, for
 236  * MacIOSurfaces the default implementation will be a simple MacIOSurface
 237  * wrapper.
 238  */
 239
 240 class ImageFactory {
 241   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory)
 242  protected:
 243   friend class ImageContainer;
 244
 245   ImageFactory() = default;
 246   virtual ~ImageFactory() = default;
 247
 248   virtual RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage(
 249       const gfx::IntSize& aScaleHint, BufferRecycleBin* aRecycleBin);
 250 };
 251
 252 // Used to notify ImageContainer::NotifyComposite()
 253 class ImageContainerListener final {
 254   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainerListener)
 255
 256  public:
 257   explicit ImageContainerListener(ImageContainer* aImageContainer);
 258
 259   void NotifyComposite(const ImageCompositeNotification& aNotification);
 260   void NotifyDropped(uint32_t aDropped);
 261   void ClearImageContainer();
 262   void DropImageClient();
 263
 264  private:
 265   typedef mozilla::Mutex Mutex;
 266
 267   ~ImageContainerListener();
 268
 269   Mutex mLock;
 270   ImageContainer* mImageContainer MOZ_GUARDED_BY(mLock);
 271 };
 272
 273 /**
 274  * A class that manages Images for an ImageLayer. The only reason
 275  * we need a separate class here is that ImageLayers aren't threadsafe
 276  * (because layers can only be used on the main thread) and we want to
 277  * be able to set the current Image from any thread, to facilitate
 278  * video playback without involving the main thread, for example.
 279  *
 280  * An ImageContainer can operate in one of these modes:
 281  * 1) Normal. Triggered by constructing the ImageContainer with
 282  * DISABLE_ASYNC or when compositing is happening on the main thread.
 283  * SetCurrentImages changes ImageContainer state but nothing is sent to the
 284  * compositor until the next layer transaction.
 285  * 2) Asynchronous. Initiated by constructing the ImageContainer with
 286  * ENABLE_ASYNC when compositing is happening on the main thread.
 287  * SetCurrentImages sends a message through the ImageBridge to the compositor
 288  * thread to update the image, without going through the main thread or
 289  * a layer transaction.
 290  * The ImageContainer uses a shared memory block containing a cross-process
 291  * mutex to communicate with the compositor thread. SetCurrentImage
 292  * synchronously updates the shared state to point to the new image and the old
 293  * image is immediately released (not true in Normal or Asynchronous modes).
 294  */
 295 class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
 296   friend class ImageContainerChild;
 297
 298  public:
 299   MOZ_DECLARE_REFCOUNTED_TYPENAME(ImageContainer)
 300
 301   enum Mode { SYNCHRONOUS = 0x0, ASYNCHRONOUS = 0x01 };
 302
 303   static const uint64_t sInvalidAsyncContainerId = 0;
 304
 305   explicit ImageContainer(ImageContainer::Mode flag = SYNCHRONOUS);
 306
 307   /**
 308    * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's
 309    * async container ID.
 310    * @param aAsyncContainerID async container ID for which we are a proxy
 311    */
 312   explicit ImageContainer(const CompositableHandle& aHandle);
 313
 314   ~ImageContainer();
 315
 316   typedef ContainerFrameID FrameID;
 317   typedef ContainerProducerID ProducerID;
 318
 319   RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage();
 320
 321   // Factory methods for shared image types.
 322   RefPtr<SharedRGBImage> CreateSharedRGBImage();
 323
 324   struct NonOwningImage {
 325     explicit NonOwningImage(Image* aImage = nullptr,
 326                             TimeStamp aTimeStamp = TimeStamp(),
 327                             FrameID aFrameID = 0, ProducerID aProducerID = 0)
 328         : mImage(aImage),
 329           mTimeStamp(aTimeStamp),
 330           mFrameID(aFrameID),
 331           mProducerID(aProducerID) {}
 332     Image* mImage;
 333     TimeStamp mTimeStamp;
 334     FrameID mFrameID;
 335     ProducerID mProducerID;
 336   };
 337   /**
 338    * Set aImages as the list of timestamped to display. The Images must have
 339    * been created by this ImageContainer.
 340    * Can be called on any thread. This method takes mRecursiveMutex
 341    * when accessing thread-shared state.
 342    * aImages must be non-empty. The first timestamp in the list may be
 343    * null but the others must not be, and the timestamps must increase.
 344    * Every element of aImages must have non-null mImage.
 345    * mFrameID can be zero, in which case you won't get meaningful
 346    * painted/dropped frame counts. Otherwise you should use a unique and
 347    * increasing ID for each decoded and submitted frame (but it's OK to
 348    * pass the same frame to SetCurrentImages).
 349    * mProducerID is a unique ID for the stream of images. A change in the
 350    * mProducerID means changing to a new mFrameID namespace. All frames in
 351    * aImages must have the same mProducerID.
 352    *
 353    * The Image data must not be modified after this method is called!
 354    * Note that this must not be called if ENABLE_ASYNC has not been set.
 355    *
 356    * The implementation calls CurrentImageChanged() while holding
 357    * mRecursiveMutex.
 358    *
 359    * If this ImageContainer has an ImageClient for async video:
 360    * Schedule a task to send the image to the compositor using the
 361    * PImageBridge protcol without using the main thread.
 362    */
 363   void SetCurrentImages(const nsTArray<NonOwningImage>& aImages);
 364
 365   /**
 366    * Clear all images. Let ImageClient release all TextureClients. Because we
 367    * may release the lock after acquiring it in this method, it cannot be called
 368    * with the lock held.
 369    */
 370   void ClearAllImages() MOZ_EXCLUDES(mRecursiveMutex);
 371
 372   /**
 373    * Clear any resources that are not immediately necessary. This may be called
 374    * in low-memory conditions.
 375    */
 376   void ClearCachedResources();
 377
 378   /**
 379    * Clear the current images.
 380    * This function is expect to be called only from a CompositableClient
 381    * that belongs to ImageBridgeChild. Created to prevent dead lock.
 382    * See Bug 901224.
 383    */
 384   void ClearImagesFromImageBridge();
 385
 386   /**
 387    * Set an Image as the current image to display. The Image must have
 388    * been created by this ImageContainer.
 389    * Must be called on the main thread, within a layers transaction.
 390    *
 391    * This method takes mRecursiveMutex
 392    * when accessing thread-shared state.
 393    * aImage can be null. While it's null, nothing will be painted.
 394    *
 395    * The Image data must not be modified after this method is called!
 396    * Note that this must not be called if ENABLE_ASYNC been set.
 397    *
 398    * You won't get meaningful painted/dropped counts when using this method.
 399    */
 400   void SetCurrentImageInTransaction(Image* aImage);
 401   void SetCurrentImagesInTransaction(const nsTArray<NonOwningImage>& aImages);
 402
 403   /**
 404    * Returns true if this ImageContainer uses the ImageBridge IPDL protocol.
 405    *
 406    * Can be called from any thread.
 407    */
 408   bool IsAsync() const;
 409
 410   /**
 411    * If this ImageContainer uses ImageBridge, returns the ID associated to
 412    * this container, for use in the ImageBridge protocol.
 413    * Returns 0 if this ImageContainer does not use ImageBridge. Note that
 414    * 0 is always an invalid ID for asynchronous image containers.
 415    *
 416    * Can be called from any thread.
 417    */
 418   CompositableHandle GetAsyncContainerHandle();
 419
 420   /**
 421    * Returns if the container currently has an image.
 422    * Can be called on any thread. This method takes mRecursiveMutex
 423    * when accessing thread-shared state.
 424    */
 425   bool HasCurrentImage();
 426
 427   struct OwningImage {
 428     OwningImage() : mFrameID(0), mProducerID(0), mComposited(false) {}
 429     RefPtr<Image> mImage;
 430     TimeStamp mTimeStamp;
 431     FrameID mFrameID;
 432     ProducerID mProducerID;
 433     bool mComposited;
 434   };
 435   /**
 436    * Copy the current Image list to aImages.
 437    * This has to add references since otherwise there are race conditions
 438    * where the current image is destroyed before the caller can add
 439    * a reference.
 440    * Can be called on any thread.
 441    * May return an empty list to indicate there is no current image.
 442    * If aGenerationCounter is non-null, sets *aGenerationCounter to a value
 443    * that's unique for this ImageContainer state.
 444    */
 445   void GetCurrentImages(nsTArray<OwningImage>* aImages,
 446                         uint32_t* aGenerationCounter = nullptr);
 447
 448   /**
 449    * Returns the size of the image in pixels.
 450    * Can be called on any thread. This method takes mRecursiveMutex when
 451    * accessing thread-shared state.
 452    */
 453   gfx::IntSize GetCurrentSize();
 454
 455   /**
 456    * Sets a size that the image is expected to be rendered at.
 457    * This is a hint for image backends to optimize scaling.
 458    * Default implementation in this class is to ignore the hint.
 459    * Can be called on any thread. This method takes mRecursiveMutex
 460    * when accessing thread-shared state.
 461    */
 462   void SetScaleHint(const gfx::IntSize& aScaleHint) {
 463     RecursiveMutexAutoLock lock(mRecursiveMutex);
 464     mScaleHint = aScaleHint;
 465   }
 466
 467   const gfx::IntSize GetScaleHint() const {
 468     RecursiveMutexAutoLock lock(mRecursiveMutex);
 469     return mScaleHint;
 470   }
 471
 472   void SetTransformHint(const gfx::Matrix& aTransformHint) {
 473     RecursiveMutexAutoLock lock(mRecursiveMutex);
 474     mTransformHint = aTransformHint;
 475   }
 476
 477   const gfx::Matrix GetTransformHint() const {
 478     RecursiveMutexAutoLock lock(mRecursiveMutex);
 479     return mTransformHint;
 480   }
 481
 482   void SetRotation(VideoInfo::Rotation aRotation) {
 483     MOZ_ASSERT(NS_IsMainThread());
 484     mRotation = aRotation;
 485   }
 486
 487   VideoInfo::Rotation GetRotation() const {
 488     MOZ_ASSERT(NS_IsMainThread());
 489     return mRotation;
 490   }
 491
 492   void SetImageFactory(ImageFactory* aFactory) {
 493     RecursiveMutexAutoLock lock(mRecursiveMutex);
 494     mImageFactory = aFactory ? aFactory : new ImageFactory();
 495   }
 496
 497   already_AddRefed<ImageFactory> GetImageFactory() const {
 498     RecursiveMutexAutoLock lock(mRecursiveMutex);
 499     return do_AddRef(mImageFactory);
 500   }
 501
 502   void EnsureRecycleAllocatorForRDD(KnowsCompositor* aKnowsCompositor);
 503
 504 #ifdef XP_WIN
 505   already_AddRefed<D3D11RecycleAllocator> GetD3D11RecycleAllocator(
 506       KnowsCompositor* aKnowsCompositor, gfx::SurfaceFormat aPreferredFormat);
 507   already_AddRefed<D3D11YCbCrRecycleAllocator> GetD3D11YCbCrRecycleAllocator(
 508       KnowsCompositor* aKnowsCompositor);
 509 #endif
 510
 511 #ifdef XP_MACOSX
 512   already_AddRefed<MacIOSurfaceRecycleAllocator>
 513   GetMacIOSurfaceRecycleAllocator();
 514 #endif
 515
 516   /**
 517    * Returns the delay between the last composited image's presentation
 518    * timestamp and when it was first composited. It's possible for the delay
 519    * to be negative if the first image in the list passed to SetCurrentImages
 520    * has a presentation timestamp greater than "now".
 521    * Returns 0 if the composited image had a null timestamp, or if no
 522    * image has been composited yet.
 523    */
 524   TimeDuration GetPaintDelay() {
 525     RecursiveMutexAutoLock lock(mRecursiveMutex);
 526     return mPaintDelay;
 527   }
 528
 529   /**
 530    * Returns the number of images which have been contained in this container
 531    * and painted at least once.  Can be called from any thread.
 532    */
 533   uint32_t GetPaintCount() {
 534     RecursiveMutexAutoLock lock(mRecursiveMutex);
 535     return mPaintCount;
 536   }
 537
 538   /**
 539    * An entry in the current image list "expires" when the entry has an
 540    * non-null timestamp, and in a SetCurrentImages call the new image list is
 541    * non-empty, the timestamp of the first new image is non-null and greater
 542    * than the timestamp associated with the image, and the first new image's
 543    * frameID is not the same as the entry's.
 544    * Every expired image that is never composited is counted as dropped.
 545    */
 546   uint32_t GetDroppedImageCount() { return mDroppedImageCount; }
 547
 548   void NotifyComposite(const ImageCompositeNotification& aNotification);
 549   void NotifyDropped(uint32_t aDropped);
 550
 551   already_AddRefed<ImageContainerListener> GetImageContainerListener() const;
 552
 553   /**
 554    * Get the ImageClient associated with this container. Returns only after
 555    * validating, and it will recreate the image client if that fails.
 556    * Returns nullptr if not applicable.
 557    */
 558   already_AddRefed<ImageClient> GetImageClient();
 559
 560   /**
 561    * Main thread only.
 562    */
 563   static ProducerID AllocateProducerID();
 564
 565   void DropImageClient();
 566
 567  private:
 568   typedef mozilla::RecursiveMutex RecursiveMutex;
 569
 570   void SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages);
 571
 572   // This is called to ensure we have an active image, this may not be true
 573   // when we're storing image information in a RemoteImageData structure.
 574   // NOTE: If we have remote data mRemoteDataMutex should be locked when
 575   // calling this function!
 576   void EnsureActiveImage();
 577
 578   void EnsureImageClient() MOZ_REQUIRES(mRecursiveMutex);
 579
 580   bool HasImageClient() const {
 581     RecursiveMutexAutoLock lock(mRecursiveMutex);
 582     return !!mImageClient;
 583   }
 584
 585   // RecursiveMutex to protect thread safe access to the "current
 586   // image", and any other state which is shared between threads.
 587   mutable RecursiveMutex mRecursiveMutex;
 588
 589   RefPtr<TextureClientRecycleAllocator> mRecycleAllocator
 590       MOZ_GUARDED_BY(mRecursiveMutex);
 591
 592 #ifdef XP_WIN
 593   RefPtr<D3D11RecycleAllocator> mD3D11RecycleAllocator
 594       MOZ_GUARDED_BY(mRecursiveMutex);
 595
 596   RefPtr<D3D11YCbCrRecycleAllocator> mD3D11YCbCrRecycleAllocator
 597       MOZ_GUARDED_BY(mRecursiveMutex);
 598 #endif
 599 #ifdef XP_MACOSX
 600   RefPtr<MacIOSurfaceRecycleAllocator> mMacIOSurfaceRecycleAllocator
 601       MOZ_GUARDED_BY(mRecursiveMutex);
 602 #endif
 603
 604   nsTArray<OwningImage> mCurrentImages MOZ_GUARDED_BY(mRecursiveMutex);
 605
 606   // Updates every time mActiveImage changes
 607   uint32_t mGenerationCounter MOZ_GUARDED_BY(mRecursiveMutex);
 608
 609   // Number of contained images that have been painted at least once.  It's up
 610   // to the ImageContainer implementation to ensure accesses to this are
 611   // threadsafe.
 612   uint32_t mPaintCount MOZ_GUARDED_BY(mRecursiveMutex);
 613
 614   // See GetPaintDelay. Accessed only with mRecursiveMutex held.
 615   TimeDuration mPaintDelay MOZ_GUARDED_BY(mRecursiveMutex);
 616
 617   // See GetDroppedImageCount.
 618   mozilla::Atomic<uint32_t> mDroppedImageCount;
 619
 620   // This is the image factory used by this container, layer managers using
 621   // this container can set an alternative image factory that will be used to
 622   // create images for this container.
 623   RefPtr<ImageFactory> mImageFactory MOZ_GUARDED_BY(mRecursiveMutex);
 624
 625   gfx::IntSize mScaleHint MOZ_GUARDED_BY(mRecursiveMutex);
 626
 627   gfx::Matrix mTransformHint MOZ_GUARDED_BY(mRecursiveMutex);
 628
 629   // Main thread only.
 630   VideoInfo::Rotation mRotation = VideoInfo::Rotation::kDegree_0;
 631
 632   RefPtr<BufferRecycleBin> mRecycleBin MOZ_GUARDED_BY(mRecursiveMutex);
 633
 634   // This member points to an ImageClient if this ImageContainer was
 635   // sucessfully created with ENABLE_ASYNC, or points to null otherwise.
 636   // 'unsuccessful' in this case only means that the ImageClient could not
 637   // be created, most likely because off-main-thread compositing is not enabled.
 638   // In this case the ImageContainer is perfectly usable, but it will forward
 639   // frames to the compositor through transactions in the main thread rather
 640   // than asynchronusly using the ImageBridge IPDL protocol.
 641   RefPtr<ImageClient> mImageClient MOZ_GUARDED_BY(mRecursiveMutex);
 642
 643   const bool mIsAsync;
 644   CompositableHandle mAsyncContainerHandle MOZ_GUARDED_BY(mRecursiveMutex);
 645
 646   // ProducerID for last current image(s)
 647   ProducerID mCurrentProducerID MOZ_GUARDED_BY(mRecursiveMutex);
 648
 649   RefPtr<ImageContainerListener> mNotifyCompositeListener;
 650
 651   static mozilla::Atomic<uint32_t> sGenerationCounter;
 652 };
 653
 654 class AutoLockImage {
 655  public:
 656   explicit AutoLockImage(ImageContainer* aContainer) {
 657     aContainer->GetCurrentImages(&mImages);
 658   }
 659
 660   bool HasImage() const { return !mImages.IsEmpty(); }
 661   Image* GetImage() const {
 662     return mImages.IsEmpty() ? nullptr : mImages[0].mImage.get();
 663   }
 664
 665   Image* GetImage(TimeStamp aTimeStamp) const {
 666     if (mImages.IsEmpty()) {
 667       return nullptr;
 668     }
 669
 670     MOZ_ASSERT(!aTimeStamp.IsNull());
 671     uint32_t chosenIndex = 0;
 672
 673     while (chosenIndex + 1 < mImages.Length() &&
 674            mImages[chosenIndex + 1].mTimeStamp <= aTimeStamp) {
 675       ++chosenIndex;
 676     }
 677
 678     return mImages[chosenIndex].mImage.get();
 679   }
 680
 681  private:
 682   AutoTArray<ImageContainer::OwningImage, 4> mImages;
 683 };
 684
 685 // This type is currently only used for AVIF and WebCodecs therefore makes some
 686 // specific assumptions (e.g., Alpha's bpc and stride is equal to Y's one)
 687 struct PlanarAlphaData {
 688   uint8_t* mChannel = nullptr;
 689   gfx::IntSize mSize = gfx::IntSize(0, 0);
 690   gfx::ColorDepth mDepth = gfx::ColorDepth::COLOR_8;
 691   bool mPremultiplied = false;
 692 };
 693 struct PlanarYCbCrData {
 694   // Luminance buffer
 695   uint8_t* mYChannel = nullptr;
 696   int32_t mYStride = 0;
 697   int32_t mYSkip = 0;
 698   // Chroma buffers
 699   uint8_t* mCbChannel = nullptr;
 700   uint8_t* mCrChannel = nullptr;
 701   int32_t mCbCrStride = 0;
 702   int32_t mCbSkip = 0;
 703   int32_t mCrSkip = 0;
 704   // Alpha buffer and its metadata
 705   Maybe<PlanarAlphaData> mAlpha = Nothing();
 706   // Picture region
 707   gfx::IntRect mPictureRect = gfx::IntRect(0, 0, 0, 0);
 708   StereoMode mStereoMode = StereoMode::MONO;
 709   gfx::ColorDepth mColorDepth = gfx::ColorDepth::COLOR_8;
 710   gfx::YUVColorSpace mYUVColorSpace = gfx::YUVColorSpace::Default;
 711   gfx::ColorSpace2 mColorPrimaries = gfx::ColorSpace2::UNKNOWN;
 712   gfx::TransferFunction mTransferFunction = gfx::TransferFunction::BT709;
 713   gfx::ColorRange mColorRange = gfx::ColorRange::LIMITED;
 714   gfx::ChromaSubsampling mChromaSubsampling = gfx::ChromaSubsampling::FULL;
 715
 716   // The cropped picture size of the Y channel.
 717   gfx::IntSize YPictureSize() const { return mPictureRect.Size(); }
 718
 719   // The cropped picture size of the Cb/Cr channels.
 720   gfx::IntSize CbCrPictureSize() const {
 721     return mCbCrStride > 0 ? gfx::ChromaSize(YPictureSize(), mChromaSubsampling)
 722                            : gfx::IntSize(0, 0);
 723   }
 724
 725   // The total uncropped size of data in the Y channel.
 726   gfx::IntSize YDataSize() const {
 727     return gfx::IntSize(mPictureRect.XMost(), mPictureRect.YMost());
 728   }
 729
 730   // The total uncropped size of data in the Cb/Cr channels.
 731   gfx::IntSize CbCrDataSize() const {
 732     return mCbCrStride > 0 ? gfx::ChromaSize(YDataSize(), mChromaSubsampling)
 733                            : gfx::IntSize(0, 0);
 734   }
 735
 736   static Maybe<PlanarYCbCrData> From(const SurfaceDescriptorBuffer&);
 737 };
 738
 739 /****** Image subtypes for the different formats ******/
 740
 741 /**
 742  * We assume that the image data is in the REC 470M color space (see
 743  * Theora specification, section 4.3.1).
 744  *
 745  * The YCbCr format can be:
 746  *
 747  * 4:4:4 - CbCr width/height are the same as Y.
 748  * 4:2:2 - CbCr width is half that of Y. Height is the same.
 749  * 4:2:0 - CbCr width and height is half that of Y.
 750  *
 751  * mChromaSubsampling specifies which YCbCr subsampling scheme to use.
 752  *
 753  * The Image that is rendered is the picture region defined by mPictureRect.
 754  *
 755  * mYSkip, mCbSkip, mCrSkip are added to support various output
 756  * formats from hardware decoder. They are per-pixel skips in the
 757  * source image.
 758  *
 759  * For example when image width is 640, mYStride is 670, mYSkip is 2,
 760  * the mYChannel buffer looks like:
 761  *
 762  * |<----------------------- mYStride ----------------------------->|
 763  * |<----------------- YDataSize().width ---------->|
 764  *  0   3   6   9   12  15  18  21                639             669
 765  * |----------------------------------------------------------------|
 766  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 767  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 768  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 769  * |            |<->|
 770  *                mYSkip
 771  */
 772 class PlanarYCbCrImage : public Image {
 773  public:
 774   typedef PlanarYCbCrData Data;
 775
 776   enum { MAX_DIMENSION = 16384 };
 777
 778   virtual ~PlanarYCbCrImage();
 779
 780   /**
 781    * This makes a copy of the data buffers, in order to support functioning
 782    * in all different layer managers.
 783    */
 784   virtual bool CopyData(const Data& aData) = 0;
 785
 786   /**
 787    * This doesn't make a copy of the data buffers.
 788    */
 789   virtual bool AdoptData(const Data& aData);
 790
 791   /**
 792    * This will create an empty data buffers according to the input data's size.
 793    */
 794   virtual bool CreateEmptyBuffer(const Data& aData, const gfx::IntSize& aYSize,
 795                                  const gfx::IntSize& aCbCrSize) {
 796     return false;
 797   }
 798   bool CreateEmptyBuffer(const Data& aData) {
 799     return CreateEmptyBuffer(aData, aData.YDataSize(), aData.CbCrDataSize());
 800   }
 801
 802   /**
 803    * Grab the original YUV data. This is optional.
 804    */
 805   virtual const Data* GetData() const { return &mData; }
 806
 807   /**
 808    * Return the number of bytes of heap memory used to store this image.
 809    */
 810   uint32_t GetDataSize() const { return mBufferSize; }
 811
 812   bool IsValid() const override { return !!mBufferSize; }
 813
 814   gfx::IntSize GetSize() const override { return mSize; }
 815
 816   gfx::IntPoint GetOrigin() const override { return mOrigin; }
 817
 818   PlanarYCbCrImage();
 819
 820   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
 821     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
 822   }
 823
 824   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const = 0;
 825
 826   PlanarYCbCrImage* AsPlanarYCbCrImage() override { return this; }
 827
 828   /**
 829    * Build a SurfaceDescriptorBuffer with this image.  A function to allocate
 830    * a MemoryOrShmem with the given capacity must be provided.
 831    */
 832   virtual nsresult BuildSurfaceDescriptorBuffer(
 833       SurfaceDescriptorBuffer& aSdBuffer,
 834       const std::function<MemoryOrShmem(uint32_t)>& aAllocate);
 835
 836  protected:
 837   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
 838
 839   void SetOffscreenFormat(gfxImageFormat aFormat) {
 840     mOffscreenFormat = aFormat;
 841   }
 842   gfxImageFormat GetOffscreenFormat() const;
 843
 844   Data mData;
 845   gfx::IntPoint mOrigin;
 846   gfx::IntSize mSize;
 847   gfxImageFormat mOffscreenFormat;
 848   RefPtr<gfx::SourceSurface> mSourceSurface;
 849   uint32_t mBufferSize;
 850 };
 851
 852 class RecyclingPlanarYCbCrImage : public PlanarYCbCrImage {
 853  public:
 854   explicit RecyclingPlanarYCbCrImage(BufferRecycleBin* aRecycleBin)
 855       : mRecycleBin(aRecycleBin) {}
 856   virtual ~RecyclingPlanarYCbCrImage();
 857   bool CopyData(const Data& aData) override;
 858   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
 859
 860  protected:
 861   /**
 862    * Return a buffer to store image data in.
 863    */
 864   mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
 865
 866   RefPtr<BufferRecycleBin> mRecycleBin;
 867   mozilla::UniquePtr<uint8_t[]> mBuffer;
 868 };
 869
 870 /**
 871  * NVImage is used to store YUV420SP_NV12 and YUV420SP_NV21 data natively, which
 872  * are not supported by PlanarYCbCrImage. (PlanarYCbCrImage only stores YUV444P,
 873  * YUV422P and YUV420P, it converts YUV420SP_NV12 and YUV420SP_NV21 data into
 874  * YUV420P in its PlanarYCbCrImage::SetData() method.)
 875  *
 876  * PlanarYCbCrData is able to express all the YUV family and so we keep use it
 877  * in NVImage.
 878  */
 879 class NVImage final : public Image {
 880   typedef PlanarYCbCrData Data;
 881
 882  public:
 883   NVImage();
 884   virtual ~NVImage();
 885
 886   // Methods inherited from layers::Image.
 887   gfx::IntSize GetSize() const override;
 888   gfx::IntRect GetPictureRect() const override;
 889   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
 890   bool IsValid() const override;
 891   NVImage* AsNVImage() override;
 892
 893   // Methods mimic layers::PlanarYCbCrImage.
 894   bool SetData(const Data& aData);
 895   const Data* GetData() const;
 896   uint32_t GetBufferSize() const;
 897
 898  protected:
 899   /**
 900    * Return a buffer to store image data in.
 901    */
 902   mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
 903
 904   mozilla::UniquePtr<uint8_t[]> mBuffer;
 905   uint32_t mBufferSize;
 906   gfx::IntSize mSize;
 907   Data mData;
 908   RefPtr<gfx::SourceSurface> mSourceSurface;
 909 };
 910
 911 /**
 912  * Currently, the data in a SourceSurfaceImage surface is treated as being in
 913  * the device output color space. This class is very simple as all backends have
 914  * to know about how to deal with drawing a cairo image.
 915  */
 916 class SourceSurfaceImage final : public Image {
 917  public:
 918   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override {
 919     RefPtr<gfx::SourceSurface> surface(mSourceSurface);
 920     return surface.forget();
 921   }
 922
 923   void SetTextureFlags(TextureFlags aTextureFlags) {
 924     mTextureFlags = aTextureFlags;
 925   }
 926   TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) override;
 927
 928   gfx::IntSize GetSize() const override { return mSize; }
 929
 930   SourceSurfaceImage(const gfx::IntSize& aSize,
 931                      gfx::SourceSurface* aSourceSurface);
 932   explicit SourceSurfaceImage(gfx::SourceSurface* aSourceSurface);
 933   virtual ~SourceSurfaceImage();
 934
 935  private:
 936   gfx::IntSize mSize;
 937   RefPtr<gfx::SourceSurface> mSourceSurface;
 938   nsTHashMap<uint32_t, RefPtr<TextureClient>> mTextureClients;
 939   TextureFlags mTextureFlags;
 940 };
 941
 942 }  // namespace layers
 943 }  // namespace mozilla
 944
 945 #endif