gfx/layers/ImageContainer.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef GFX_IMAGECONTAINER_H
   8 #define GFX_IMAGECONTAINER_H
   9
  10 #include <stdint.h>      // for int32_t, uint32_t, uint8_t, uint64_t
  11 #include "ImageTypes.h"  // for ImageFormat, etc
  12 #include "mozilla/AlreadyAddRefed.h"
  13 #include "mozilla/Assertions.h"      // for MOZ_ASSERT_HELPER2
  14 #include "mozilla/Mutex.h"           // for Mutex
  15 #include "mozilla/RecursiveMutex.h"  // for RecursiveMutex, etc
  16 #include "mozilla/ThreadSafeWeakPtr.h"
  17 #include "mozilla/TimeStamp.h"  // for TimeStamp
  18 #include "mozilla/gfx/Point.h"  // For IntSize
  19 #include "mozilla/gfx/Rect.h"
  20 #include "mozilla/gfx/Types.h"           // For ColorDepth
  21 #include "mozilla/layers/LayersTypes.h"  // for LayersBackend, etc
  22 #include "mozilla/layers/CompositorTypes.h"
  23 #include "mozilla/mozalloc.h"  // for operator delete, etc
  24 #include "mozilla/TypedEnumBits.h"
  25 #include "nsDebug.h"          // for NS_ASSERTION
  26 #include "nsISupportsImpl.h"  // for Image::Release, etc
  27 #include "nsTArray.h"         // for nsTArray
  28 #include "nsThreadUtils.h"    // for NS_IsMainThread
  29 #include "mozilla/Atomics.h"
  30 #include "mozilla/gfx/2D.h"
  31 #include "mozilla/EnumeratedArray.h"
  32 #include "mozilla/UniquePtr.h"
  33 #include "nsTHashMap.h"
  34
  35 #ifdef XP_WIN
  36 struct ID3D10Texture2D;
  37 struct ID3D10Device;
  38 struct ID3D10ShaderResourceView;
  39 #endif
  40
  41 typedef void* HANDLE;
  42
  43 namespace mozilla {
  44
  45 enum class VideoRotation;
  46
  47 namespace layers {
  48
  49 class GPUVideoImage;
  50 class ImageClient;
  51 class ImageCompositeNotification;
  52 class ImageContainer;
  53 class ImageContainerChild;
  54 class SharedPlanarYCbCrImage;
  55 class SurfaceDescriptor;
  56 class PlanarYCbCrImage;
  57 class TextureClient;
  58 class TextureClientRecycleAllocator;
  59 class KnowsCompositor;
  60 class NVImage;
  61 class MemoryOrShmem;
  62 #ifdef XP_WIN
  63 class D3D11RecycleAllocator;
  64 class D3D11YCbCrRecycleAllocator;
  65 #endif
  66 #ifdef XP_DARWIN
  67 class MacIOSurfaceRecycleAllocator;
  68 #endif
  69 class SurfaceDescriptorBuffer;
  70
  71 struct ImageBackendData {
  72   virtual ~ImageBackendData() = default;
  73
  74  protected:
  75   ImageBackendData() = default;
  76 };
  77
  78 /* Forward declarations for Image derivatives. */
  79 class GLImage;
  80 class SharedRGBImage;
  81 #ifdef MOZ_WIDGET_ANDROID
  82 class SurfaceTextureImage;
  83 #elif defined(XP_DARWIN)
  84 class MacIOSurfaceImage;
  85 #elif MOZ_WIDGET_GTK
  86 class DMABUFSurfaceImage;
  87 #endif
  88
  89 /**
  90  * A class representing a buffer of pixel data. The data can be in one
  91  * of various formats including YCbCr.
  92  *
  93  * Create an image using an ImageContainer. Fill the image with data, and
  94  * then call ImageContainer::SetImage to display it. An image must not be
  95  * modified after calling SetImage. Image implementations do not need to
  96  * perform locking; when filling an Image, the Image client is responsible
  97  * for ensuring only one thread accesses the Image at a time, and after
  98  * SetImage the image is immutable.
  99  *
 100  * When resampling an Image, only pixels within the buffer should be
 101  * sampled. For example, cairo images should be sampled in EXTEND_PAD mode.
 102  */
 103 class Image {
 104   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image)
 105
 106  public:
 107   ImageFormat GetFormat() const { return mFormat; }
 108   void* GetImplData() const { return mImplData; }
 109
 110   virtual gfx::IntSize GetSize() const = 0;
 111   virtual gfx::IntPoint GetOrigin() const { return gfx::IntPoint(0, 0); }
 112   virtual gfx::IntRect GetPictureRect() const {
 113     return gfx::IntRect(GetOrigin().x, GetOrigin().y, GetSize().width,
 114                         GetSize().height);
 115   }
 116   virtual gfx::ColorDepth GetColorDepth() const {
 117     return gfx::ColorDepth::COLOR_8;
 118   }
 119
 120   ImageBackendData* GetBackendData(LayersBackend aBackend) {
 121     return mBackendData[aBackend].get();
 122   }
 123   void SetBackendData(LayersBackend aBackend, ImageBackendData* aData) {
 124     mBackendData[aBackend] = mozilla::WrapUnique(aData);
 125   }
 126
 127   int32_t GetSerial() const { return mSerial; }
 128
 129   bool IsDRM() const { return mIsDRM; }
 130   virtual void SetIsDRM(bool aIsDRM) { mIsDRM = aIsDRM; }
 131
 132   virtual void OnPrepareForwardToHost() {}
 133   virtual void OnAbandonForwardToHost() {}
 134
 135   virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() = 0;
 136
 137   enum class BuildSdbFlags : uint8_t {
 138     Default = 0,
 139     RgbOnly = 1 << 0,
 140   };
 141
 142   virtual nsresult BuildSurfaceDescriptorBuffer(
 143       SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
 144       const std::function<MemoryOrShmem(uint32_t)>& aAllocate);
 145
 146   virtual bool IsValid() const { return true; }
 147
 148   /**
 149    * For use with the TextureForwarder only (so that the later can
 150    * synchronize the TextureClient with the TextureHost).
 151    */
 152   virtual TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) {
 153     return nullptr;
 154   }
 155
 156   /* Access to derived classes. */
 157   virtual GLImage* AsGLImage() { return nullptr; }
 158   virtual GPUVideoImage* AsGPUVideoImage() { return nullptr; }
 159 #ifdef MOZ_WIDGET_ANDROID
 160   virtual SurfaceTextureImage* AsSurfaceTextureImage() { return nullptr; }
 161 #endif
 162 #ifdef XP_DARWIN
 163   virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; }
 164 #endif
 165   virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; }
 166 #ifdef MOZ_WIDGET_GTK
 167   virtual DMABUFSurfaceImage* AsDMABUFSurfaceImage() { return nullptr; }
 168 #endif
 169
 170   virtual NVImage* AsNVImage() { return nullptr; }
 171
 172   virtual Maybe<SurfaceDescriptor> GetDesc();
 173
 174   static nsresult AllocateSurfaceDescriptorBufferRgb(
 175       const gfx::IntSize& aSize, gfx::SurfaceFormat aFormat,
 176       uint8_t*& aOutBuffer, SurfaceDescriptorBuffer& aSdBuffer,
 177       int32_t& aStride,
 178       const std::function<layers::MemoryOrShmem(uint32_t)>& aAllocate);
 179
 180  protected:
 181   Maybe<SurfaceDescriptor> GetDescFromTexClient(
 182       TextureClient* tcOverride = nullptr);
 183
 184   Image(void* aImplData, ImageFormat aFormat)
 185       : mImplData(aImplData),
 186         mSerial(++sSerialCounter),
 187         mFormat(aFormat),
 188         mIsDRM(false) {}
 189
 190   // Protected destructor, to discourage deletion outside of Release():
 191   virtual ~Image() = default;
 192
 193   mozilla::EnumeratedArray<mozilla::layers::LayersBackend,
 194                            UniquePtr<ImageBackendData>,
 195                            size_t(mozilla::layers::LayersBackend::LAYERS_LAST)>
 196       mBackendData;
 197
 198   void* mImplData;
 199   int32_t mSerial;
 200   ImageFormat mFormat;
 201   bool mIsDRM;
 202
 203   static mozilla::Atomic<int32_t> sSerialCounter;
 204 };
 205
 206 MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(Image::BuildSdbFlags)
 207
 208 /**
 209  * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
 210  * want to recycle from one image to the next.It's a separate object from
 211  * ImageContainer because images need to store a strong ref to their RecycleBin
 212  * and we must avoid creating a reference loop between an ImageContainer and
 213  * its active image.
 214  */
 215 class BufferRecycleBin final {
 216   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferRecycleBin)
 217
 218   // typedef mozilla::gl::GLContext GLContext;
 219
 220  public:
 221   BufferRecycleBin();
 222
 223   void RecycleBuffer(mozilla::UniquePtr<uint8_t[]> aBuffer, uint32_t aSize);
 224   // Returns a recycled buffer of the right size, or allocates a new buffer.
 225   mozilla::UniquePtr<uint8_t[]> GetBuffer(uint32_t aSize);
 226   virtual void ClearRecycledBuffers();
 227
 228  private:
 229   typedef mozilla::Mutex Mutex;
 230
 231   // Private destructor, to discourage deletion outside of Release():
 232   ~BufferRecycleBin() = default;
 233
 234   // This protects mRecycledBuffers, mRecycledBufferSize, mRecycledTextures
 235   // and mRecycledTextureSizes
 236   Mutex mLock;
 237
 238   // We should probably do something to prune this list on a timer so we don't
 239   // eat excess memory while video is paused...
 240   nsTArray<mozilla::UniquePtr<uint8_t[]>> mRecycledBuffers
 241       MOZ_GUARDED_BY(mLock);
 242   // This is only valid if mRecycledBuffers is non-empty
 243   uint32_t mRecycledBufferSize MOZ_GUARDED_BY(mLock);
 244 };
 245
 246 /**
 247  * A class that manages Image creation for a LayerManager. The only reason
 248  * we need a separate class here is that LayerManagers aren't threadsafe
 249  * (because layers can only be used on the main thread) and we want to
 250  * be able to create images from any thread, to facilitate video playback
 251  * without involving the main thread, for example.
 252  * Different layer managers can implement child classes of this making it
 253  * possible to create layer manager specific images.
 254  * This class is not meant to be used directly but rather can be set on an
 255  * image container. This is usually done by the layer system internally and
 256  * not explicitly by users. For PlanarYCbCr or Cairo images the default
 257  * implementation will creates images whose data lives in system memory, for
 258  * MacIOSurfaces the default implementation will be a simple MacIOSurface
 259  * wrapper.
 260  */
 261
 262 class ImageFactory {
 263   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory)
 264  protected:
 265   friend class ImageContainer;
 266
 267   ImageFactory() = default;
 268   virtual ~ImageFactory() = default;
 269
 270   virtual RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage(
 271       const gfx::IntSize& aScaleHint, BufferRecycleBin* aRecycleBin);
 272 };
 273
 274 // Used to notify ImageContainer::NotifyComposite()
 275 class ImageContainerListener final {
 276   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainerListener)
 277
 278  public:
 279   explicit ImageContainerListener(ImageContainer* aImageContainer);
 280
 281   void NotifyComposite(const ImageCompositeNotification& aNotification);
 282   void NotifyDropped(uint32_t aDropped);
 283   void ClearImageContainer();
 284   void DropImageClient();
 285
 286  private:
 287   typedef mozilla::Mutex Mutex;
 288
 289   ~ImageContainerListener();
 290
 291   Mutex mLock;
 292   ImageContainer* mImageContainer MOZ_GUARDED_BY(mLock);
 293 };
 294
 295 /**
 296  * A class that manages Images for an ImageLayer. The only reason
 297  * we need a separate class here is that ImageLayers aren't threadsafe
 298  * (because layers can only be used on the main thread) and we want to
 299  * be able to set the current Image from any thread, to facilitate
 300  * video playback without involving the main thread, for example.
 301  *
 302  * An ImageContainer can operate in one of these modes:
 303  * 1) Normal. Triggered by constructing the ImageContainer with
 304  * DISABLE_ASYNC or when compositing is happening on the main thread.
 305  * SetCurrentImages changes ImageContainer state but nothing is sent to the
 306  * compositor until the next layer transaction.
 307  * 2) Asynchronous. Initiated by constructing the ImageContainer with
 308  * ENABLE_ASYNC when compositing is happening on the main thread.
 309  * SetCurrentImages sends a message through the ImageBridge to the compositor
 310  * thread to update the image, without going through the main thread or
 311  * a layer transaction.
 312  * The ImageContainer uses a shared memory block containing a cross-process
 313  * mutex to communicate with the compositor thread. SetCurrentImage
 314  * synchronously updates the shared state to point to the new image and the old
 315  * image is immediately released (not true in Normal or Asynchronous modes).
 316  */
 317 class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
 318   friend class ImageContainerChild;
 319
 320  public:
 321   MOZ_DECLARE_REFCOUNTED_TYPENAME(ImageContainer)
 322
 323   enum Mode { SYNCHRONOUS = 0x0, ASYNCHRONOUS = 0x01 };
 324
 325   static const uint64_t sInvalidAsyncContainerId = 0;
 326
 327   explicit ImageContainer(ImageContainer::Mode flag = SYNCHRONOUS);
 328
 329   /**
 330    * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's
 331    * async container ID.
 332    * @param aAsyncContainerID async container ID for which we are a proxy
 333    */
 334   explicit ImageContainer(const CompositableHandle& aHandle);
 335
 336   ~ImageContainer();
 337
 338   typedef ContainerFrameID FrameID;
 339   typedef ContainerProducerID ProducerID;
 340
 341   RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage();
 342
 343   // Factory methods for shared image types.
 344   RefPtr<SharedRGBImage> CreateSharedRGBImage();
 345
 346   struct NonOwningImage {
 347     explicit NonOwningImage(Image* aImage = nullptr,
 348                             TimeStamp aTimeStamp = TimeStamp(),
 349                             FrameID aFrameID = 0, ProducerID aProducerID = 0)
 350         : mImage(aImage),
 351           mTimeStamp(aTimeStamp),
 352           mFrameID(aFrameID),
 353           mProducerID(aProducerID) {}
 354     Image* mImage;
 355     TimeStamp mTimeStamp;
 356     FrameID mFrameID;
 357     ProducerID mProducerID;
 358   };
 359   /**
 360    * Set aImages as the list of timestamped to display. The Images must have
 361    * been created by this ImageContainer.
 362    * Can be called on any thread. This method takes mRecursiveMutex
 363    * when accessing thread-shared state.
 364    * aImages must be non-empty. The first timestamp in the list may be
 365    * null but the others must not be, and the timestamps must increase.
 366    * Every element of aImages must have non-null mImage.
 367    * mFrameID can be zero, in which case you won't get meaningful
 368    * painted/dropped frame counts. Otherwise you should use a unique and
 369    * increasing ID for each decoded and submitted frame (but it's OK to
 370    * pass the same frame to SetCurrentImages).
 371    * mProducerID is a unique ID for the stream of images. A change in the
 372    * mProducerID means changing to a new mFrameID namespace. All frames in
 373    * aImages must have the same mProducerID.
 374    *
 375    * The Image data must not be modified after this method is called!
 376    * Note that this must not be called if ENABLE_ASYNC has not been set.
 377    *
 378    * The implementation calls CurrentImageChanged() while holding
 379    * mRecursiveMutex.
 380    *
 381    * If this ImageContainer has an ImageClient for async video:
 382    * Schedule a task to send the image to the compositor using the
 383    * PImageBridge protcol without using the main thread.
 384    */
 385   void SetCurrentImages(const nsTArray<NonOwningImage>& aImages);
 386
 387   /**
 388    * Clear all images. Let ImageClient release all TextureClients. Because we
 389    * may release the lock after acquiring it in this method, it cannot be called
 390    * with the lock held.
 391    */
 392   void ClearAllImages() MOZ_EXCLUDES(mRecursiveMutex);
 393
 394   /**
 395    * Clear any resources that are not immediately necessary. This may be called
 396    * in low-memory conditions.
 397    */
 398   void ClearCachedResources();
 399
 400   /**
 401    * Clear the current images.
 402    * This function is expect to be called only from a CompositableClient
 403    * that belongs to ImageBridgeChild. Created to prevent dead lock.
 404    * See Bug 901224.
 405    */
 406   void ClearImagesFromImageBridge();
 407
 408   /**
 409    * Set an Image as the current image to display. The Image must have
 410    * been created by this ImageContainer.
 411    * Must be called on the main thread, within a layers transaction.
 412    *
 413    * This method takes mRecursiveMutex
 414    * when accessing thread-shared state.
 415    * aImage can be null. While it's null, nothing will be painted.
 416    *
 417    * The Image data must not be modified after this method is called!
 418    * Note that this must not be called if ENABLE_ASYNC been set.
 419    *
 420    * You won't get meaningful painted/dropped counts when using this method.
 421    */
 422   void SetCurrentImageInTransaction(Image* aImage);
 423   void SetCurrentImagesInTransaction(const nsTArray<NonOwningImage>& aImages);
 424
 425   /**
 426    * Returns true if this ImageContainer uses the ImageBridge IPDL protocol.
 427    *
 428    * Can be called from any thread.
 429    */
 430   bool IsAsync() const;
 431
 432   /**
 433    * If this ImageContainer uses ImageBridge, returns the ID associated to
 434    * this container, for use in the ImageBridge protocol.
 435    * Returns 0 if this ImageContainer does not use ImageBridge. Note that
 436    * 0 is always an invalid ID for asynchronous image containers.
 437    *
 438    * Can be called from any thread.
 439    */
 440   CompositableHandle GetAsyncContainerHandle();
 441
 442   /**
 443    * Returns if the container currently has an image.
 444    * Can be called on any thread. This method takes mRecursiveMutex
 445    * when accessing thread-shared state.
 446    */
 447   bool HasCurrentImage();
 448
 449   struct OwningImage {
 450     OwningImage() : mFrameID(0), mProducerID(0), mComposited(false) {}
 451     RefPtr<Image> mImage;
 452     TimeStamp mTimeStamp;
 453     FrameID mFrameID;
 454     ProducerID mProducerID;
 455     bool mComposited;
 456   };
 457   /**
 458    * Copy the current Image list to aImages.
 459    * This has to add references since otherwise there are race conditions
 460    * where the current image is destroyed before the caller can add
 461    * a reference.
 462    * Can be called on any thread.
 463    * May return an empty list to indicate there is no current image.
 464    * If aGenerationCounter is non-null, sets *aGenerationCounter to a value
 465    * that's unique for this ImageContainer state.
 466    */
 467   void GetCurrentImages(nsTArray<OwningImage>* aImages,
 468                         uint32_t* aGenerationCounter = nullptr);
 469
 470   /**
 471    * Returns the size of the image in pixels.
 472    * Can be called on any thread. This method takes mRecursiveMutex when
 473    * accessing thread-shared state.
 474    */
 475   gfx::IntSize GetCurrentSize();
 476
 477   /**
 478    * Sets a size that the image is expected to be rendered at.
 479    * This is a hint for image backends to optimize scaling.
 480    * Default implementation in this class is to ignore the hint.
 481    * Can be called on any thread. This method takes mRecursiveMutex
 482    * when accessing thread-shared state.
 483    */
 484   void SetScaleHint(const gfx::IntSize& aScaleHint) {
 485     RecursiveMutexAutoLock lock(mRecursiveMutex);
 486     mScaleHint = aScaleHint;
 487   }
 488
 489   const gfx::IntSize GetScaleHint() const {
 490     RecursiveMutexAutoLock lock(mRecursiveMutex);
 491     return mScaleHint;
 492   }
 493
 494   void SetTransformHint(const gfx::Matrix& aTransformHint) {
 495     RecursiveMutexAutoLock lock(mRecursiveMutex);
 496     mTransformHint = aTransformHint;
 497   }
 498
 499   const gfx::Matrix GetTransformHint() const {
 500     RecursiveMutexAutoLock lock(mRecursiveMutex);
 501     return mTransformHint;
 502   }
 503
 504   void SetRotation(VideoRotation aRotation) {
 505     MOZ_ASSERT(NS_IsMainThread());
 506     mRotation = aRotation;
 507   }
 508
 509   VideoRotation GetRotation() const {
 510     MOZ_ASSERT(NS_IsMainThread());
 511     return mRotation;
 512   }
 513
 514   void SetImageFactory(ImageFactory* aFactory) {
 515     RecursiveMutexAutoLock lock(mRecursiveMutex);
 516     mImageFactory = aFactory ? aFactory : new ImageFactory();
 517   }
 518
 519   already_AddRefed<ImageFactory> GetImageFactory() const {
 520     RecursiveMutexAutoLock lock(mRecursiveMutex);
 521     return do_AddRef(mImageFactory);
 522   }
 523
 524   void EnsureRecycleAllocatorForRDD(KnowsCompositor* aKnowsCompositor);
 525
 526 #ifdef XP_WIN
 527   already_AddRefed<D3D11RecycleAllocator> GetD3D11RecycleAllocator(
 528       KnowsCompositor* aKnowsCompositor, gfx::SurfaceFormat aPreferredFormat);
 529   already_AddRefed<D3D11YCbCrRecycleAllocator> GetD3D11YCbCrRecycleAllocator(
 530       KnowsCompositor* aKnowsCompositor);
 531 #endif
 532
 533 #ifdef XP_DARWIN
 534   already_AddRefed<MacIOSurfaceRecycleAllocator>
 535   GetMacIOSurfaceRecycleAllocator();
 536 #endif
 537
 538   /**
 539    * Returns the delay between the last composited image's presentation
 540    * timestamp and when it was first composited. It's possible for the delay
 541    * to be negative if the first image in the list passed to SetCurrentImages
 542    * has a presentation timestamp greater than "now".
 543    * Returns 0 if the composited image had a null timestamp, or if no
 544    * image has been composited yet.
 545    */
 546   TimeDuration GetPaintDelay() {
 547     RecursiveMutexAutoLock lock(mRecursiveMutex);
 548     return mPaintDelay;
 549   }
 550
 551   /**
 552    * Returns the number of images which have been contained in this container
 553    * and painted at least once.  Can be called from any thread.
 554    */
 555   uint32_t GetPaintCount() {
 556     RecursiveMutexAutoLock lock(mRecursiveMutex);
 557     return mPaintCount;
 558   }
 559
 560   /**
 561    * An entry in the current image list "expires" when the entry has an
 562    * non-null timestamp, and in a SetCurrentImages call the new image list is
 563    * non-empty, the timestamp of the first new image is non-null and greater
 564    * than the timestamp associated with the image, and the first new image's
 565    * frameID is not the same as the entry's.
 566    * Every expired image that is never composited is counted as dropped.
 567    */
 568   uint32_t GetDroppedImageCount() { return mDroppedImageCount; }
 569
 570   void NotifyComposite(const ImageCompositeNotification& aNotification);
 571   void NotifyDropped(uint32_t aDropped);
 572
 573   already_AddRefed<ImageContainerListener> GetImageContainerListener() const;
 574
 575   /**
 576    * Get the ImageClient associated with this container. Returns only after
 577    * validating, and it will recreate the image client if that fails.
 578    * Returns nullptr if not applicable.
 579    */
 580   already_AddRefed<ImageClient> GetImageClient();
 581
 582   /**
 583    * Main thread only.
 584    */
 585   static ProducerID AllocateProducerID();
 586
 587   void DropImageClient();
 588
 589  private:
 590   typedef mozilla::RecursiveMutex RecursiveMutex;
 591
 592   void SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages);
 593
 594   // This is called to ensure we have an active image, this may not be true
 595   // when we're storing image information in a RemoteImageData structure.
 596   // NOTE: If we have remote data mRemoteDataMutex should be locked when
 597   // calling this function!
 598   void EnsureActiveImage();
 599
 600   void EnsureImageClient() MOZ_REQUIRES(mRecursiveMutex);
 601
 602   bool HasImageClient() const {
 603     RecursiveMutexAutoLock lock(mRecursiveMutex);
 604     return !!mImageClient;
 605   }
 606
 607   // RecursiveMutex to protect thread safe access to the "current
 608   // image", and any other state which is shared between threads.
 609   mutable RecursiveMutex mRecursiveMutex;
 610
 611   RefPtr<TextureClientRecycleAllocator> mRecycleAllocator
 612       MOZ_GUARDED_BY(mRecursiveMutex);
 613
 614 #ifdef XP_WIN
 615   RefPtr<D3D11RecycleAllocator> mD3D11RecycleAllocator
 616       MOZ_GUARDED_BY(mRecursiveMutex);
 617
 618   RefPtr<D3D11YCbCrRecycleAllocator> mD3D11YCbCrRecycleAllocator
 619       MOZ_GUARDED_BY(mRecursiveMutex);
 620 #endif
 621 #ifdef XP_DARWIN
 622   RefPtr<MacIOSurfaceRecycleAllocator> mMacIOSurfaceRecycleAllocator
 623       MOZ_GUARDED_BY(mRecursiveMutex);
 624 #endif
 625
 626   nsTArray<OwningImage> mCurrentImages MOZ_GUARDED_BY(mRecursiveMutex);
 627
 628   // Updates every time mActiveImage changes
 629   uint32_t mGenerationCounter MOZ_GUARDED_BY(mRecursiveMutex);
 630
 631   // Number of contained images that have been painted at least once.  It's up
 632   // to the ImageContainer implementation to ensure accesses to this are
 633   // threadsafe.
 634   uint32_t mPaintCount MOZ_GUARDED_BY(mRecursiveMutex);
 635
 636   // See GetPaintDelay. Accessed only with mRecursiveMutex held.
 637   TimeDuration mPaintDelay MOZ_GUARDED_BY(mRecursiveMutex);
 638
 639   // See GetDroppedImageCount.
 640   mozilla::Atomic<uint32_t> mDroppedImageCount;
 641
 642   // This is the image factory used by this container, layer managers using
 643   // this container can set an alternative image factory that will be used to
 644   // create images for this container.
 645   RefPtr<ImageFactory> mImageFactory MOZ_GUARDED_BY(mRecursiveMutex);
 646
 647   gfx::IntSize mScaleHint MOZ_GUARDED_BY(mRecursiveMutex);
 648
 649   gfx::Matrix mTransformHint MOZ_GUARDED_BY(mRecursiveMutex);
 650
 651   // Main thread only.
 652   VideoRotation mRotation;
 653
 654   RefPtr<BufferRecycleBin> mRecycleBin MOZ_GUARDED_BY(mRecursiveMutex);
 655
 656   // This member points to an ImageClient if this ImageContainer was
 657   // sucessfully created with ENABLE_ASYNC, or points to null otherwise.
 658   // 'unsuccessful' in this case only means that the ImageClient could not
 659   // be created, most likely because off-main-thread compositing is not enabled.
 660   // In this case the ImageContainer is perfectly usable, but it will forward
 661   // frames to the compositor through transactions in the main thread rather
 662   // than asynchronusly using the ImageBridge IPDL protocol.
 663   RefPtr<ImageClient> mImageClient MOZ_GUARDED_BY(mRecursiveMutex);
 664
 665   const bool mIsAsync;
 666   CompositableHandle mAsyncContainerHandle MOZ_GUARDED_BY(mRecursiveMutex);
 667
 668   // ProducerID for last current image(s)
 669   ProducerID mCurrentProducerID MOZ_GUARDED_BY(mRecursiveMutex);
 670
 671   RefPtr<ImageContainerListener> mNotifyCompositeListener;
 672
 673   static mozilla::Atomic<uint32_t> sGenerationCounter;
 674 };
 675
 676 class AutoLockImage {
 677  public:
 678   explicit AutoLockImage(ImageContainer* aContainer) {
 679     aContainer->GetCurrentImages(&mImages);
 680   }
 681
 682   bool HasImage() const { return !mImages.IsEmpty(); }
 683   Image* GetImage() const {
 684     return mImages.IsEmpty() ? nullptr : mImages[0].mImage.get();
 685   }
 686
 687   Image* GetImage(TimeStamp aTimeStamp) const {
 688     if (mImages.IsEmpty()) {
 689       return nullptr;
 690     }
 691
 692     MOZ_ASSERT(!aTimeStamp.IsNull());
 693     uint32_t chosenIndex = 0;
 694
 695     while (chosenIndex + 1 < mImages.Length() &&
 696            mImages[chosenIndex + 1].mTimeStamp <= aTimeStamp) {
 697       ++chosenIndex;
 698     }
 699
 700     return mImages[chosenIndex].mImage.get();
 701   }
 702
 703  private:
 704   AutoTArray<ImageContainer::OwningImage, 4> mImages;
 705 };
 706
 707 // This type is currently only used for AVIF and WebCodecs therefore makes some
 708 // specific assumptions (e.g., Alpha's bpc and stride is equal to Y's one)
 709 struct PlanarAlphaData {
 710   uint8_t* mChannel = nullptr;
 711   gfx::IntSize mSize = gfx::IntSize(0, 0);
 712   gfx::ColorDepth mDepth = gfx::ColorDepth::COLOR_8;
 713   bool mPremultiplied = false;
 714 };
 715 struct PlanarYCbCrData {
 716   // Luminance buffer
 717   uint8_t* mYChannel = nullptr;
 718   int32_t mYStride = 0;
 719   int32_t mYSkip = 0;
 720   // Chroma buffers
 721   uint8_t* mCbChannel = nullptr;
 722   uint8_t* mCrChannel = nullptr;
 723   int32_t mCbCrStride = 0;
 724   int32_t mCbSkip = 0;
 725   int32_t mCrSkip = 0;
 726   // Alpha buffer and its metadata
 727   Maybe<PlanarAlphaData> mAlpha = Nothing();
 728   // Picture region
 729   gfx::IntRect mPictureRect = gfx::IntRect(0, 0, 0, 0);
 730   StereoMode mStereoMode = StereoMode::MONO;
 731   gfx::ColorDepth mColorDepth = gfx::ColorDepth::COLOR_8;
 732   gfx::YUVColorSpace mYUVColorSpace = gfx::YUVColorSpace::Default;
 733   gfx::ColorSpace2 mColorPrimaries = gfx::ColorSpace2::UNKNOWN;
 734   gfx::TransferFunction mTransferFunction = gfx::TransferFunction::BT709;
 735   gfx::ColorRange mColorRange = gfx::ColorRange::LIMITED;
 736   gfx::ChromaSubsampling mChromaSubsampling = gfx::ChromaSubsampling::FULL;
 737
 738   // The cropped picture size of the Y channel.
 739   gfx::IntSize YPictureSize() const { return mPictureRect.Size(); }
 740
 741   // The cropped picture size of the Cb/Cr channels.
 742   gfx::IntSize CbCrPictureSize() const {
 743     return mCbCrStride > 0 ? gfx::ChromaSize(YPictureSize(), mChromaSubsampling)
 744                            : gfx::IntSize(0, 0);
 745   }
 746
 747   // The total uncropped size of data in the Y channel.
 748   gfx::IntSize YDataSize() const {
 749     return gfx::IntSize(mPictureRect.XMost(), mPictureRect.YMost());
 750   }
 751
 752   // The total uncropped size of data in the Cb/Cr channels.
 753   gfx::IntSize CbCrDataSize() const {
 754     return mCbCrStride > 0 ? gfx::ChromaSize(YDataSize(), mChromaSubsampling)
 755                            : gfx::IntSize(0, 0);
 756   }
 757
 758   static Maybe<PlanarYCbCrData> From(const SurfaceDescriptorBuffer&);
 759 };
 760
 761 /****** Image subtypes for the different formats ******/
 762
 763 /**
 764  * We assume that the image data is in the REC 470M color space (see
 765  * Theora specification, section 4.3.1).
 766  *
 767  * The YCbCr format can be:
 768  *
 769  * 4:4:4 - CbCr width/height are the same as Y.
 770  * 4:2:2 - CbCr width is half that of Y. Height is the same.
 771  * 4:2:0 - CbCr width and height is half that of Y.
 772  *
 773  * mChromaSubsampling specifies which YCbCr subsampling scheme to use.
 774  *
 775  * The Image that is rendered is the picture region defined by mPictureRect.
 776  *
 777  * mYSkip, mCbSkip, mCrSkip are added to support various output
 778  * formats from hardware decoder. They are per-pixel skips in the
 779  * source image.
 780  *
 781  * For example when image width is 640, mYStride is 670, mYSkip is 2,
 782  * the mYChannel buffer looks like:
 783  *
 784  * |<----------------------- mYStride ----------------------------->|
 785  * |<----------------- YDataSize().width ---------->|
 786  *  0   3   6   9   12  15  18  21                639             669
 787  * |----------------------------------------------------------------|
 788  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 789  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 790  * |Y___Y___Y___Y___Y___Y___Y___Y...                |%%%%%%%%%%%%%%%|
 791  * |            |<->|
 792  *                mYSkip
 793  */
 794 class PlanarYCbCrImage : public Image {
 795  public:
 796   typedef PlanarYCbCrData Data;
 797
 798   enum { MAX_DIMENSION = 16384 };
 799
 800   virtual ~PlanarYCbCrImage();
 801
 802   /**
 803    * This makes a copy of the data buffers, in order to support functioning
 804    * in all different layer managers.
 805    */
 806   virtual nsresult CopyData(const Data& aData) = 0;
 807
 808   /**
 809    * This doesn't make a copy of the data buffers.
 810    */
 811   virtual nsresult AdoptData(const Data& aData);
 812
 813   /**
 814    * This will create an empty data buffers according to the input data's size.
 815    */
 816   virtual nsresult CreateEmptyBuffer(const Data& aData,
 817                                      const gfx::IntSize& aYSize,
 818                                      const gfx::IntSize& aCbCrSize) {
 819     return NS_ERROR_NOT_IMPLEMENTED;
 820   }
 821
 822   /**
 823    * Grab the original YUV data. This is optional.
 824    */
 825   virtual const Data* GetData() const { return &mData; }
 826
 827   /**
 828    * Return the number of bytes of heap memory used to store this image.
 829    */
 830   uint32_t GetDataSize() const { return mBufferSize; }
 831
 832   bool IsValid() const override { return !!mBufferSize; }
 833
 834   gfx::IntSize GetSize() const override { return mSize; }
 835
 836   gfx::IntPoint GetOrigin() const override { return mOrigin; }
 837
 838   PlanarYCbCrImage();
 839
 840   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
 841     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
 842   }
 843
 844   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const = 0;
 845
 846   PlanarYCbCrImage* AsPlanarYCbCrImage() override { return this; }
 847
 848   /**
 849    * Build a SurfaceDescriptorBuffer with this image.  A function to allocate
 850    * a MemoryOrShmem with the given capacity must be provided.
 851    */
 852   nsresult BuildSurfaceDescriptorBuffer(
 853       SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
 854       const std::function<MemoryOrShmem(uint32_t)>& aAllocate) override;
 855
 856  protected:
 857   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
 858
 859   void SetOffscreenFormat(gfxImageFormat aFormat) {
 860     mOffscreenFormat = aFormat;
 861   }
 862   gfxImageFormat GetOffscreenFormat() const;
 863
 864   Data mData;
 865   gfx::IntPoint mOrigin;
 866   gfx::IntSize mSize;
 867   gfxImageFormat mOffscreenFormat;
 868   RefPtr<gfx::DataSourceSurface> mSourceSurface;
 869   uint32_t mBufferSize;
 870 };
 871
 872 class RecyclingPlanarYCbCrImage : public PlanarYCbCrImage {
 873  public:
 874   explicit RecyclingPlanarYCbCrImage(BufferRecycleBin* aRecycleBin)
 875       : mRecycleBin(aRecycleBin) {}
 876   virtual ~RecyclingPlanarYCbCrImage();
 877   nsresult CopyData(const Data& aData) override;
 878   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
 879
 880  protected:
 881   /**
 882    * Return a buffer to store image data in.
 883    */
 884   mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
 885
 886   RefPtr<BufferRecycleBin> mRecycleBin;
 887   mozilla::UniquePtr<uint8_t[]> mBuffer;
 888 };
 889
 890 /**
 891  * NVImage is used to store YUV420SP_NV12 and YUV420SP_NV21 data natively, which
 892  * are not supported by PlanarYCbCrImage. (PlanarYCbCrImage only stores YUV444P,
 893  * YUV422P and YUV420P, it converts YUV420SP_NV12 and YUV420SP_NV21 data into
 894  * YUV420P in its PlanarYCbCrImage::SetData() method.)
 895  *
 896  * PlanarYCbCrData is able to express all the YUV family and so we keep use it
 897  * in NVImage.
 898  */
 899 class NVImage final : public Image {
 900   typedef PlanarYCbCrData Data;
 901
 902  public:
 903   NVImage();
 904   virtual ~NVImage();
 905
 906   // Methods inherited from layers::Image.
 907   gfx::IntSize GetSize() const override;
 908   gfx::IntRect GetPictureRect() const override;
 909   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
 910   nsresult BuildSurfaceDescriptorBuffer(
 911       SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
 912       const std::function<MemoryOrShmem(uint32_t)>& aAllocate) override;
 913   bool IsValid() const override;
 914   NVImage* AsNVImage() override;
 915
 916   // Methods mimic layers::PlanarYCbCrImage.
 917   bool SetData(const Data& aData);
 918   const Data* GetData() const;
 919   uint32_t GetBufferSize() const;
 920
 921  protected:
 922   /**
 923    * Return a buffer to store image data in.
 924    */
 925   mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
 926
 927   mozilla::UniquePtr<uint8_t[]> mBuffer;
 928   uint32_t mBufferSize;
 929   gfx::IntSize mSize;
 930   Data mData;
 931   RefPtr<gfx::DataSourceSurface> mSourceSurface;
 932 };
 933
 934 /**
 935  * Currently, the data in a SourceSurfaceImage surface is treated as being in
 936  * the device output color space. This class is very simple as all backends have
 937  * to know about how to deal with drawing a cairo image.
 938  */
 939 class SourceSurfaceImage final : public Image {
 940  public:
 941   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override {
 942     RefPtr<gfx::SourceSurface> surface(mSourceSurface);
 943     return surface.forget();
 944   }
 945
 946   void SetTextureFlags(TextureFlags aTextureFlags) {
 947     mTextureFlags = aTextureFlags;
 948   }
 949   TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) override;
 950
 951   gfx::IntSize GetSize() const override { return mSize; }
 952
 953   SourceSurfaceImage(const gfx::IntSize& aSize,
 954                      gfx::SourceSurface* aSourceSurface);
 955   explicit SourceSurfaceImage(gfx::SourceSurface* aSourceSurface);
 956   virtual ~SourceSurfaceImage();
 957
 958  private:
 959   gfx::IntSize mSize;
 960   RefPtr<gfx::SourceSurface> mSourceSurface;
 961   nsTHashMap<uint32_t, RefPtr<TextureClient>> mTextureClients;
 962   TextureFlags mTextureFlags;
 963 };
 964
 965 }  // namespace layers
 966 }  // namespace mozilla
 967
 968 #endif