1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef GFX_IMAGECONTAINER_H
8 #define GFX_IMAGECONTAINER_H
10 #include <stdint.h> // for int32_t, uint32_t, uint8_t, uint64_t
11 #include "ImageTypes.h" // for ImageFormat, etc
12 #include "mozilla/AlreadyAddRefed.h"
13 #include "mozilla/Assertions.h" // for MOZ_ASSERT_HELPER2
14 #include "mozilla/Mutex.h" // for Mutex
15 #include "mozilla/RecursiveMutex.h" // for RecursiveMutex, etc
16 #include "mozilla/ThreadSafeWeakPtr.h"
17 #include "mozilla/TimeStamp.h" // for TimeStamp
18 #include "mozilla/gfx/Point.h" // For IntSize
19 #include "mozilla/gfx/Rect.h"
20 #include "mozilla/gfx/Types.h" // For ColorDepth
21 #include "mozilla/layers/LayersTypes.h" // for LayersBackend, etc
22 #include "mozilla/layers/CompositorTypes.h"
23 #include "mozilla/mozalloc.h" // for operator delete, etc
24 #include "mozilla/TypedEnumBits.h"
25 #include "nsDebug.h" // for NS_ASSERTION
26 #include "nsISupportsImpl.h" // for Image::Release, etc
27 #include "nsTArray.h" // for nsTArray
28 #include "nsThreadUtils.h" // for NS_IsMainThread
29 #include "mozilla/Atomics.h"
30 #include "mozilla/gfx/2D.h"
31 #include "mozilla/EnumeratedArray.h"
32 #include "mozilla/UniquePtr.h"
33 #include "nsTHashMap.h"
36 struct ID3D10Texture2D
;
38 struct ID3D10ShaderResourceView
;
45 enum class VideoRotation
;
51 class ImageCompositeNotification
;
53 class ImageContainerChild
;
54 class SharedPlanarYCbCrImage
;
55 class SurfaceDescriptor
;
56 class PlanarYCbCrImage
;
58 class TextureClientRecycleAllocator
;
59 class KnowsCompositor
;
63 class D3D11RecycleAllocator
;
64 class D3D11YCbCrRecycleAllocator
;
67 class MacIOSurfaceRecycleAllocator
;
69 class SurfaceDescriptorBuffer
;
71 struct ImageBackendData
{
72 virtual ~ImageBackendData() = default;
75 ImageBackendData() = default;
78 /* Forward declarations for Image derivatives. */
81 #ifdef MOZ_WIDGET_ANDROID
82 class SurfaceTextureImage
;
83 #elif defined(XP_DARWIN)
84 class MacIOSurfaceImage
;
86 class DMABUFSurfaceImage
;
90 * A class representing a buffer of pixel data. The data can be in one
91 * of various formats including YCbCr.
93 * Create an image using an ImageContainer. Fill the image with data, and
94 * then call ImageContainer::SetImage to display it. An image must not be
95 * modified after calling SetImage. Image implementations do not need to
96 * perform locking; when filling an Image, the Image client is responsible
97 * for ensuring only one thread accesses the Image at a time, and after
98 * SetImage the image is immutable.
100 * When resampling an Image, only pixels within the buffer should be
101 * sampled. For example, cairo images should be sampled in EXTEND_PAD mode.
104 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image
)
107 ImageFormat
GetFormat() const { return mFormat
; }
108 void* GetImplData() const { return mImplData
; }
110 virtual gfx::IntSize
GetSize() const = 0;
111 virtual gfx::IntPoint
GetOrigin() const { return gfx::IntPoint(0, 0); }
112 virtual gfx::IntRect
GetPictureRect() const {
113 return gfx::IntRect(GetOrigin().x
, GetOrigin().y
, GetSize().width
,
116 virtual gfx::ColorDepth
GetColorDepth() const {
117 return gfx::ColorDepth::COLOR_8
;
120 ImageBackendData
* GetBackendData(LayersBackend aBackend
) {
121 return mBackendData
[aBackend
].get();
123 void SetBackendData(LayersBackend aBackend
, ImageBackendData
* aData
) {
124 mBackendData
[aBackend
] = mozilla::WrapUnique(aData
);
127 int32_t GetSerial() const { return mSerial
; }
129 bool IsDRM() const { return mIsDRM
; }
130 virtual void SetIsDRM(bool aIsDRM
) { mIsDRM
= aIsDRM
; }
132 virtual void OnPrepareForwardToHost() {}
133 virtual void OnAbandonForwardToHost() {}
135 virtual already_AddRefed
<gfx::SourceSurface
> GetAsSourceSurface() = 0;
137 enum class BuildSdbFlags
: uint8_t {
142 virtual nsresult
BuildSurfaceDescriptorBuffer(
143 SurfaceDescriptorBuffer
& aSdBuffer
, BuildSdbFlags aFlags
,
144 const std::function
<MemoryOrShmem(uint32_t)>& aAllocate
);
146 virtual bool IsValid() const { return true; }
149 * For use with the TextureForwarder only (so that the later can
150 * synchronize the TextureClient with the TextureHost).
152 virtual TextureClient
* GetTextureClient(KnowsCompositor
* aKnowsCompositor
) {
156 /* Access to derived classes. */
157 virtual GLImage
* AsGLImage() { return nullptr; }
158 virtual GPUVideoImage
* AsGPUVideoImage() { return nullptr; }
159 #ifdef MOZ_WIDGET_ANDROID
160 virtual SurfaceTextureImage
* AsSurfaceTextureImage() { return nullptr; }
163 virtual MacIOSurfaceImage
* AsMacIOSurfaceImage() { return nullptr; }
165 virtual PlanarYCbCrImage
* AsPlanarYCbCrImage() { return nullptr; }
166 #ifdef MOZ_WIDGET_GTK
167 virtual DMABUFSurfaceImage
* AsDMABUFSurfaceImage() { return nullptr; }
170 virtual NVImage
* AsNVImage() { return nullptr; }
172 virtual Maybe
<SurfaceDescriptor
> GetDesc();
174 static nsresult
AllocateSurfaceDescriptorBufferRgb(
175 const gfx::IntSize
& aSize
, gfx::SurfaceFormat aFormat
,
176 uint8_t*& aOutBuffer
, SurfaceDescriptorBuffer
& aSdBuffer
,
178 const std::function
<layers::MemoryOrShmem(uint32_t)>& aAllocate
);
181 Maybe
<SurfaceDescriptor
> GetDescFromTexClient(
182 TextureClient
* tcOverride
= nullptr);
184 Image(void* aImplData
, ImageFormat aFormat
)
185 : mImplData(aImplData
),
186 mSerial(++sSerialCounter
),
190 // Protected destructor, to discourage deletion outside of Release():
191 virtual ~Image() = default;
193 mozilla::EnumeratedArray
<mozilla::layers::LayersBackend
,
194 UniquePtr
<ImageBackendData
>,
195 size_t(mozilla::layers::LayersBackend::LAYERS_LAST
)>
203 static mozilla::Atomic
<int32_t> sSerialCounter
;
206 MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(Image::BuildSdbFlags
)
209 * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
210 * want to recycle from one image to the next.It's a separate object from
211 * ImageContainer because images need to store a strong ref to their RecycleBin
212 * and we must avoid creating a reference loop between an ImageContainer and
215 class BufferRecycleBin final
{
216 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferRecycleBin
)
218 // typedef mozilla::gl::GLContext GLContext;
223 void RecycleBuffer(mozilla::UniquePtr
<uint8_t[]> aBuffer
, uint32_t aSize
);
224 // Returns a recycled buffer of the right size, or allocates a new buffer.
225 mozilla::UniquePtr
<uint8_t[]> GetBuffer(uint32_t aSize
);
226 virtual void ClearRecycledBuffers();
229 typedef mozilla::Mutex Mutex
;
231 // Private destructor, to discourage deletion outside of Release():
232 ~BufferRecycleBin() = default;
234 // This protects mRecycledBuffers, mRecycledBufferSize, mRecycledTextures
235 // and mRecycledTextureSizes
238 // We should probably do something to prune this list on a timer so we don't
239 // eat excess memory while video is paused...
240 nsTArray
<mozilla::UniquePtr
<uint8_t[]>> mRecycledBuffers
241 MOZ_GUARDED_BY(mLock
);
242 // This is only valid if mRecycledBuffers is non-empty
243 uint32_t mRecycledBufferSize
MOZ_GUARDED_BY(mLock
);
247 * A class that manages Image creation for a LayerManager. The only reason
248 * we need a separate class here is that LayerManagers aren't threadsafe
249 * (because layers can only be used on the main thread) and we want to
250 * be able to create images from any thread, to facilitate video playback
251 * without involving the main thread, for example.
252 * Different layer managers can implement child classes of this making it
253 * possible to create layer manager specific images.
254 * This class is not meant to be used directly but rather can be set on an
255 * image container. This is usually done by the layer system internally and
256 * not explicitly by users. For PlanarYCbCr or Cairo images the default
257 * implementation will creates images whose data lives in system memory, for
258 * MacIOSurfaces the default implementation will be a simple MacIOSurface
263 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory
)
265 friend class ImageContainer
;
267 ImageFactory() = default;
268 virtual ~ImageFactory() = default;
270 virtual RefPtr
<PlanarYCbCrImage
> CreatePlanarYCbCrImage(
271 const gfx::IntSize
& aScaleHint
, BufferRecycleBin
* aRecycleBin
);
274 // Used to notify ImageContainer::NotifyComposite()
275 class ImageContainerListener final
{
276 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainerListener
)
279 explicit ImageContainerListener(ImageContainer
* aImageContainer
);
281 void NotifyComposite(const ImageCompositeNotification
& aNotification
);
282 void NotifyDropped(uint32_t aDropped
);
283 void ClearImageContainer();
284 void DropImageClient();
287 typedef mozilla::Mutex Mutex
;
289 ~ImageContainerListener();
292 ImageContainer
* mImageContainer
MOZ_GUARDED_BY(mLock
);
296 * A class that manages Images for an ImageLayer. The only reason
297 * we need a separate class here is that ImageLayers aren't threadsafe
298 * (because layers can only be used on the main thread) and we want to
299 * be able to set the current Image from any thread, to facilitate
300 * video playback without involving the main thread, for example.
302 * An ImageContainer can operate in one of these modes:
303 * 1) Normal. Triggered by constructing the ImageContainer with
304 * DISABLE_ASYNC or when compositing is happening on the main thread.
305 * SetCurrentImages changes ImageContainer state but nothing is sent to the
306 * compositor until the next layer transaction.
307 * 2) Asynchronous. Initiated by constructing the ImageContainer with
308 * ENABLE_ASYNC when compositing is happening on the main thread.
309 * SetCurrentImages sends a message through the ImageBridge to the compositor
310 * thread to update the image, without going through the main thread or
311 * a layer transaction.
312 * The ImageContainer uses a shared memory block containing a cross-process
313 * mutex to communicate with the compositor thread. SetCurrentImage
314 * synchronously updates the shared state to point to the new image and the old
315 * image is immediately released (not true in Normal or Asynchronous modes).
317 class ImageContainer final
: public SupportsThreadSafeWeakPtr
<ImageContainer
> {
318 friend class ImageContainerChild
;
321 MOZ_DECLARE_REFCOUNTED_TYPENAME(ImageContainer
)
323 enum Mode
{ SYNCHRONOUS
= 0x0, ASYNCHRONOUS
= 0x01 };
325 static const uint64_t sInvalidAsyncContainerId
= 0;
327 explicit ImageContainer(ImageContainer::Mode flag
= SYNCHRONOUS
);
330 * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's
331 * async container ID.
332 * @param aAsyncContainerID async container ID for which we are a proxy
334 explicit ImageContainer(const CompositableHandle
& aHandle
);
338 typedef ContainerFrameID FrameID
;
339 typedef ContainerProducerID ProducerID
;
341 RefPtr
<PlanarYCbCrImage
> CreatePlanarYCbCrImage();
343 // Factory methods for shared image types.
344 RefPtr
<SharedRGBImage
> CreateSharedRGBImage();
346 struct NonOwningImage
{
347 explicit NonOwningImage(Image
* aImage
= nullptr,
348 TimeStamp aTimeStamp
= TimeStamp(),
349 FrameID aFrameID
= 0, ProducerID aProducerID
= 0)
351 mTimeStamp(aTimeStamp
),
353 mProducerID(aProducerID
) {}
355 TimeStamp mTimeStamp
;
357 ProducerID mProducerID
;
360 * Set aImages as the list of timestamped to display. The Images must have
361 * been created by this ImageContainer.
362 * Can be called on any thread. This method takes mRecursiveMutex
363 * when accessing thread-shared state.
364 * aImages must be non-empty. The first timestamp in the list may be
365 * null but the others must not be, and the timestamps must increase.
366 * Every element of aImages must have non-null mImage.
367 * mFrameID can be zero, in which case you won't get meaningful
368 * painted/dropped frame counts. Otherwise you should use a unique and
369 * increasing ID for each decoded and submitted frame (but it's OK to
370 * pass the same frame to SetCurrentImages).
371 * mProducerID is a unique ID for the stream of images. A change in the
372 * mProducerID means changing to a new mFrameID namespace. All frames in
373 * aImages must have the same mProducerID.
375 * The Image data must not be modified after this method is called!
376 * Note that this must not be called if ENABLE_ASYNC has not been set.
378 * The implementation calls CurrentImageChanged() while holding
381 * If this ImageContainer has an ImageClient for async video:
382 * Schedule a task to send the image to the compositor using the
383 * PImageBridge protcol without using the main thread.
385 void SetCurrentImages(const nsTArray
<NonOwningImage
>& aImages
);
388 * Clear all images. Let ImageClient release all TextureClients. Because we
389 * may release the lock after acquiring it in this method, it cannot be called
390 * with the lock held.
392 void ClearAllImages() MOZ_EXCLUDES(mRecursiveMutex
);
395 * Clear any resources that are not immediately necessary. This may be called
396 * in low-memory conditions.
398 void ClearCachedResources();
401 * Clear the current images.
402 * This function is expect to be called only from a CompositableClient
403 * that belongs to ImageBridgeChild. Created to prevent dead lock.
406 void ClearImagesFromImageBridge();
409 * Set an Image as the current image to display. The Image must have
410 * been created by this ImageContainer.
411 * Must be called on the main thread, within a layers transaction.
413 * This method takes mRecursiveMutex
414 * when accessing thread-shared state.
415 * aImage can be null. While it's null, nothing will be painted.
417 * The Image data must not be modified after this method is called!
418 * Note that this must not be called if ENABLE_ASYNC been set.
420 * You won't get meaningful painted/dropped counts when using this method.
422 void SetCurrentImageInTransaction(Image
* aImage
);
423 void SetCurrentImagesInTransaction(const nsTArray
<NonOwningImage
>& aImages
);
426 * Returns true if this ImageContainer uses the ImageBridge IPDL protocol.
428 * Can be called from any thread.
430 bool IsAsync() const;
433 * If this ImageContainer uses ImageBridge, returns the ID associated to
434 * this container, for use in the ImageBridge protocol.
435 * Returns 0 if this ImageContainer does not use ImageBridge. Note that
436 * 0 is always an invalid ID for asynchronous image containers.
438 * Can be called from any thread.
440 CompositableHandle
GetAsyncContainerHandle();
443 * Returns if the container currently has an image.
444 * Can be called on any thread. This method takes mRecursiveMutex
445 * when accessing thread-shared state.
447 bool HasCurrentImage();
450 OwningImage() : mFrameID(0), mProducerID(0), mComposited(false) {}
451 RefPtr
<Image
> mImage
;
452 TimeStamp mTimeStamp
;
454 ProducerID mProducerID
;
458 * Copy the current Image list to aImages.
459 * This has to add references since otherwise there are race conditions
460 * where the current image is destroyed before the caller can add
462 * Can be called on any thread.
463 * May return an empty list to indicate there is no current image.
464 * If aGenerationCounter is non-null, sets *aGenerationCounter to a value
465 * that's unique for this ImageContainer state.
467 void GetCurrentImages(nsTArray
<OwningImage
>* aImages
,
468 uint32_t* aGenerationCounter
= nullptr);
471 * Returns the size of the image in pixels.
472 * Can be called on any thread. This method takes mRecursiveMutex when
473 * accessing thread-shared state.
475 gfx::IntSize
GetCurrentSize();
478 * Sets a size that the image is expected to be rendered at.
479 * This is a hint for image backends to optimize scaling.
480 * Default implementation in this class is to ignore the hint.
481 * Can be called on any thread. This method takes mRecursiveMutex
482 * when accessing thread-shared state.
484 void SetScaleHint(const gfx::IntSize
& aScaleHint
) {
485 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
486 mScaleHint
= aScaleHint
;
489 const gfx::IntSize
GetScaleHint() const {
490 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
494 void SetTransformHint(const gfx::Matrix
& aTransformHint
) {
495 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
496 mTransformHint
= aTransformHint
;
499 const gfx::Matrix
GetTransformHint() const {
500 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
501 return mTransformHint
;
504 void SetRotation(VideoRotation aRotation
) {
505 MOZ_ASSERT(NS_IsMainThread());
506 mRotation
= aRotation
;
509 VideoRotation
GetRotation() const {
510 MOZ_ASSERT(NS_IsMainThread());
514 void SetImageFactory(ImageFactory
* aFactory
) {
515 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
516 mImageFactory
= aFactory
? aFactory
: new ImageFactory();
519 already_AddRefed
<ImageFactory
> GetImageFactory() const {
520 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
521 return do_AddRef(mImageFactory
);
524 void EnsureRecycleAllocatorForRDD(KnowsCompositor
* aKnowsCompositor
);
527 already_AddRefed
<D3D11RecycleAllocator
> GetD3D11RecycleAllocator(
528 KnowsCompositor
* aKnowsCompositor
, gfx::SurfaceFormat aPreferredFormat
);
529 already_AddRefed
<D3D11YCbCrRecycleAllocator
> GetD3D11YCbCrRecycleAllocator(
530 KnowsCompositor
* aKnowsCompositor
);
534 already_AddRefed
<MacIOSurfaceRecycleAllocator
>
535 GetMacIOSurfaceRecycleAllocator();
539 * Returns the delay between the last composited image's presentation
540 * timestamp and when it was first composited. It's possible for the delay
541 * to be negative if the first image in the list passed to SetCurrentImages
542 * has a presentation timestamp greater than "now".
543 * Returns 0 if the composited image had a null timestamp, or if no
544 * image has been composited yet.
546 TimeDuration
GetPaintDelay() {
547 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
552 * Returns the number of images which have been contained in this container
553 * and painted at least once. Can be called from any thread.
555 uint32_t GetPaintCount() {
556 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
561 * An entry in the current image list "expires" when the entry has an
562 * non-null timestamp, and in a SetCurrentImages call the new image list is
563 * non-empty, the timestamp of the first new image is non-null and greater
564 * than the timestamp associated with the image, and the first new image's
565 * frameID is not the same as the entry's.
566 * Every expired image that is never composited is counted as dropped.
568 uint32_t GetDroppedImageCount() { return mDroppedImageCount
; }
570 void NotifyComposite(const ImageCompositeNotification
& aNotification
);
571 void NotifyDropped(uint32_t aDropped
);
573 already_AddRefed
<ImageContainerListener
> GetImageContainerListener() const;
576 * Get the ImageClient associated with this container. Returns only after
577 * validating, and it will recreate the image client if that fails.
578 * Returns nullptr if not applicable.
580 already_AddRefed
<ImageClient
> GetImageClient();
585 static ProducerID
AllocateProducerID();
587 void DropImageClient();
590 typedef mozilla::RecursiveMutex RecursiveMutex
;
592 void SetCurrentImageInternal(const nsTArray
<NonOwningImage
>& aImages
);
594 // This is called to ensure we have an active image, this may not be true
595 // when we're storing image information in a RemoteImageData structure.
596 // NOTE: If we have remote data mRemoteDataMutex should be locked when
597 // calling this function!
598 void EnsureActiveImage();
600 void EnsureImageClient() MOZ_REQUIRES(mRecursiveMutex
);
602 bool HasImageClient() const {
603 RecursiveMutexAutoLock
lock(mRecursiveMutex
);
604 return !!mImageClient
;
607 // RecursiveMutex to protect thread safe access to the "current
608 // image", and any other state which is shared between threads.
609 mutable RecursiveMutex mRecursiveMutex
;
611 RefPtr
<TextureClientRecycleAllocator
> mRecycleAllocator
612 MOZ_GUARDED_BY(mRecursiveMutex
);
615 RefPtr
<D3D11RecycleAllocator
> mD3D11RecycleAllocator
616 MOZ_GUARDED_BY(mRecursiveMutex
);
618 RefPtr
<D3D11YCbCrRecycleAllocator
> mD3D11YCbCrRecycleAllocator
619 MOZ_GUARDED_BY(mRecursiveMutex
);
622 RefPtr
<MacIOSurfaceRecycleAllocator
> mMacIOSurfaceRecycleAllocator
623 MOZ_GUARDED_BY(mRecursiveMutex
);
626 nsTArray
<OwningImage
> mCurrentImages
MOZ_GUARDED_BY(mRecursiveMutex
);
628 // Updates every time mActiveImage changes
629 uint32_t mGenerationCounter
MOZ_GUARDED_BY(mRecursiveMutex
);
631 // Number of contained images that have been painted at least once. It's up
632 // to the ImageContainer implementation to ensure accesses to this are
634 uint32_t mPaintCount
MOZ_GUARDED_BY(mRecursiveMutex
);
636 // See GetPaintDelay. Accessed only with mRecursiveMutex held.
637 TimeDuration mPaintDelay
MOZ_GUARDED_BY(mRecursiveMutex
);
639 // See GetDroppedImageCount.
640 mozilla::Atomic
<uint32_t> mDroppedImageCount
;
642 // This is the image factory used by this container, layer managers using
643 // this container can set an alternative image factory that will be used to
644 // create images for this container.
645 RefPtr
<ImageFactory
> mImageFactory
MOZ_GUARDED_BY(mRecursiveMutex
);
647 gfx::IntSize mScaleHint
MOZ_GUARDED_BY(mRecursiveMutex
);
649 gfx::Matrix mTransformHint
MOZ_GUARDED_BY(mRecursiveMutex
);
652 VideoRotation mRotation
;
654 RefPtr
<BufferRecycleBin
> mRecycleBin
MOZ_GUARDED_BY(mRecursiveMutex
);
656 // This member points to an ImageClient if this ImageContainer was
657 // sucessfully created with ENABLE_ASYNC, or points to null otherwise.
658 // 'unsuccessful' in this case only means that the ImageClient could not
659 // be created, most likely because off-main-thread compositing is not enabled.
660 // In this case the ImageContainer is perfectly usable, but it will forward
661 // frames to the compositor through transactions in the main thread rather
662 // than asynchronusly using the ImageBridge IPDL protocol.
663 RefPtr
<ImageClient
> mImageClient
MOZ_GUARDED_BY(mRecursiveMutex
);
666 CompositableHandle mAsyncContainerHandle
MOZ_GUARDED_BY(mRecursiveMutex
);
668 // ProducerID for last current image(s)
669 ProducerID mCurrentProducerID
MOZ_GUARDED_BY(mRecursiveMutex
);
671 RefPtr
<ImageContainerListener
> mNotifyCompositeListener
;
673 static mozilla::Atomic
<uint32_t> sGenerationCounter
;
676 class AutoLockImage
{
678 explicit AutoLockImage(ImageContainer
* aContainer
) {
679 aContainer
->GetCurrentImages(&mImages
);
682 bool HasImage() const { return !mImages
.IsEmpty(); }
683 Image
* GetImage() const {
684 return mImages
.IsEmpty() ? nullptr : mImages
[0].mImage
.get();
687 Image
* GetImage(TimeStamp aTimeStamp
) const {
688 if (mImages
.IsEmpty()) {
692 MOZ_ASSERT(!aTimeStamp
.IsNull());
693 uint32_t chosenIndex
= 0;
695 while (chosenIndex
+ 1 < mImages
.Length() &&
696 mImages
[chosenIndex
+ 1].mTimeStamp
<= aTimeStamp
) {
700 return mImages
[chosenIndex
].mImage
.get();
704 AutoTArray
<ImageContainer::OwningImage
, 4> mImages
;
707 // This type is currently only used for AVIF and WebCodecs therefore makes some
708 // specific assumptions (e.g., Alpha's bpc and stride is equal to Y's one)
709 struct PlanarAlphaData
{
710 uint8_t* mChannel
= nullptr;
711 gfx::IntSize mSize
= gfx::IntSize(0, 0);
712 gfx::ColorDepth mDepth
= gfx::ColorDepth::COLOR_8
;
713 bool mPremultiplied
= false;
715 struct PlanarYCbCrData
{
717 uint8_t* mYChannel
= nullptr;
718 int32_t mYStride
= 0;
721 uint8_t* mCbChannel
= nullptr;
722 uint8_t* mCrChannel
= nullptr;
723 int32_t mCbCrStride
= 0;
726 // Alpha buffer and its metadata
727 Maybe
<PlanarAlphaData
> mAlpha
= Nothing();
729 gfx::IntRect mPictureRect
= gfx::IntRect(0, 0, 0, 0);
730 StereoMode mStereoMode
= StereoMode::MONO
;
731 gfx::ColorDepth mColorDepth
= gfx::ColorDepth::COLOR_8
;
732 gfx::YUVColorSpace mYUVColorSpace
= gfx::YUVColorSpace::Default
;
733 gfx::ColorSpace2 mColorPrimaries
= gfx::ColorSpace2::UNKNOWN
;
734 gfx::TransferFunction mTransferFunction
= gfx::TransferFunction::BT709
;
735 gfx::ColorRange mColorRange
= gfx::ColorRange::LIMITED
;
736 gfx::ChromaSubsampling mChromaSubsampling
= gfx::ChromaSubsampling::FULL
;
738 // The cropped picture size of the Y channel.
739 gfx::IntSize
YPictureSize() const { return mPictureRect
.Size(); }
741 // The cropped picture size of the Cb/Cr channels.
742 gfx::IntSize
CbCrPictureSize() const {
743 return mCbCrStride
> 0 ? gfx::ChromaSize(YPictureSize(), mChromaSubsampling
)
744 : gfx::IntSize(0, 0);
747 // The total uncropped size of data in the Y channel.
748 gfx::IntSize
YDataSize() const {
749 return gfx::IntSize(mPictureRect
.XMost(), mPictureRect
.YMost());
752 // The total uncropped size of data in the Cb/Cr channels.
753 gfx::IntSize
CbCrDataSize() const {
754 return mCbCrStride
> 0 ? gfx::ChromaSize(YDataSize(), mChromaSubsampling
)
755 : gfx::IntSize(0, 0);
758 static Maybe
<PlanarYCbCrData
> From(const SurfaceDescriptorBuffer
&);
761 /****** Image subtypes for the different formats ******/
764 * We assume that the image data is in the REC 470M color space (see
765 * Theora specification, section 4.3.1).
767 * The YCbCr format can be:
769 * 4:4:4 - CbCr width/height are the same as Y.
770 * 4:2:2 - CbCr width is half that of Y. Height is the same.
771 * 4:2:0 - CbCr width and height is half that of Y.
773 * mChromaSubsampling specifies which YCbCr subsampling scheme to use.
775 * The Image that is rendered is the picture region defined by mPictureRect.
777 * mYSkip, mCbSkip, mCrSkip are added to support various output
778 * formats from hardware decoder. They are per-pixel skips in the
781 * For example when image width is 640, mYStride is 670, mYSkip is 2,
782 * the mYChannel buffer looks like:
784 * |<----------------------- mYStride ----------------------------->|
785 * |<----------------- YDataSize().width ---------->|
786 * 0 3 6 9 12 15 18 21 639 669
787 * |----------------------------------------------------------------|
788 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
789 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
790 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
794 class PlanarYCbCrImage
: public Image
{
796 typedef PlanarYCbCrData Data
;
798 enum { MAX_DIMENSION
= 16384 };
800 virtual ~PlanarYCbCrImage();
803 * This makes a copy of the data buffers, in order to support functioning
804 * in all different layer managers.
806 virtual nsresult
CopyData(const Data
& aData
) = 0;
809 * This doesn't make a copy of the data buffers.
811 virtual nsresult
AdoptData(const Data
& aData
);
814 * This will create an empty data buffers according to the input data's size.
816 virtual nsresult
CreateEmptyBuffer(const Data
& aData
,
817 const gfx::IntSize
& aYSize
,
818 const gfx::IntSize
& aCbCrSize
) {
819 return NS_ERROR_NOT_IMPLEMENTED
;
823 * Grab the original YUV data. This is optional.
825 virtual const Data
* GetData() const { return &mData
; }
828 * Return the number of bytes of heap memory used to store this image.
830 uint32_t GetDataSize() const { return mBufferSize
; }
832 bool IsValid() const override
{ return !!mBufferSize
; }
834 gfx::IntSize
GetSize() const override
{ return mSize
; }
836 gfx::IntPoint
GetOrigin() const override
{ return mOrigin
; }
840 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf
) const {
841 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf
);
844 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const = 0;
846 PlanarYCbCrImage
* AsPlanarYCbCrImage() override
{ return this; }
849 * Build a SurfaceDescriptorBuffer with this image. A function to allocate
850 * a MemoryOrShmem with the given capacity must be provided.
852 nsresult
BuildSurfaceDescriptorBuffer(
853 SurfaceDescriptorBuffer
& aSdBuffer
, BuildSdbFlags aFlags
,
854 const std::function
<MemoryOrShmem(uint32_t)>& aAllocate
) override
;
857 already_AddRefed
<gfx::SourceSurface
> GetAsSourceSurface() override
;
859 void SetOffscreenFormat(gfxImageFormat aFormat
) {
860 mOffscreenFormat
= aFormat
;
862 gfxImageFormat
GetOffscreenFormat() const;
865 gfx::IntPoint mOrigin
;
867 gfxImageFormat mOffscreenFormat
;
868 RefPtr
<gfx::DataSourceSurface
> mSourceSurface
;
869 uint32_t mBufferSize
;
872 class RecyclingPlanarYCbCrImage
: public PlanarYCbCrImage
{
874 explicit RecyclingPlanarYCbCrImage(BufferRecycleBin
* aRecycleBin
)
875 : mRecycleBin(aRecycleBin
) {}
876 virtual ~RecyclingPlanarYCbCrImage();
877 nsresult
CopyData(const Data
& aData
) override
;
878 size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf
) const override
;
882 * Return a buffer to store image data in.
884 mozilla::UniquePtr
<uint8_t[]> AllocateBuffer(uint32_t aSize
);
886 RefPtr
<BufferRecycleBin
> mRecycleBin
;
887 mozilla::UniquePtr
<uint8_t[]> mBuffer
;
891 * NVImage is used to store YUV420SP_NV12 and YUV420SP_NV21 data natively, which
892 * are not supported by PlanarYCbCrImage. (PlanarYCbCrImage only stores YUV444P,
893 * YUV422P and YUV420P, it converts YUV420SP_NV12 and YUV420SP_NV21 data into
894 * YUV420P in its PlanarYCbCrImage::SetData() method.)
896 * PlanarYCbCrData is able to express all the YUV family and so we keep use it
899 class NVImage final
: public Image
{
900 typedef PlanarYCbCrData Data
;
906 // Methods inherited from layers::Image.
907 gfx::IntSize
GetSize() const override
;
908 gfx::IntRect
GetPictureRect() const override
;
909 already_AddRefed
<gfx::SourceSurface
> GetAsSourceSurface() override
;
910 nsresult
BuildSurfaceDescriptorBuffer(
911 SurfaceDescriptorBuffer
& aSdBuffer
, BuildSdbFlags aFlags
,
912 const std::function
<MemoryOrShmem(uint32_t)>& aAllocate
) override
;
913 bool IsValid() const override
;
914 NVImage
* AsNVImage() override
;
916 // Methods mimic layers::PlanarYCbCrImage.
917 bool SetData(const Data
& aData
);
918 const Data
* GetData() const;
919 uint32_t GetBufferSize() const;
923 * Return a buffer to store image data in.
925 mozilla::UniquePtr
<uint8_t[]> AllocateBuffer(uint32_t aSize
);
927 mozilla::UniquePtr
<uint8_t[]> mBuffer
;
928 uint32_t mBufferSize
;
931 RefPtr
<gfx::DataSourceSurface
> mSourceSurface
;
935 * Currently, the data in a SourceSurfaceImage surface is treated as being in
936 * the device output color space. This class is very simple as all backends have
937 * to know about how to deal with drawing a cairo image.
939 class SourceSurfaceImage final
: public Image
{
941 already_AddRefed
<gfx::SourceSurface
> GetAsSourceSurface() override
{
942 RefPtr
<gfx::SourceSurface
> surface(mSourceSurface
);
943 return surface
.forget();
946 void SetTextureFlags(TextureFlags aTextureFlags
) {
947 mTextureFlags
= aTextureFlags
;
949 TextureClient
* GetTextureClient(KnowsCompositor
* aKnowsCompositor
) override
;
951 gfx::IntSize
GetSize() const override
{ return mSize
; }
953 SourceSurfaceImage(const gfx::IntSize
& aSize
,
954 gfx::SourceSurface
* aSourceSurface
);
955 explicit SourceSurfaceImage(gfx::SourceSurface
* aSourceSurface
);
956 virtual ~SourceSurfaceImage();
960 RefPtr
<gfx::SourceSurface
> mSourceSurface
;
961 nsTHashMap
<uint32_t, RefPtr
<TextureClient
>> mTextureClients
;
962 TextureFlags mTextureFlags
;
965 } // namespace layers
966 } // namespace mozilla