Bug 1882703 [wpt PR 44848] - Update wpt metadata, a=testonly
[gecko.git] / gfx / layers / ImageContainer.cpp
blob8a4c03f5a2f9f6e0eb318045861b461f419d7830
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "ImageContainer.h"
9 #include <string.h> // for memcpy, memset
11 #include "GLImages.h" // for SurfaceTextureImage
12 #include "MediaInfo.h" // VideoInfo::Rotation
13 #include "YCbCrUtils.h" // for YCbCr conversions
14 #include "gfx2DGlue.h"
15 #include "gfxPlatform.h" // for gfxPlatform
16 #include "gfxUtils.h" // for gfxUtils
17 #include "libyuv.h"
18 #include "mozilla/CheckedInt.h"
19 #include "mozilla/ProfilerLabels.h"
20 #include "mozilla/RefPtr.h" // for already_AddRefed
21 #include "mozilla/StaticPrefs_layers.h"
22 #include "mozilla/gfx/2D.h"
23 #include "mozilla/gfx/gfxVars.h"
24 #include "mozilla/gfx/Swizzle.h"
25 #include "mozilla/ipc/CrossProcessMutex.h" // for CrossProcessMutex, etc
26 #include "mozilla/layers/CompositorTypes.h"
27 #include "mozilla/layers/ImageBridgeChild.h" // for ImageBridgeChild
28 #include "mozilla/layers/ImageClient.h" // for ImageClient
29 #include "mozilla/layers/ImageDataSerializer.h" // for SurfaceDescriptorBuffer
30 #include "mozilla/layers/LayersMessages.h"
31 #include "mozilla/layers/SharedPlanarYCbCrImage.h"
32 #include "mozilla/layers/SharedRGBImage.h"
33 #include "mozilla/layers/TextureClientRecycleAllocator.h"
34 #include "nsProxyRelease.h"
35 #include "nsISupportsUtils.h" // for NS_IF_ADDREF
37 #ifdef XP_MACOSX
38 # include "MacIOSurfaceImage.h"
39 #endif
41 #ifdef XP_WIN
42 # include <d3d10_1.h>
44 # include "gfxWindowsPlatform.h"
45 # include "mozilla/gfx/DeviceManagerDx.h"
46 # include "mozilla/layers/D3D11ShareHandleImage.h"
47 # include "mozilla/layers/D3D11YCbCrImage.h"
48 #endif
50 namespace mozilla::layers {
52 using namespace mozilla::gfx;
53 using namespace mozilla::ipc;
55 Atomic<int32_t> Image::sSerialCounter(0);
57 Atomic<uint32_t> ImageContainer::sGenerationCounter(0);
59 static void CopyPlane(uint8_t* aDst, const uint8_t* aSrc,
60 const gfx::IntSize& aSize, int32_t aStride,
61 int32_t aSkip);
63 RefPtr<PlanarYCbCrImage> ImageFactory::CreatePlanarYCbCrImage(
64 const gfx::IntSize& aScaleHint, BufferRecycleBin* aRecycleBin) {
65 return new RecyclingPlanarYCbCrImage(aRecycleBin);
68 BufferRecycleBin::BufferRecycleBin()
69 : mLock("mozilla.layers.BufferRecycleBin.mLock")
70 // This member is only valid when the bin is not empty and will be
71 // properly initialized in RecycleBuffer, but initializing it here avoids
72 // static analysis noise.
74 mRecycledBufferSize(0) {}
76 void BufferRecycleBin::RecycleBuffer(UniquePtr<uint8_t[]> aBuffer,
77 uint32_t aSize) {
78 MutexAutoLock lock(mLock);
80 if (!mRecycledBuffers.IsEmpty() && aSize != mRecycledBufferSize) {
81 mRecycledBuffers.Clear();
83 mRecycledBufferSize = aSize;
84 mRecycledBuffers.AppendElement(std::move(aBuffer));
87 UniquePtr<uint8_t[]> BufferRecycleBin::GetBuffer(uint32_t aSize) {
88 MutexAutoLock lock(mLock);
90 if (mRecycledBuffers.IsEmpty() || mRecycledBufferSize != aSize) {
91 return UniquePtr<uint8_t[]>(new (fallible) uint8_t[aSize]);
94 return mRecycledBuffers.PopLastElement();
97 void BufferRecycleBin::ClearRecycledBuffers() {
98 MutexAutoLock lock(mLock);
99 if (!mRecycledBuffers.IsEmpty()) {
100 mRecycledBuffers.Clear();
102 mRecycledBufferSize = 0;
105 ImageContainerListener::ImageContainerListener(ImageContainer* aImageContainer)
106 : mLock("mozilla.layers.ImageContainerListener.mLock"),
107 mImageContainer(aImageContainer) {}
109 ImageContainerListener::~ImageContainerListener() = default;
111 void ImageContainerListener::NotifyComposite(
112 const ImageCompositeNotification& aNotification) {
113 MutexAutoLock lock(mLock);
114 if (mImageContainer) {
115 mImageContainer->NotifyComposite(aNotification);
119 void ImageContainerListener::NotifyDropped(uint32_t aDropped) {
120 MutexAutoLock lock(mLock);
121 if (mImageContainer) {
122 mImageContainer->NotifyDropped(aDropped);
126 void ImageContainerListener::ClearImageContainer() {
127 MutexAutoLock lock(mLock);
128 mImageContainer = nullptr;
131 void ImageContainerListener::DropImageClient() {
132 MutexAutoLock lock(mLock);
133 if (mImageContainer) {
134 mImageContainer->DropImageClient();
138 already_AddRefed<ImageClient> ImageContainer::GetImageClient() {
139 RecursiveMutexAutoLock mon(mRecursiveMutex);
140 EnsureImageClient();
141 RefPtr<ImageClient> imageClient = mImageClient;
142 return imageClient.forget();
145 void ImageContainer::DropImageClient() {
146 RecursiveMutexAutoLock mon(mRecursiveMutex);
147 if (mImageClient) {
148 mImageClient->ClearCachedResources();
149 mImageClient = nullptr;
153 void ImageContainer::EnsureImageClient() {
154 // If we're not forcing a new ImageClient, then we can skip this if we don't
155 // have an existing ImageClient, or if the existing one belongs to an IPC
156 // actor that is still open.
157 if (!mIsAsync) {
158 return;
160 if (mImageClient &&
161 mImageClient->GetForwarder()->GetLayersIPCActor()->IPCOpen()) {
162 return;
165 RefPtr<ImageBridgeChild> imageBridge = ImageBridgeChild::GetSingleton();
166 if (imageBridge) {
167 mImageClient =
168 imageBridge->CreateImageClient(CompositableType::IMAGE, this);
169 if (mImageClient) {
170 mAsyncContainerHandle = mImageClient->GetAsyncHandle();
171 } else {
172 // It's okay to drop the async container handle since the ImageBridgeChild
173 // is going to die anyway.
174 mAsyncContainerHandle = CompositableHandle();
179 ImageContainer::ImageContainer(Mode flag)
180 : mRecursiveMutex("ImageContainer.mRecursiveMutex"),
181 mGenerationCounter(++sGenerationCounter),
182 mPaintCount(0),
183 mDroppedImageCount(0),
184 mImageFactory(new ImageFactory()),
185 mRotation(VideoRotation::kDegree_0),
186 mRecycleBin(new BufferRecycleBin()),
187 mIsAsync(flag == ASYNCHRONOUS),
188 mCurrentProducerID(-1) {
189 if (flag == ASYNCHRONOUS) {
190 mNotifyCompositeListener = new ImageContainerListener(this);
191 EnsureImageClient();
195 ImageContainer::ImageContainer(const CompositableHandle& aHandle)
196 : mRecursiveMutex("ImageContainer.mRecursiveMutex"),
197 mGenerationCounter(++sGenerationCounter),
198 mPaintCount(0),
199 mDroppedImageCount(0),
200 mImageFactory(nullptr),
201 mRotation(VideoRotation::kDegree_0),
202 mRecycleBin(nullptr),
203 mIsAsync(true),
204 mAsyncContainerHandle(aHandle),
205 mCurrentProducerID(-1) {
206 MOZ_ASSERT(mAsyncContainerHandle);
209 ImageContainer::~ImageContainer() {
210 if (mNotifyCompositeListener) {
211 mNotifyCompositeListener->ClearImageContainer();
213 if (mAsyncContainerHandle) {
214 if (RefPtr<ImageBridgeChild> imageBridge =
215 ImageBridgeChild::GetSingleton()) {
216 imageBridge->ForgetImageContainer(mAsyncContainerHandle);
221 /* static */ nsresult Image::AllocateSurfaceDescriptorBufferRgb(
222 const gfx::IntSize& aSize, gfx::SurfaceFormat aFormat, uint8_t*& aOutBuffer,
223 SurfaceDescriptorBuffer& aSdBuffer, int32_t& aStride,
224 const std::function<layers::MemoryOrShmem(uint32_t)>& aAllocate) {
225 aStride = ImageDataSerializer::ComputeRGBStride(aFormat, aSize.width);
226 size_t length = ImageDataSerializer::ComputeRGBBufferSize(aSize, aFormat);
228 if (aStride <= 0 || length == 0) {
229 return NS_ERROR_INVALID_ARG;
232 aSdBuffer.desc() = RGBDescriptor(aSize, aFormat);
233 aSdBuffer.data() = aAllocate(length);
235 const layers::MemoryOrShmem& memOrShmem = aSdBuffer.data();
236 switch (memOrShmem.type()) {
237 case layers::MemoryOrShmem::Tuintptr_t:
238 aOutBuffer = reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t());
239 break;
240 case layers::MemoryOrShmem::TShmem:
241 aOutBuffer = memOrShmem.get_Shmem().get<uint8_t>();
242 break;
243 default:
244 return NS_ERROR_OUT_OF_MEMORY;
247 MOZ_ASSERT(aOutBuffer);
248 return NS_OK;
251 nsresult Image::BuildSurfaceDescriptorBuffer(
252 SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
253 const std::function<MemoryOrShmem(uint32_t)>& aAllocate) {
254 return NS_ERROR_NOT_IMPLEMENTED;
257 Maybe<SurfaceDescriptor> Image::GetDesc() { return GetDescFromTexClient(); }
259 Maybe<SurfaceDescriptor> Image::GetDescFromTexClient(
260 TextureClient* const tcOverride) {
261 RefPtr<TextureClient> tc = tcOverride;
262 if (!tcOverride) {
263 tc = GetTextureClient(nullptr);
265 if (!tc) {
266 return {};
269 const auto& tcd = tc->GetInternalData();
271 SurfaceDescriptor ret;
272 if (!tcd->Serialize(ret)) {
273 return {};
275 return Some(ret);
278 already_AddRefed<ImageContainerListener>
279 ImageContainer::GetImageContainerListener() const {
280 MOZ_ASSERT(InImageBridgeChildThread());
281 return do_AddRef(mNotifyCompositeListener);
284 RefPtr<PlanarYCbCrImage> ImageContainer::CreatePlanarYCbCrImage() {
285 RecursiveMutexAutoLock lock(mRecursiveMutex);
286 EnsureImageClient();
287 if (mImageClient && mImageClient->AsImageClientSingle()) {
288 return new SharedPlanarYCbCrImage(mImageClient);
290 if (mRecycleAllocator) {
291 return new SharedPlanarYCbCrImage(mRecycleAllocator);
293 return mImageFactory->CreatePlanarYCbCrImage(mScaleHint, mRecycleBin);
296 RefPtr<SharedRGBImage> ImageContainer::CreateSharedRGBImage() {
297 RecursiveMutexAutoLock lock(mRecursiveMutex);
298 EnsureImageClient();
299 if (mImageClient && mImageClient->AsImageClientSingle()) {
300 return new SharedRGBImage(mImageClient);
302 if (mRecycleAllocator) {
303 return new SharedRGBImage(mRecycleAllocator);
305 return nullptr;
308 void ImageContainer::SetCurrentImageInternal(
309 const nsTArray<NonOwningImage>& aImages) {
310 RecursiveMutexAutoLock lock(mRecursiveMutex);
312 mGenerationCounter = ++sGenerationCounter;
314 if (!aImages.IsEmpty()) {
315 NS_ASSERTION(mCurrentImages.IsEmpty() ||
316 mCurrentImages[0].mProducerID != aImages[0].mProducerID ||
317 mCurrentImages[0].mFrameID <= aImages[0].mFrameID,
318 "frame IDs shouldn't go backwards");
319 if (aImages[0].mProducerID != mCurrentProducerID) {
320 mCurrentProducerID = aImages[0].mProducerID;
322 for (auto& img : mCurrentImages) {
323 img.mImage->OnAbandonForwardToHost();
327 nsTArray<OwningImage> newImages;
329 for (uint32_t i = 0; i < aImages.Length(); ++i) {
330 NS_ASSERTION(aImages[i].mImage, "image can't be null");
331 NS_ASSERTION(!aImages[i].mTimeStamp.IsNull() || aImages.Length() == 1,
332 "Multiple images require timestamps");
333 if (i > 0) {
334 NS_ASSERTION(aImages[i].mTimeStamp >= aImages[i - 1].mTimeStamp,
335 "Timestamps must not decrease");
336 NS_ASSERTION(aImages[i].mFrameID > aImages[i - 1].mFrameID,
337 "FrameIDs must increase");
338 NS_ASSERTION(aImages[i].mProducerID == aImages[i - 1].mProducerID,
339 "ProducerIDs must be the same");
341 OwningImage* img = newImages.AppendElement();
342 img->mImage = aImages[i].mImage;
343 img->mTimeStamp = aImages[i].mTimeStamp;
344 img->mFrameID = aImages[i].mFrameID;
345 img->mProducerID = aImages[i].mProducerID;
346 for (const auto& oldImg : mCurrentImages) {
347 if (oldImg.mFrameID == img->mFrameID &&
348 oldImg.mProducerID == img->mProducerID) {
349 img->mComposited = oldImg.mComposited;
350 break;
353 img->mImage->OnPrepareForwardToHost();
356 mCurrentImages = std::move(newImages);
359 void ImageContainer::ClearImagesFromImageBridge() {
360 RecursiveMutexAutoLock lock(mRecursiveMutex);
361 SetCurrentImageInternal(nsTArray<NonOwningImage>());
364 void ImageContainer::SetCurrentImages(const nsTArray<NonOwningImage>& aImages) {
365 AUTO_PROFILER_LABEL("ImageContainer::SetCurrentImages", GRAPHICS);
366 MOZ_ASSERT(!aImages.IsEmpty());
367 RecursiveMutexAutoLock lock(mRecursiveMutex);
368 if (mIsAsync) {
369 if (RefPtr<ImageBridgeChild> imageBridge =
370 ImageBridgeChild::GetSingleton()) {
371 imageBridge->UpdateImageClient(this);
374 SetCurrentImageInternal(aImages);
377 void ImageContainer::ClearAllImages() {
378 mRecursiveMutex.Lock();
379 if (mImageClient) {
380 RefPtr<ImageClient> imageClient = mImageClient;
381 mRecursiveMutex.Unlock();
383 // Let ImageClient release all TextureClients. This doesn't return
384 // until ImageBridge has called ClearCurrentImageFromImageBridge.
385 if (RefPtr<ImageBridgeChild> imageBridge =
386 ImageBridgeChild::GetSingleton()) {
387 imageBridge->FlushAllImages(imageClient, this);
389 return;
392 SetCurrentImageInternal(nsTArray<NonOwningImage>());
393 mRecursiveMutex.Unlock();
396 void ImageContainer::ClearCachedResources() {
397 RecursiveMutexAutoLock lock(mRecursiveMutex);
398 if (mImageClient && mImageClient->AsImageClientSingle()) {
399 if (!mImageClient->HasTextureClientRecycler()) {
400 return;
402 mImageClient->GetTextureClientRecycler()->ShrinkToMinimumSize();
403 return;
405 return mRecycleBin->ClearRecycledBuffers();
408 void ImageContainer::SetCurrentImageInTransaction(Image* aImage) {
409 AutoTArray<NonOwningImage, 1> images;
410 images.AppendElement(NonOwningImage(aImage));
411 SetCurrentImagesInTransaction(images);
414 void ImageContainer::SetCurrentImagesInTransaction(
415 const nsTArray<NonOwningImage>& aImages) {
416 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
417 NS_ASSERTION(!HasImageClient(),
418 "Should use async image transfer with ImageBridge.");
420 SetCurrentImageInternal(aImages);
423 bool ImageContainer::IsAsync() const { return mIsAsync; }
425 CompositableHandle ImageContainer::GetAsyncContainerHandle() {
426 NS_ASSERTION(IsAsync(),
427 "Shared image ID is only relevant to async ImageContainers");
428 RecursiveMutexAutoLock mon(mRecursiveMutex);
429 NS_ASSERTION(mAsyncContainerHandle, "Should have a shared image ID");
430 EnsureImageClient();
431 return mAsyncContainerHandle;
434 bool ImageContainer::HasCurrentImage() {
435 RecursiveMutexAutoLock lock(mRecursiveMutex);
437 return !mCurrentImages.IsEmpty();
440 void ImageContainer::GetCurrentImages(nsTArray<OwningImage>* aImages,
441 uint32_t* aGenerationCounter) {
442 RecursiveMutexAutoLock lock(mRecursiveMutex);
444 *aImages = mCurrentImages.Clone();
445 if (aGenerationCounter) {
446 *aGenerationCounter = mGenerationCounter;
450 gfx::IntSize ImageContainer::GetCurrentSize() {
451 RecursiveMutexAutoLock lock(mRecursiveMutex);
453 if (mCurrentImages.IsEmpty()) {
454 return gfx::IntSize(0, 0);
457 return mCurrentImages[0].mImage->GetSize();
460 void ImageContainer::NotifyComposite(
461 const ImageCompositeNotification& aNotification) {
462 RecursiveMutexAutoLock lock(mRecursiveMutex);
464 // An image composition notification is sent the first time a particular
465 // image is composited by an ImageHost. Thus, every time we receive such
466 // a notification, a new image has been painted.
467 ++mPaintCount;
469 if (aNotification.producerID() == mCurrentProducerID) {
470 for (auto& img : mCurrentImages) {
471 if (img.mFrameID == aNotification.frameID()) {
472 img.mComposited = true;
477 if (!aNotification.imageTimeStamp().IsNull()) {
478 mPaintDelay = aNotification.firstCompositeTimeStamp() -
479 aNotification.imageTimeStamp();
483 void ImageContainer::NotifyDropped(uint32_t aDropped) {
484 mDroppedImageCount += aDropped;
487 void ImageContainer::EnsureRecycleAllocatorForRDD(
488 KnowsCompositor* aKnowsCompositor) {
489 MOZ_ASSERT(!mIsAsync);
490 MOZ_ASSERT(XRE_IsRDDProcess());
492 RecursiveMutexAutoLock lock(mRecursiveMutex);
493 MOZ_ASSERT(!mImageClient);
495 if (mRecycleAllocator &&
496 aKnowsCompositor == mRecycleAllocator->GetKnowsCompositor()) {
497 return;
500 if (!StaticPrefs::layers_recycle_allocator_rdd_AtStartup()) {
501 return;
504 static const uint32_t MAX_POOLED_VIDEO_COUNT = 5;
506 mRecycleAllocator =
507 new layers::TextureClientRecycleAllocator(aKnowsCompositor);
508 mRecycleAllocator->SetMaxPoolSize(MAX_POOLED_VIDEO_COUNT);
511 #ifdef XP_WIN
512 already_AddRefed<D3D11RecycleAllocator>
513 ImageContainer::GetD3D11RecycleAllocator(KnowsCompositor* aKnowsCompositor,
514 gfx::SurfaceFormat aPreferredFormat) {
515 MOZ_ASSERT(aKnowsCompositor);
517 if (!aKnowsCompositor->SupportsD3D11()) {
518 return nullptr;
521 RefPtr<ID3D11Device> device = gfx::DeviceManagerDx::Get()->GetImageDevice();
522 if (!device) {
523 return nullptr;
526 RecursiveMutexAutoLock lock(mRecursiveMutex);
527 if (mD3D11RecycleAllocator && mD3D11RecycleAllocator->mDevice == device &&
528 mD3D11RecycleAllocator->GetKnowsCompositor() == aKnowsCompositor) {
529 return do_AddRef(mD3D11RecycleAllocator);
532 mD3D11RecycleAllocator =
533 new D3D11RecycleAllocator(aKnowsCompositor, device, aPreferredFormat);
535 if (device != DeviceManagerDx::Get()->GetCompositorDevice()) {
536 RefPtr<SyncObjectClient> syncObject =
537 SyncObjectClient::CreateSyncObjectClient(
538 aKnowsCompositor->GetTextureFactoryIdentifier().mSyncHandle,
539 device);
540 mD3D11RecycleAllocator->SetSyncObject(syncObject);
543 return do_AddRef(mD3D11RecycleAllocator);
546 already_AddRefed<D3D11YCbCrRecycleAllocator>
547 ImageContainer::GetD3D11YCbCrRecycleAllocator(
548 KnowsCompositor* aKnowsCompositor) {
549 if (!aKnowsCompositor->SupportsD3D11() ||
550 !gfx::DeviceManagerDx::Get()->GetImageDevice()) {
551 return nullptr;
554 RecursiveMutexAutoLock lock(mRecursiveMutex);
555 if (mD3D11YCbCrRecycleAllocator &&
556 aKnowsCompositor == mD3D11YCbCrRecycleAllocator->GetKnowsCompositor()) {
557 return do_AddRef(mD3D11YCbCrRecycleAllocator);
560 mD3D11YCbCrRecycleAllocator =
561 new D3D11YCbCrRecycleAllocator(aKnowsCompositor);
562 return do_AddRef(mD3D11YCbCrRecycleAllocator);
564 #endif
566 #ifdef XP_MACOSX
567 already_AddRefed<MacIOSurfaceRecycleAllocator>
568 ImageContainer::GetMacIOSurfaceRecycleAllocator() {
569 RecursiveMutexAutoLock lock(mRecursiveMutex);
570 if (!mMacIOSurfaceRecycleAllocator) {
571 mMacIOSurfaceRecycleAllocator = new MacIOSurfaceRecycleAllocator();
574 return do_AddRef(mMacIOSurfaceRecycleAllocator);
576 #endif
578 // -
579 // https://searchfox.org/mozilla-central/source/dom/media/ipc/RemoteImageHolder.cpp#46
581 Maybe<PlanarYCbCrData> PlanarYCbCrData::From(
582 const SurfaceDescriptorBuffer& sdb) {
583 if (sdb.desc().type() != BufferDescriptor::TYCbCrDescriptor) {
584 return {};
586 const YCbCrDescriptor& yuvDesc = sdb.desc().get_YCbCrDescriptor();
588 Maybe<Range<uint8_t>> buffer;
589 const MemoryOrShmem& memOrShmem = sdb.data();
590 switch (memOrShmem.type()) {
591 case MemoryOrShmem::Tuintptr_t:
592 gfxCriticalError() << "PlanarYCbCrData::From SurfaceDescriptorBuffer "
593 "w/uintptr_t unsupported.";
594 break;
595 case MemoryOrShmem::TShmem:
596 buffer.emplace(memOrShmem.get_Shmem().Range<uint8_t>());
597 break;
598 default:
599 MOZ_ASSERT(false, "Unknown MemoryOrShmem type");
600 break;
602 if (!buffer) {
603 return {};
606 PlanarYCbCrData yuvData;
607 yuvData.mYStride = AssertedCast<int32_t>(yuvDesc.yStride());
608 yuvData.mCbCrStride = AssertedCast<int32_t>(yuvDesc.cbCrStride());
609 // default mYSkip, mCbSkip, mCrSkip because not held in YCbCrDescriptor
610 yuvData.mYSkip = yuvData.mCbSkip = yuvData.mCrSkip = 0;
611 yuvData.mPictureRect = yuvDesc.display();
612 yuvData.mStereoMode = yuvDesc.stereoMode();
613 yuvData.mColorDepth = yuvDesc.colorDepth();
614 yuvData.mYUVColorSpace = yuvDesc.yUVColorSpace();
615 yuvData.mColorRange = yuvDesc.colorRange();
616 yuvData.mChromaSubsampling = yuvDesc.chromaSubsampling();
618 const auto GetPlanePtr = [&](const uint32_t beginOffset,
619 const gfx::IntSize size,
620 const int32_t stride) -> uint8_t* {
621 if (size.width > stride) return nullptr;
622 auto bytesNeeded = CheckedInt<uintptr_t>(stride) *
623 size.height; // Don't accept `stride*(h-1)+w`.
624 bytesNeeded += beginOffset;
625 if (!bytesNeeded.isValid() || bytesNeeded.value() > buffer->length()) {
626 gfxCriticalError()
627 << "PlanarYCbCrData::From asked for out-of-bounds plane data.";
628 return nullptr;
630 return (buffer->begin() + beginOffset).get();
632 yuvData.mYChannel =
633 GetPlanePtr(yuvDesc.yOffset(), yuvDesc.ySize(), yuvData.mYStride);
634 yuvData.mCbChannel =
635 GetPlanePtr(yuvDesc.cbOffset(), yuvDesc.cbCrSize(), yuvData.mCbCrStride);
636 yuvData.mCrChannel =
637 GetPlanePtr(yuvDesc.crOffset(), yuvDesc.cbCrSize(), yuvData.mCbCrStride);
639 if (yuvData.mYSkip || yuvData.mCbSkip || yuvData.mCrSkip ||
640 yuvDesc.ySize().width < 0 || yuvDesc.ySize().height < 0 ||
641 yuvDesc.cbCrSize().width < 0 || yuvDesc.cbCrSize().height < 0 ||
642 yuvData.mYStride < 0 || yuvData.mCbCrStride < 0 || !yuvData.mYChannel ||
643 !yuvData.mCbChannel || !yuvData.mCrChannel) {
644 gfxCriticalError() << "Unusual PlanarYCbCrData: " << yuvData.mYSkip << ","
645 << yuvData.mCbSkip << "," << yuvData.mCrSkip << ", "
646 << yuvDesc.ySize().width << "," << yuvDesc.ySize().height
647 << ", " << yuvDesc.cbCrSize().width << ","
648 << yuvDesc.cbCrSize().height << ", " << yuvData.mYStride
649 << "," << yuvData.mCbCrStride << ", "
650 << yuvData.mYChannel << "," << yuvData.mCbChannel << ","
651 << yuvData.mCrChannel;
652 return {};
655 return Some(yuvData);
658 // -
660 PlanarYCbCrImage::PlanarYCbCrImage()
661 : Image(nullptr, ImageFormat::PLANAR_YCBCR),
662 mOffscreenFormat(SurfaceFormat::UNKNOWN),
663 mBufferSize(0) {}
665 nsresult PlanarYCbCrImage::BuildSurfaceDescriptorBuffer(
666 SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
667 const std::function<MemoryOrShmem(uint32_t)>& aAllocate) {
668 const PlanarYCbCrData* pdata = GetData();
669 MOZ_ASSERT(pdata, "must have PlanarYCbCrData");
670 MOZ_ASSERT(pdata->mYSkip == 0 && pdata->mCbSkip == 0 && pdata->mCrSkip == 0,
671 "YCbCrDescriptor doesn't hold skip values");
673 if (aFlags & BuildSdbFlags::RgbOnly) {
674 gfx::IntSize size(mSize);
675 auto format = gfx::ImageFormatToSurfaceFormat(GetOffscreenFormat());
676 gfx::GetYCbCrToRGBDestFormatAndSize(mData, format, size);
678 uint8_t* buffer = nullptr;
679 int32_t stride = 0;
680 nsresult rv = AllocateSurfaceDescriptorBufferRgb(
681 size, format, buffer, aSdBuffer, stride, aAllocate);
682 if (NS_WARN_IF(NS_FAILED(rv))) {
683 return rv;
686 // If we can copy directly from the surface, let's do that to avoid the YUV
687 // to RGB conversion.
688 if (mSourceSurface && mSourceSurface->GetSize() == size) {
689 DataSourceSurface::ScopedMap map(mSourceSurface, DataSourceSurface::READ);
690 if (map.IsMapped() && SwizzleData(map.GetData(), map.GetStride(),
691 mSourceSurface->GetFormat(), buffer,
692 stride, format, size)) {
693 return NS_OK;
697 gfx::ConvertYCbCrToRGB(mData, format, size, buffer, stride);
698 return NS_OK;
701 auto ySize = pdata->YDataSize();
702 auto cbcrSize = pdata->CbCrDataSize();
703 uint32_t yOffset;
704 uint32_t cbOffset;
705 uint32_t crOffset;
706 ImageDataSerializer::ComputeYCbCrOffsets(pdata->mYStride, ySize.height,
707 pdata->mCbCrStride, cbcrSize.height,
708 yOffset, cbOffset, crOffset);
710 uint32_t bufferSize = ImageDataSerializer::ComputeYCbCrBufferSize(
711 ySize, pdata->mYStride, cbcrSize, pdata->mCbCrStride, yOffset, cbOffset,
712 crOffset);
714 aSdBuffer.data() = aAllocate(bufferSize);
716 uint8_t* buffer = nullptr;
717 const MemoryOrShmem& memOrShmem = aSdBuffer.data();
718 switch (memOrShmem.type()) {
719 case MemoryOrShmem::Tuintptr_t:
720 buffer = reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t());
721 break;
722 case MemoryOrShmem::TShmem:
723 buffer = memOrShmem.get_Shmem().get<uint8_t>();
724 break;
725 default:
726 buffer = nullptr;
727 break;
729 if (!buffer) {
730 return NS_ERROR_OUT_OF_MEMORY;
733 aSdBuffer.desc() = YCbCrDescriptor(
734 pdata->mPictureRect, ySize, pdata->mYStride, cbcrSize, pdata->mCbCrStride,
735 yOffset, cbOffset, crOffset, pdata->mStereoMode, pdata->mColorDepth,
736 pdata->mYUVColorSpace, pdata->mColorRange, pdata->mChromaSubsampling);
738 CopyPlane(buffer + yOffset, pdata->mYChannel, ySize, pdata->mYStride,
739 pdata->mYSkip);
740 CopyPlane(buffer + cbOffset, pdata->mCbChannel, cbcrSize, pdata->mCbCrStride,
741 pdata->mCbSkip);
742 CopyPlane(buffer + crOffset, pdata->mCrChannel, cbcrSize, pdata->mCbCrStride,
743 pdata->mCrSkip);
744 return NS_OK;
747 RecyclingPlanarYCbCrImage::~RecyclingPlanarYCbCrImage() {
748 if (mBuffer) {
749 mRecycleBin->RecycleBuffer(std::move(mBuffer), mBufferSize);
753 size_t RecyclingPlanarYCbCrImage::SizeOfExcludingThis(
754 MallocSizeOf aMallocSizeOf) const {
755 // Ignoring:
756 // - mData - just wraps mBuffer
757 // - Surfaces should be reported under gfx-surfaces-*:
758 // - mSourceSurface
759 // - Base class:
760 // - mImplData is not used
761 // Not owned:
762 // - mRecycleBin
763 size_t size = aMallocSizeOf(mBuffer.get());
765 // Could add in the future:
766 // - mBackendData (from base class)
768 return size;
771 UniquePtr<uint8_t[]> RecyclingPlanarYCbCrImage::AllocateBuffer(uint32_t aSize) {
772 return mRecycleBin->GetBuffer(aSize);
775 static void CopyPlane(uint8_t* aDst, const uint8_t* aSrc,
776 const gfx::IntSize& aSize, int32_t aStride,
777 int32_t aSkip) {
778 int32_t height = aSize.height;
779 int32_t width = aSize.width;
781 MOZ_RELEASE_ASSERT(width <= aStride);
783 if (!aSkip) {
784 // Fast path: planar input.
785 memcpy(aDst, aSrc, height * aStride);
786 } else {
787 for (int y = 0; y < height; ++y) {
788 const uint8_t* src = aSrc;
789 uint8_t* dst = aDst;
790 // Slow path
791 for (int x = 0; x < width; ++x) {
792 *dst++ = *src++;
793 src += aSkip;
795 aSrc += aStride;
796 aDst += aStride;
801 nsresult RecyclingPlanarYCbCrImage::CopyData(const Data& aData) {
802 // update buffer size
803 // Use uint32_t throughout to match AllocateBuffer's param and mBufferSize
804 auto ySize = aData.YDataSize();
805 auto cbcrSize = aData.CbCrDataSize();
806 const auto checkedSize =
807 CheckedInt<uint32_t>(aData.mCbCrStride) * cbcrSize.height * 2 +
808 CheckedInt<uint32_t>(aData.mYStride) * ySize.height *
809 (aData.mAlpha ? 2 : 1);
811 if (!checkedSize.isValid()) return NS_ERROR_INVALID_ARG;
813 const auto size = checkedSize.value();
815 // get new buffer
816 mBuffer = AllocateBuffer(size);
817 if (!mBuffer) return NS_ERROR_OUT_OF_MEMORY;
819 // update buffer size
820 mBufferSize = size;
822 mData = aData; // mAlpha will be set if aData has it
823 mData.mYChannel = mBuffer.get();
824 mData.mCbChannel = mData.mYChannel + mData.mYStride * ySize.height;
825 mData.mCrChannel = mData.mCbChannel + mData.mCbCrStride * cbcrSize.height;
826 mData.mYSkip = mData.mCbSkip = mData.mCrSkip = 0;
828 CopyPlane(mData.mYChannel, aData.mYChannel, ySize, aData.mYStride,
829 aData.mYSkip);
830 CopyPlane(mData.mCbChannel, aData.mCbChannel, cbcrSize, aData.mCbCrStride,
831 aData.mCbSkip);
832 CopyPlane(mData.mCrChannel, aData.mCrChannel, cbcrSize, aData.mCbCrStride,
833 aData.mCrSkip);
834 if (aData.mAlpha) {
835 MOZ_ASSERT(mData.mAlpha);
836 mData.mAlpha->mChannel =
837 mData.mCrChannel + mData.mCbCrStride * cbcrSize.height;
838 CopyPlane(mData.mAlpha->mChannel, aData.mAlpha->mChannel, ySize,
839 aData.mYStride, aData.mYSkip);
842 mSize = aData.mPictureRect.Size();
843 mOrigin = aData.mPictureRect.TopLeft();
844 return NS_OK;
847 gfxImageFormat PlanarYCbCrImage::GetOffscreenFormat() const {
848 return mOffscreenFormat == SurfaceFormat::UNKNOWN ? gfxVars::OffscreenFormat()
849 : mOffscreenFormat;
852 nsresult PlanarYCbCrImage::AdoptData(const Data& aData) {
853 mData = aData;
854 mSize = aData.mPictureRect.Size();
855 mOrigin = aData.mPictureRect.TopLeft();
856 return NS_OK;
859 already_AddRefed<gfx::SourceSurface> PlanarYCbCrImage::GetAsSourceSurface() {
860 if (mSourceSurface) {
861 RefPtr<gfx::SourceSurface> surface(mSourceSurface);
862 return surface.forget();
865 gfx::IntSize size(mSize);
866 gfx::SurfaceFormat format =
867 gfx::ImageFormatToSurfaceFormat(GetOffscreenFormat());
868 gfx::GetYCbCrToRGBDestFormatAndSize(mData, format, size);
869 if (mSize.width > PlanarYCbCrImage::MAX_DIMENSION ||
870 mSize.height > PlanarYCbCrImage::MAX_DIMENSION) {
871 NS_ERROR("Illegal image dest width or height");
872 return nullptr;
875 RefPtr<gfx::DataSourceSurface> surface =
876 gfx::Factory::CreateDataSourceSurface(size, format);
877 if (NS_WARN_IF(!surface)) {
878 return nullptr;
881 DataSourceSurface::ScopedMap mapping(surface, DataSourceSurface::WRITE);
882 if (NS_WARN_IF(!mapping.IsMapped())) {
883 return nullptr;
886 gfx::ConvertYCbCrToRGB(mData, format, size, mapping.GetData(),
887 mapping.GetStride());
889 mSourceSurface = surface;
891 return surface.forget();
894 PlanarYCbCrImage::~PlanarYCbCrImage() {
895 NS_ReleaseOnMainThread("PlanarYCbCrImage::mSourceSurface",
896 mSourceSurface.forget());
899 NVImage::NVImage() : Image(nullptr, ImageFormat::NV_IMAGE), mBufferSize(0) {}
901 NVImage::~NVImage() {
902 NS_ReleaseOnMainThread("NVImage::mSourceSurface", mSourceSurface.forget());
905 IntSize NVImage::GetSize() const { return mSize; }
907 IntRect NVImage::GetPictureRect() const { return mData.mPictureRect; }
909 already_AddRefed<SourceSurface> NVImage::GetAsSourceSurface() {
910 if (mSourceSurface) {
911 RefPtr<gfx::SourceSurface> surface(mSourceSurface);
912 return surface.forget();
915 // Convert the current NV12 or NV21 data to YUV420P so that we can follow the
916 // logics in PlanarYCbCrImage::GetAsSourceSurface().
917 auto ySize = mData.YDataSize();
918 auto cbcrSize = mData.CbCrDataSize();
919 const int bufferLength =
920 ySize.height * mData.mYStride + cbcrSize.height * cbcrSize.width * 2;
921 UniquePtr<uint8_t[]> buffer(new uint8_t[bufferLength]);
923 Data aData = mData;
924 aData.mCbCrStride = cbcrSize.width;
925 aData.mCbSkip = 0;
926 aData.mCrSkip = 0;
927 aData.mYChannel = buffer.get();
928 aData.mCbChannel = aData.mYChannel + ySize.height * aData.mYStride;
929 aData.mCrChannel = aData.mCbChannel + cbcrSize.height * aData.mCbCrStride;
931 if (mData.mCbChannel < mData.mCrChannel) { // NV12
932 libyuv::NV12ToI420(mData.mYChannel, mData.mYStride, mData.mCbChannel,
933 mData.mCbCrStride, aData.mYChannel, aData.mYStride,
934 aData.mCbChannel, aData.mCbCrStride, aData.mCrChannel,
935 aData.mCbCrStride, ySize.width, ySize.height);
936 } else { // NV21
937 libyuv::NV21ToI420(mData.mYChannel, mData.mYStride, mData.mCrChannel,
938 mData.mCbCrStride, aData.mYChannel, aData.mYStride,
939 aData.mCbChannel, aData.mCbCrStride, aData.mCrChannel,
940 aData.mCbCrStride, ySize.width, ySize.height);
943 // The logics in PlanarYCbCrImage::GetAsSourceSurface().
944 gfx::IntSize size(mSize);
945 gfx::SurfaceFormat format = gfx::ImageFormatToSurfaceFormat(
946 gfxPlatform::GetPlatform()->GetOffscreenFormat());
947 gfx::GetYCbCrToRGBDestFormatAndSize(aData, format, size);
948 if (mSize.width > PlanarYCbCrImage::MAX_DIMENSION ||
949 mSize.height > PlanarYCbCrImage::MAX_DIMENSION) {
950 NS_ERROR("Illegal image dest width or height");
951 return nullptr;
954 RefPtr<gfx::DataSourceSurface> surface =
955 gfx::Factory::CreateDataSourceSurface(size, format);
956 if (NS_WARN_IF(!surface)) {
957 return nullptr;
960 DataSourceSurface::ScopedMap mapping(surface, DataSourceSurface::WRITE);
961 if (NS_WARN_IF(!mapping.IsMapped())) {
962 return nullptr;
965 gfx::ConvertYCbCrToRGB(aData, format, size, mapping.GetData(),
966 mapping.GetStride());
968 mSourceSurface = surface;
970 return surface.forget();
973 nsresult NVImage::BuildSurfaceDescriptorBuffer(
974 SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
975 const std::function<MemoryOrShmem(uint32_t)>& aAllocate) {
976 // Convert the current NV12 or NV21 data to YUV420P so that we can follow the
977 // logics in PlanarYCbCrImage::GetAsSourceSurface().
978 auto ySize = mData.YDataSize();
979 auto cbcrSize = mData.CbCrDataSize();
981 Data aData = mData;
982 aData.mCbCrStride = cbcrSize.width;
983 aData.mCbSkip = 0;
984 aData.mCrSkip = 0;
985 aData.mCbChannel = aData.mYChannel + ySize.height * aData.mYStride;
986 aData.mCrChannel = aData.mCbChannel + cbcrSize.height * aData.mCbCrStride;
988 UniquePtr<uint8_t[]> buffer;
990 if (!mSourceSurface) {
991 const int bufferLength =
992 ySize.height * mData.mYStride + cbcrSize.height * cbcrSize.width * 2;
993 buffer = MakeUnique<uint8_t[]>(bufferLength);
994 aData.mYChannel = buffer.get();
996 if (mData.mCbChannel < mData.mCrChannel) { // NV12
997 libyuv::NV12ToI420(mData.mYChannel, mData.mYStride, mData.mCbChannel,
998 mData.mCbCrStride, aData.mYChannel, aData.mYStride,
999 aData.mCbChannel, aData.mCbCrStride, aData.mCrChannel,
1000 aData.mCbCrStride, ySize.width, ySize.height);
1001 } else { // NV21
1002 libyuv::NV21ToI420(mData.mYChannel, mData.mYStride, mData.mCrChannel,
1003 mData.mCbCrStride, aData.mYChannel, aData.mYStride,
1004 aData.mCbChannel, aData.mCbCrStride, aData.mCrChannel,
1005 aData.mCbCrStride, ySize.width, ySize.height);
1009 // The logics in PlanarYCbCrImage::GetAsSourceSurface().
1010 gfx::IntSize size(mSize);
1011 gfx::SurfaceFormat format = gfx::ImageFormatToSurfaceFormat(
1012 gfxPlatform::GetPlatform()->GetOffscreenFormat());
1013 gfx::GetYCbCrToRGBDestFormatAndSize(aData, format, size);
1014 if (mSize.width > PlanarYCbCrImage::MAX_DIMENSION ||
1015 mSize.height > PlanarYCbCrImage::MAX_DIMENSION) {
1016 NS_ERROR("Illegal image dest width or height");
1017 return NS_ERROR_FAILURE;
1020 if (mSourceSurface && mSourceSurface->GetSize() != size) {
1021 return NS_ERROR_NOT_IMPLEMENTED;
1024 uint8_t* output = nullptr;
1025 int32_t stride = 0;
1026 nsresult rv = AllocateSurfaceDescriptorBufferRgb(
1027 size, format, output, aSdBuffer, stride, aAllocate);
1028 if (NS_WARN_IF(NS_FAILED(rv))) {
1029 return rv;
1032 if (!mSourceSurface) {
1033 gfx::ConvertYCbCrToRGB(aData, format, size, output, stride);
1034 return NS_OK;
1037 DataSourceSurface::ScopedMap map(mSourceSurface, DataSourceSurface::WRITE);
1038 if (NS_WARN_IF(!map.IsMapped())) {
1039 return NS_ERROR_FAILURE;
1042 if (!SwizzleData(map.GetData(), map.GetStride(), mSourceSurface->GetFormat(),
1043 output, stride, format, size)) {
1044 return NS_ERROR_FAILURE;
1047 return NS_OK;
1050 bool NVImage::IsValid() const { return !!mBufferSize; }
1052 uint32_t NVImage::GetBufferSize() const { return mBufferSize; }
1054 NVImage* NVImage::AsNVImage() { return this; };
1056 bool NVImage::SetData(const Data& aData) {
1057 MOZ_ASSERT(aData.mCbSkip == 1 && aData.mCrSkip == 1);
1058 MOZ_ASSERT((int)std::abs(aData.mCbChannel - aData.mCrChannel) == 1);
1060 // Calculate buffer size
1061 // Use uint32_t throughout to match AllocateBuffer's param and mBufferSize
1062 const auto checkedSize =
1063 CheckedInt<uint32_t>(aData.YDataSize().height) * aData.mYStride +
1064 CheckedInt<uint32_t>(aData.CbCrDataSize().height) * aData.mCbCrStride;
1066 if (!checkedSize.isValid()) return false;
1068 const auto size = checkedSize.value();
1070 // Allocate a new buffer.
1071 mBuffer = AllocateBuffer(size);
1072 if (!mBuffer) {
1073 return false;
1076 // Update mBufferSize.
1077 mBufferSize = size;
1079 // Update mData.
1080 mData = aData;
1081 mData.mYChannel = mBuffer.get();
1082 mData.mCbChannel = mData.mYChannel + (aData.mCbChannel - aData.mYChannel);
1083 mData.mCrChannel = mData.mYChannel + (aData.mCrChannel - aData.mYChannel);
1085 // Update mSize.
1086 mSize = aData.mPictureRect.Size();
1088 // Copy the input data into mBuffer.
1089 // This copies the y-channel and the interleaving CbCr-channel.
1090 memcpy(mData.mYChannel, aData.mYChannel, mBufferSize);
1092 return true;
1095 const NVImage::Data* NVImage::GetData() const { return &mData; }
1097 UniquePtr<uint8_t[]> NVImage::AllocateBuffer(uint32_t aSize) {
1098 UniquePtr<uint8_t[]> buffer(new uint8_t[aSize]);
1099 return buffer;
1102 SourceSurfaceImage::SourceSurfaceImage(const gfx::IntSize& aSize,
1103 gfx::SourceSurface* aSourceSurface)
1104 : Image(nullptr, ImageFormat::MOZ2D_SURFACE),
1105 mSize(aSize),
1106 mSourceSurface(aSourceSurface),
1107 mTextureFlags(TextureFlags::DEFAULT) {}
1109 SourceSurfaceImage::SourceSurfaceImage(gfx::SourceSurface* aSourceSurface)
1110 : Image(nullptr, ImageFormat::MOZ2D_SURFACE),
1111 mSize(aSourceSurface->GetSize()),
1112 mSourceSurface(aSourceSurface),
1113 mTextureFlags(TextureFlags::DEFAULT) {}
1115 SourceSurfaceImage::~SourceSurfaceImage() {
1116 NS_ReleaseOnMainThread("SourceSurfaceImage::mSourceSurface",
1117 mSourceSurface.forget());
1120 TextureClient* SourceSurfaceImage::GetTextureClient(
1121 KnowsCompositor* aKnowsCompositor) {
1122 if (!aKnowsCompositor) {
1123 return nullptr;
1126 return mTextureClients.WithEntryHandle(
1127 aKnowsCompositor->GetSerial(), [&](auto&& entry) -> TextureClient* {
1128 if (entry) {
1129 return entry->get();
1132 RefPtr<TextureClient> textureClient;
1133 RefPtr<SourceSurface> surface = GetAsSourceSurface();
1134 MOZ_ASSERT(surface);
1135 if (surface) {
1136 // gfx::BackendType::NONE means default to content backend
1137 textureClient = TextureClient::CreateFromSurface(
1138 aKnowsCompositor, surface, BackendSelector::Content,
1139 mTextureFlags, ALLOC_DEFAULT);
1141 if (textureClient) {
1142 textureClient->SyncWithObject(aKnowsCompositor->GetSyncObject());
1143 return entry.Insert(std::move(textureClient)).get();
1146 return nullptr;
1150 ImageContainer::ProducerID ImageContainer::AllocateProducerID() {
1151 // Callable on all threads.
1152 static Atomic<ImageContainer::ProducerID> sProducerID(0u);
1153 return ++sProducerID;
1156 } // namespace mozilla::layers