1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "mozilla/dom/ImageBitmap.h"
8 #include "mozilla/CheckedInt.h"
9 #include "mozilla/dom/BlobImpl.h"
10 #include "mozilla/dom/CanvasRenderingContext2D.h"
11 #include "mozilla/dom/CanvasUtils.h"
12 #include "mozilla/dom/Document.h"
13 #include "mozilla/dom/HTMLCanvasElement.h"
14 #include "mozilla/dom/HTMLImageElement.h"
15 #include "mozilla/dom/HTMLMediaElementBinding.h"
16 #include "mozilla/dom/HTMLVideoElement.h"
17 #include "mozilla/dom/ImageBitmapBinding.h"
18 #include "mozilla/dom/OffscreenCanvas.h"
19 #include "mozilla/dom/Promise.h"
20 #include "mozilla/dom/StructuredCloneTags.h"
21 #include "mozilla/dom/SVGImageElement.h"
22 #include "mozilla/dom/WorkerPrivate.h"
23 #include "mozilla/dom/WorkerRef.h"
24 #include "mozilla/dom/WorkerRunnable.h"
25 #include "mozilla/dom/VideoFrame.h"
26 #include "mozilla/gfx/2D.h"
27 #include "mozilla/gfx/Logging.h"
28 #include "mozilla/gfx/Scale.h"
29 #include "mozilla/gfx/Swizzle.h"
30 #include "mozilla/Mutex.h"
31 #include "mozilla/ScopeExit.h"
32 #include "nsGlobalWindowInner.h"
33 #include "nsIAsyncInputStream.h"
34 #include "nsNetUtil.h"
35 #include "nsLayoutUtils.h"
36 #include "nsStreamUtils.h"
37 #include "imgLoader.h"
40 using namespace mozilla::gfx
;
41 using namespace mozilla::layers
;
42 using mozilla::dom::HTMLMediaElement_Binding::HAVE_METADATA
;
43 using mozilla::dom::HTMLMediaElement_Binding::NETWORK_EMPTY
;
45 namespace mozilla::dom
{
47 NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(ImageBitmap
, mParent
)
48 NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageBitmap
)
49 NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageBitmap
)
50 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageBitmap
)
51 NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
52 NS_INTERFACE_MAP_ENTRY(nsISupports
)
55 class ImageBitmapShutdownObserver
;
57 static StaticMutex sShutdownMutex
;
58 static ImageBitmapShutdownObserver
* sShutdownObserver
= nullptr;
60 class SendShutdownToWorkerThread
: public MainThreadWorkerControlRunnable
{
62 explicit SendShutdownToWorkerThread(ImageBitmap
* aImageBitmap
)
63 : MainThreadWorkerControlRunnable(GetCurrentThreadWorkerPrivate()),
64 mImageBitmap(aImageBitmap
) {
65 MOZ_ASSERT(GetCurrentThreadWorkerPrivate());
68 bool WorkerRun(JSContext
* aCx
, WorkerPrivate
* aWorkerPrivate
) override
{
70 mImageBitmap
->OnShutdown();
71 mImageBitmap
= nullptr;
76 ImageBitmap
* mImageBitmap
;
79 /* This class observes shutdown notifications and sends that notification
80 * to the worker thread if the image bitmap is on a worker thread.
82 class ImageBitmapShutdownObserver final
: public nsIObserver
{
84 explicit ImageBitmapShutdownObserver() {
85 sShutdownMutex
.AssertCurrentThreadOwns();
86 if (NS_IsMainThread()) {
91 WorkerPrivate
* workerPrivate
= GetCurrentThreadWorkerPrivate();
92 MOZ_ASSERT(workerPrivate
);
93 auto* mainThreadEventTarget
= workerPrivate
->MainThreadEventTarget();
94 MOZ_ASSERT(mainThreadEventTarget
);
95 RefPtr
<ImageBitmapShutdownObserver
> self
= this;
96 nsCOMPtr
<nsIRunnable
> r
=
97 NS_NewRunnableFunction("ImageBitmapShutdownObserver::RegisterObserver",
98 [self
]() { self
->RegisterObserver(); });
99 mainThreadEventTarget
->Dispatch(r
.forget());
102 void RegisterObserver() {
103 MOZ_ASSERT(NS_IsMainThread());
104 nsContentUtils::RegisterShutdownObserver(this);
107 already_AddRefed
<SendShutdownToWorkerThread
> Track(
108 ImageBitmap
* aImageBitmap
) {
109 sShutdownMutex
.AssertCurrentThreadOwns();
110 MOZ_ASSERT(!mBitmaps
.Contains(aImageBitmap
));
112 RefPtr
<SendShutdownToWorkerThread
> runnable
= nullptr;
113 if (!NS_IsMainThread()) {
114 runnable
= new SendShutdownToWorkerThread(aImageBitmap
);
117 RefPtr
<SendShutdownToWorkerThread
> retval
= runnable
;
118 mBitmaps
.Insert(aImageBitmap
);
119 return retval
.forget();
122 void Untrack(ImageBitmap
* aImageBitmap
) {
123 sShutdownMutex
.AssertCurrentThreadOwns();
124 MOZ_ASSERT(mBitmaps
.Contains(aImageBitmap
));
126 mBitmaps
.Remove(aImageBitmap
);
129 NS_DECL_THREADSAFE_ISUPPORTS
132 ~ImageBitmapShutdownObserver() = default;
134 nsTHashSet
<ImageBitmap
*> mBitmaps
;
137 NS_IMPL_ISUPPORTS(ImageBitmapShutdownObserver
, nsIObserver
)
140 ImageBitmapShutdownObserver::Observe(nsISupports
* aSubject
, const char* aTopic
,
141 const char16_t
* aData
) {
142 if (strcmp(aTopic
, NS_XPCOM_SHUTDOWN_OBSERVER_ID
) == 0) {
143 StaticMutexAutoLock
lock(sShutdownMutex
);
145 for (const auto& bitmap
: mBitmaps
) {
146 const auto& runnable
= bitmap
->mShutdownRunnable
;
148 runnable
->Dispatch();
150 bitmap
->OnShutdown();
154 nsContentUtils::UnregisterShutdownObserver(this);
156 sShutdownObserver
= nullptr;
163 * If either aRect.width or aRect.height are negative, then return a new IntRect
164 * which represents the same rectangle as the aRect does but with positive width
167 static IntRect
FixUpNegativeDimension(const IntRect
& aRect
, ErrorResult
& aRv
) {
168 gfx::IntRect rect
= aRect
;
170 // fix up negative dimensions
171 if (rect
.width
< 0) {
172 CheckedInt32 checkedX
= CheckedInt32(rect
.x
) + rect
.width
;
174 if (!checkedX
.isValid()) {
175 aRv
.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR
);
179 rect
.x
= checkedX
.value();
180 rect
.width
= -(rect
.width
);
183 if (rect
.height
< 0) {
184 CheckedInt32 checkedY
= CheckedInt32(rect
.y
) + rect
.height
;
186 if (!checkedY
.isValid()) {
187 aRv
.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR
);
191 rect
.y
= checkedY
.value();
192 rect
.height
= -(rect
.height
);
199 * This helper function copies the data of the given DataSourceSurface,
200 * _aSurface_, in the given area, _aCropRect_, into a new DataSourceSurface.
201 * This might return null if it can not create a new SourceSurface or it cannot
202 * read data from the given _aSurface_.
204 * Warning: Even though the area of _aCropRect_ is just the same as the size of
205 * _aSurface_, this function still copy data into a new
208 static already_AddRefed
<DataSourceSurface
> CropAndCopyDataSourceSurface(
209 DataSourceSurface
* aSurface
, const IntRect
& aCropRect
) {
210 MOZ_ASSERT(aSurface
);
212 // Check the aCropRect
214 const IntRect positiveCropRect
= FixUpNegativeDimension(aCropRect
, error
);
215 if (NS_WARN_IF(error
.Failed())) {
216 error
.SuppressException();
220 // Calculate the size of the new SourceSurface.
221 // We cannot keep using aSurface->GetFormat() to create new DataSourceSurface,
222 // since it might be SurfaceFormat::B8G8R8X8 which does not handle opacity,
223 // however the specification explicitly define that "If any of the pixels on
224 // this rectangle are outside the area where the input bitmap was placed, then
225 // they will be transparent black in output."
226 // So, instead, we force the output format to be SurfaceFormat::B8G8R8A8.
227 const SurfaceFormat format
= SurfaceFormat::B8G8R8A8
;
228 const int bytesPerPixel
= BytesPerPixel(format
);
229 const IntSize dstSize
=
230 IntSize(positiveCropRect
.width
, positiveCropRect
.height
);
231 const uint32_t dstStride
= dstSize
.width
* bytesPerPixel
;
233 // Create a new SourceSurface.
234 RefPtr
<DataSourceSurface
> dstDataSurface
=
235 Factory::CreateDataSourceSurfaceWithStride(dstSize
, format
, dstStride
,
238 if (NS_WARN_IF(!dstDataSurface
)) {
242 // Only do copying and cropping when the positiveCropRect intersects with
243 // the size of aSurface.
244 const IntRect
surfRect(IntPoint(0, 0), aSurface
->GetSize());
245 if (surfRect
.Intersects(positiveCropRect
)) {
246 const IntRect surfPortion
= surfRect
.Intersect(positiveCropRect
);
247 const IntPoint
dest(std::max(0, surfPortion
.X() - positiveCropRect
.X()),
248 std::max(0, surfPortion
.Y() - positiveCropRect
.Y()));
250 // Copy the raw data into the newly created DataSourceSurface.
251 DataSourceSurface::ScopedMap
srcMap(aSurface
, DataSourceSurface::READ
);
252 DataSourceSurface::ScopedMap
dstMap(dstDataSurface
,
253 DataSourceSurface::WRITE
);
254 if (NS_WARN_IF(!srcMap
.IsMapped()) || NS_WARN_IF(!dstMap
.IsMapped())) {
258 uint8_t* srcBufferPtr
= srcMap
.GetData() +
259 surfPortion
.y
* srcMap
.GetStride() +
260 surfPortion
.x
* bytesPerPixel
;
261 uint8_t* dstBufferPtr
=
262 dstMap
.GetData() + dest
.y
* dstMap
.GetStride() + dest
.x
* bytesPerPixel
;
263 CheckedInt
<uint32_t> copiedBytesPerRaw
=
264 CheckedInt
<uint32_t>(surfPortion
.width
) * bytesPerPixel
;
265 if (!copiedBytesPerRaw
.isValid()) {
269 for (int i
= 0; i
< surfPortion
.height
; ++i
) {
270 memcpy(dstBufferPtr
, srcBufferPtr
, copiedBytesPerRaw
.value());
271 srcBufferPtr
+= srcMap
.GetStride();
272 dstBufferPtr
+= dstMap
.GetStride();
276 return dstDataSurface
.forget();
280 * This helper function scales the data of the given DataSourceSurface,
281 * _aSurface_, in the given area, _aCropRect_, into a new DataSourceSurface.
282 * This might return null if it can not create a new SourceSurface or it cannot
283 * read data from the given _aSurface_.
286 static already_AddRefed
<DataSourceSurface
> ScaleDataSourceSurface(
287 DataSourceSurface
* aSurface
, const ImageBitmapOptions
& aOptions
) {
288 MOZ_ASSERT(aSurface
);
290 const SurfaceFormat format
= aSurface
->GetFormat();
291 const int bytesPerPixel
= BytesPerPixel(format
);
293 const IntSize srcSize
= aSurface
->GetSize();
296 CheckedInt
<int32_t> checked
;
297 CheckedInt
<int32_t> dstWidth(
298 aOptions
.mResizeWidth
.WasPassed() ? aOptions
.mResizeWidth
.Value() : 0);
299 CheckedInt
<int32_t> dstHeight(
300 aOptions
.mResizeHeight
.WasPassed() ? aOptions
.mResizeHeight
.Value() : 0);
302 if (!dstWidth
.isValid() || !dstHeight
.isValid()) {
306 if (!dstWidth
.value()) {
307 checked
= srcSize
.width
* dstHeight
;
308 if (!checked
.isValid()) {
312 tmp
= ceil(checked
.value() / double(srcSize
.height
));
314 } else if (!dstHeight
.value()) {
315 checked
= srcSize
.height
* dstWidth
;
316 if (!checked
.isValid()) {
320 tmp
= ceil(checked
.value() / double(srcSize
.width
));
324 const IntSize
dstSize(dstWidth
.value(), dstHeight
.value());
325 const int32_t dstStride
= dstSize
.width
* bytesPerPixel
;
327 // Create a new SourceSurface.
328 RefPtr
<DataSourceSurface
> dstDataSurface
=
329 Factory::CreateDataSourceSurfaceWithStride(dstSize
, format
, dstStride
,
332 if (NS_WARN_IF(!dstDataSurface
)) {
336 // Copy the raw data into the newly created DataSourceSurface.
337 DataSourceSurface::ScopedMap
srcMap(aSurface
, DataSourceSurface::READ
);
338 DataSourceSurface::ScopedMap
dstMap(dstDataSurface
, DataSourceSurface::WRITE
);
339 if (NS_WARN_IF(!srcMap
.IsMapped()) || NS_WARN_IF(!dstMap
.IsMapped())) {
343 uint8_t* srcBufferPtr
= srcMap
.GetData();
344 uint8_t* dstBufferPtr
= dstMap
.GetData();
346 bool res
= Scale(srcBufferPtr
, srcSize
.width
, srcSize
.height
,
347 srcMap
.GetStride(), dstBufferPtr
, dstSize
.width
,
348 dstSize
.height
, dstMap
.GetStride(), aSurface
->GetFormat());
353 return dstDataSurface
.forget();
356 static DataSourceSurface
* FlipYDataSourceSurface(DataSourceSurface
* aSurface
) {
357 MOZ_ASSERT(aSurface
);
359 // Invert in y direction.
360 DataSourceSurface::ScopedMap
srcMap(aSurface
, DataSourceSurface::READ_WRITE
);
361 if (NS_WARN_IF(!srcMap
.IsMapped())) {
365 const IntSize srcSize
= aSurface
->GetSize();
366 uint8_t* srcBufferPtr
= srcMap
.GetData();
367 const uint32_t stride
= srcMap
.GetStride();
369 CheckedInt
<uint32_t> copiedBytesPerRaw
= CheckedInt
<uint32_t>(stride
);
370 if (!copiedBytesPerRaw
.isValid()) {
374 for (int i
= 0; i
< srcSize
.height
/ 2; ++i
) {
375 std::swap_ranges(srcBufferPtr
+ stride
* i
, srcBufferPtr
+ stride
* (i
+ 1),
376 srcBufferPtr
+ stride
* (srcSize
.height
- 1 - i
));
382 static DataSourceSurface
* AlphaPremultiplyDataSourceSurface(
383 DataSourceSurface
* aSurface
, const bool forward
= true) {
384 MOZ_ASSERT(aSurface
);
386 DataSourceSurface::MappedSurface surfaceMap
;
388 if (aSurface
->Map(DataSourceSurface::MapType::READ_WRITE
, &surfaceMap
)) {
390 PremultiplyData(surfaceMap
.mData
, surfaceMap
.mStride
,
391 aSurface
->GetFormat(), surfaceMap
.mData
,
392 surfaceMap
.mStride
, aSurface
->GetFormat(),
393 aSurface
->GetSize());
395 UnpremultiplyData(surfaceMap
.mData
, surfaceMap
.mStride
,
396 aSurface
->GetFormat(), surfaceMap
.mData
,
397 surfaceMap
.mStride
, aSurface
->GetFormat(),
398 aSurface
->GetSize());
410 * Encapsulate the given _aSurface_ into a layers::SourceSurfaceImage.
412 static already_AddRefed
<layers::Image
> CreateImageFromSurface(
413 SourceSurface
* aSurface
) {
414 MOZ_ASSERT(aSurface
);
415 RefPtr
<layers::SourceSurfaceImage
> image
=
416 new layers::SourceSurfaceImage(aSurface
->GetSize(), aSurface
);
417 return image
.forget();
421 * CreateImageFromRawData(), CreateSurfaceFromRawData() and
422 * CreateImageFromRawDataInMainThreadSyncTask are helpers for
423 * create-from-ImageData case
425 static already_AddRefed
<SourceSurface
> CreateSurfaceFromRawData(
426 const gfx::IntSize
& aSize
, uint32_t aStride
, gfx::SurfaceFormat aFormat
,
427 uint8_t* aBuffer
, uint32_t aBufferLength
, const Maybe
<IntRect
>& aCropRect
,
428 const ImageBitmapOptions
& aOptions
) {
429 MOZ_ASSERT(!aSize
.IsEmpty());
432 // Wrap the source buffer into a SourceSurface.
433 RefPtr
<DataSourceSurface
> dataSurface
=
434 Factory::CreateWrappingDataSourceSurface(aBuffer
, aStride
, aSize
,
437 if (NS_WARN_IF(!dataSurface
)) {
441 // The temporary cropRect variable is equal to the size of source buffer if we
442 // do not need to crop, or it equals to the given cropping size.
443 const IntRect cropRect
=
444 aCropRect
.valueOr(IntRect(0, 0, aSize
.width
, aSize
.height
));
446 // Copy the source buffer in the _cropRect_ area into a new SourceSurface.
447 RefPtr
<DataSourceSurface
> result
=
448 CropAndCopyDataSourceSurface(dataSurface
, cropRect
);
449 if (NS_WARN_IF(!result
)) {
453 if (aOptions
.mImageOrientation
== ImageOrientation::FlipY
) {
454 result
= FlipYDataSourceSurface(result
);
456 if (NS_WARN_IF(!result
)) {
461 if (aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::Premultiply
) {
462 result
= AlphaPremultiplyDataSourceSurface(result
);
464 if (NS_WARN_IF(!result
)) {
469 if (aOptions
.mResizeWidth
.WasPassed() || aOptions
.mResizeHeight
.WasPassed()) {
470 dataSurface
= result
->GetDataSurface();
471 result
= ScaleDataSourceSurface(dataSurface
, aOptions
);
472 if (NS_WARN_IF(!result
)) {
477 return result
.forget();
480 static already_AddRefed
<layers::Image
> CreateImageFromRawData(
481 const gfx::IntSize
& aSize
, uint32_t aStride
, gfx::SurfaceFormat aFormat
,
482 uint8_t* aBuffer
, uint32_t aBufferLength
, const Maybe
<IntRect
>& aCropRect
,
483 const ImageBitmapOptions
& aOptions
) {
484 MOZ_ASSERT(NS_IsMainThread());
486 // Copy and crop the source buffer into a SourceSurface.
487 RefPtr
<SourceSurface
> rgbaSurface
= CreateSurfaceFromRawData(
488 aSize
, aStride
, aFormat
, aBuffer
, aBufferLength
, aCropRect
, aOptions
);
490 if (NS_WARN_IF(!rgbaSurface
)) {
494 // Convert RGBA to BGRA
495 RefPtr
<DataSourceSurface
> rgbaDataSurface
= rgbaSurface
->GetDataSurface();
496 DataSourceSurface::ScopedMap
rgbaMap(rgbaDataSurface
,
497 DataSourceSurface::READ
);
498 if (NS_WARN_IF(!rgbaMap
.IsMapped())) {
502 RefPtr
<DataSourceSurface
> bgraDataSurface
=
503 Factory::CreateDataSourceSurfaceWithStride(rgbaDataSurface
->GetSize(),
504 SurfaceFormat::B8G8R8A8
,
505 rgbaMap
.GetStride());
506 if (NS_WARN_IF(!bgraDataSurface
)) {
510 DataSourceSurface::ScopedMap
bgraMap(bgraDataSurface
,
511 DataSourceSurface::WRITE
);
512 if (NS_WARN_IF(!bgraMap
.IsMapped())) {
516 SwizzleData(rgbaMap
.GetData(), rgbaMap
.GetStride(), SurfaceFormat::R8G8B8A8
,
517 bgraMap
.GetData(), bgraMap
.GetStride(), SurfaceFormat::B8G8R8A8
,
518 bgraDataSurface
->GetSize());
520 // Create an Image from the BGRA SourceSurface.
521 return CreateImageFromSurface(bgraDataSurface
);
525 * This is a synchronous task.
526 * This class is used to create a layers::SourceSurfaceImage from raw data in
527 * the main thread. While creating an ImageBitmap from an ImageData, we need to
528 * create a SouceSurface from the ImageData's raw data and then set the
529 * SourceSurface into a layers::SourceSurfaceImage. However, the
530 * layers::SourceSurfaceImage asserts the setting operation in the main thread,
531 * so if we are going to create an ImageBitmap from an ImageData off the main
532 * thread, we post an event to the main thread to create a
533 * layers::SourceSurfaceImage from an ImageData's raw data.
535 class CreateImageFromRawDataInMainThreadSyncTask final
536 : public WorkerMainThreadRunnable
{
538 CreateImageFromRawDataInMainThreadSyncTask(
539 uint8_t* aBuffer
, uint32_t aBufferLength
, uint32_t aStride
,
540 gfx::SurfaceFormat aFormat
, const gfx::IntSize
& aSize
,
541 const Maybe
<IntRect
>& aCropRect
, layers::Image
** aImage
,
542 const ImageBitmapOptions
& aOptions
)
543 : WorkerMainThreadRunnable(
544 GetCurrentThreadWorkerPrivate(),
545 "ImageBitmap :: Create Image from Raw Data"_ns
),
548 mBufferLength(aBufferLength
),
552 mCropRect(aCropRect
),
554 MOZ_ASSERT(!(*aImage
),
555 "Don't pass an existing Image into "
556 "CreateImageFromRawDataInMainThreadSyncTask.");
559 bool MainThreadRun() override
{
560 RefPtr
<layers::Image
> image
= CreateImageFromRawData(
561 mSize
, mStride
, mFormat
, mBuffer
, mBufferLength
, mCropRect
, mOptions
);
563 if (NS_WARN_IF(!image
)) {
567 image
.forget(mImage
);
573 layers::Image
** mImage
;
575 uint32_t mBufferLength
;
577 gfx::SurfaceFormat mFormat
;
579 const Maybe
<IntRect
>& mCropRect
;
580 const ImageBitmapOptions mOptions
;
584 * A wrapper to the nsLayoutUtils::SurfaceFromElement() function followed by the
587 template <class ElementType
>
588 static already_AddRefed
<SourceSurface
> GetSurfaceFromElement(
589 nsIGlobalObject
* aGlobal
, ElementType
& aElement
, bool* aWriteOnly
,
590 const ImageBitmapOptions
& aOptions
, gfxAlphaType
* aAlphaType
,
592 uint32_t flags
= nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE
|
593 nsLayoutUtils::SFE_ORIENTATION_FROM_IMAGE
|
594 nsLayoutUtils::SFE_EXACT_SIZE_SURFACE
;
596 // by default surfaces have premultiplied alpha
597 // attempt to get non premultiplied if required
598 if (aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
599 flags
|= nsLayoutUtils::SFE_ALLOW_NON_PREMULT
;
602 if (aOptions
.mColorSpaceConversion
== ColorSpaceConversion::None
&&
603 aElement
.IsHTMLElement(nsGkAtoms::img
)) {
604 flags
|= nsLayoutUtils::SFE_NO_COLORSPACE_CONVERSION
;
607 Maybe
<int32_t> resizeWidth
, resizeHeight
;
608 if (aOptions
.mResizeWidth
.WasPassed()) {
609 if (!CheckedInt32(aOptions
.mResizeWidth
.Value()).isValid()) {
610 aRv
.ThrowInvalidStateError("resizeWidth is too large");
613 resizeWidth
.emplace(aOptions
.mResizeWidth
.Value());
615 if (aOptions
.mResizeHeight
.WasPassed()) {
616 if (!CheckedInt32(aOptions
.mResizeHeight
.Value()).isValid()) {
617 aRv
.ThrowInvalidStateError("resizeHeight is too large");
620 resizeHeight
.emplace(aOptions
.mResizeHeight
.Value());
622 SurfaceFromElementResult res
= nsLayoutUtils::SurfaceFromElement(
623 &aElement
, resizeWidth
, resizeHeight
, flags
);
625 RefPtr
<SourceSurface
> surface
= res
.GetSourceSurface();
626 if (NS_WARN_IF(!surface
)) {
627 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
631 *aWriteOnly
= res
.mIsWriteOnly
;
632 *aAlphaType
= res
.mAlphaType
;
634 return surface
.forget();
637 ImageBitmap::ImageBitmap(nsIGlobalObject
* aGlobal
, layers::Image
* aData
,
638 bool aWriteOnly
, gfxAlphaType aAlphaType
)
642 mPictureRect(aData
->GetPictureRect()),
643 mAlphaType(aAlphaType
),
644 mAllocatedImageData(false),
645 mWriteOnly(aWriteOnly
) {
646 MOZ_ASSERT(aData
, "aData is null in ImageBitmap constructor.");
648 StaticMutexAutoLock
lock(sShutdownMutex
);
649 if (!sShutdownObserver
&&
650 !AppShutdown::IsInOrBeyond(ShutdownPhase::XPCOMShutdown
)) {
651 sShutdownObserver
= new ImageBitmapShutdownObserver();
653 if (sShutdownObserver
) {
654 mShutdownRunnable
= sShutdownObserver
->Track(this);
658 ImageBitmap::~ImageBitmap() {
659 StaticMutexAutoLock
lock(sShutdownMutex
);
660 if (mShutdownRunnable
) {
661 mShutdownRunnable
->mImageBitmap
= nullptr;
663 mShutdownRunnable
= nullptr;
664 if (sShutdownObserver
) {
665 sShutdownObserver
->Untrack(this);
669 JSObject
* ImageBitmap::WrapObject(JSContext
* aCx
,
670 JS::Handle
<JSObject
*> aGivenProto
) {
671 return ImageBitmap_Binding::Wrap(aCx
, this, aGivenProto
);
674 void ImageBitmap::Close() {
677 mPictureRect
.SetEmpty();
680 void ImageBitmap::OnShutdown() { Close(); }
682 void ImageBitmap::SetPictureRect(const IntRect
& aRect
, ErrorResult
& aRv
) {
683 mPictureRect
= FixUpNegativeDimension(aRect
, aRv
);
686 SurfaceFromElementResult
ImageBitmap::SurfaceFrom(uint32_t aSurfaceFlags
) {
687 SurfaceFromElementResult sfer
;
693 // An ImageBitmap, not being a DOM element, only has `origin-clean`
694 // (via our `IsWriteOnly`), and does not participate in CORS.
695 // Right now we mark this by setting mCORSUsed to true.
696 sfer
.mCORSUsed
= true;
697 sfer
.mIsWriteOnly
= mWriteOnly
;
700 sfer
.mPrincipal
= mParent
->PrincipalOrNull();
703 IntSize
imageSize(mData
->GetSize());
704 IntRect
imageRect(IntPoint(0, 0), imageSize
);
705 bool hasCropRect
= mPictureRect
.IsEqualEdges(imageRect
);
708 bool(aSurfaceFlags
& nsLayoutUtils::SFE_EXACT_SIZE_SURFACE
);
709 bool allowNonPremult
=
710 bool(aSurfaceFlags
& nsLayoutUtils::SFE_ALLOW_NON_PREMULT
);
711 bool allowUncropped
=
712 bool(aSurfaceFlags
& nsLayoutUtils::SFE_ALLOW_UNCROPPED_UNSCALED
);
713 bool requiresPremult
=
714 !allowNonPremult
&& mAlphaType
== gfxAlphaType::NonPremult
;
715 bool requiresCrop
= !allowUncropped
&& hasCropRect
;
716 if (wantExactSize
|| requiresPremult
|| requiresCrop
|| mSurface
) {
717 RefPtr
<DrawTarget
> dt
= Factory::CreateDrawTarget(
718 BackendType::SKIA
, IntSize(1, 1), SurfaceFormat::B8G8R8A8
);
719 sfer
.mSourceSurface
= PrepareForDrawTarget(dt
);
721 if (!sfer
.mSourceSurface
) {
725 MOZ_ASSERT(mSurface
);
727 sfer
.mSize
= sfer
.mIntrinsicSize
= sfer
.mSourceSurface
->GetSize();
728 sfer
.mHasSize
= true;
729 sfer
.mAlphaType
= IsOpaque(sfer
.mSourceSurface
->GetFormat())
730 ? gfxAlphaType::Opaque
731 : gfxAlphaType::Premult
;
736 IntRect imagePortion
= imageRect
.Intersect(mPictureRect
);
738 // the crop lies entirely outside the surface area, nothing to draw
739 if (imagePortion
.IsEmpty()) {
743 sfer
.mCropRect
= Some(imagePortion
);
744 sfer
.mIntrinsicSize
= imagePortion
.Size();
746 sfer
.mIntrinsicSize
= imageSize
;
749 sfer
.mSize
= imageSize
;
750 sfer
.mHasSize
= true;
751 sfer
.mAlphaType
= mAlphaType
;
752 sfer
.mLayersImage
= mData
;
757 * The functionality of PrepareForDrawTarget method:
758 * (1) Get a SourceSurface from the mData (which is a layers::Image).
759 * (2) Convert the SourceSurface to format B8G8R8A8 if the original format is
760 * R8G8B8, B8G8R8, HSV or Lab.
761 * Note: if the original format is A8 or Depth, then return null directly.
762 * (3) Do cropping if the size of SourceSurface does not equal to the
764 * (4) Pre-multiply alpha if needed.
766 already_AddRefed
<SourceSurface
> ImageBitmap::PrepareForDrawTarget(
767 gfx::DrawTarget
* aTarget
) {
775 mSurface
= mData
->GetAsSourceSurface();
782 IntRect
surfRect(0, 0, mSurface
->GetSize().width
, mSurface
->GetSize().height
);
784 // Check if we still need to crop our surface
785 if (!mPictureRect
.IsEqualEdges(surfRect
)) {
786 IntRect surfPortion
= surfRect
.Intersect(mPictureRect
);
788 // the crop lies entirely outside the surface area, nothing to draw
789 if (surfPortion
.IsEmpty()) {
794 IntPoint
dest(std::max(0, surfPortion
.X() - mPictureRect
.X()),
795 std::max(0, surfPortion
.Y() - mPictureRect
.Y()));
797 // We must initialize this target with mPictureRect.Size() because the
798 // specification states that if the cropping area is given, then return an
799 // ImageBitmap with the size equals to the cropping area. Ensure that the
800 // format matches the surface, even though the DT type is similar to the
801 // destination, i.e. blending an alpha surface to an opaque DT. However,
802 // any pixels outside the surface portion must be filled with transparent
803 // black, even if the surface is opaque, so force to an alpha format in
805 SurfaceFormat format
= mSurface
->GetFormat();
806 if (!surfPortion
.IsEqualEdges(mPictureRect
) && IsOpaque(format
)) {
807 format
= SurfaceFormat::B8G8R8A8
;
809 RefPtr
<DrawTarget
> cropped
=
810 aTarget
->CreateSimilarDrawTarget(mPictureRect
.Size(), format
);
816 cropped
->CopySurface(mSurface
, surfPortion
, dest
);
817 mSurface
= cropped
->Snapshot();
822 // Make mCropRect match new surface we've cropped to
823 mPictureRect
.MoveTo(0, 0);
826 // Pre-multiply alpha here.
827 // Ignore this step if the source surface does not have alpha channel; this
828 // kind of source surfaces might come form layers::PlanarYCbCrImage.
829 if (mAlphaType
== gfxAlphaType::NonPremult
&&
830 !IsOpaque(mSurface
->GetFormat())) {
831 MOZ_ASSERT(mSurface
->GetFormat() == SurfaceFormat::R8G8B8A8
||
832 mSurface
->GetFormat() == SurfaceFormat::B8G8R8A8
||
833 mSurface
->GetFormat() == SurfaceFormat::A8R8G8B8
);
835 RefPtr
<DataSourceSurface
> srcSurface
= mSurface
->GetDataSurface();
836 if (NS_WARN_IF(!srcSurface
)) {
839 RefPtr
<DataSourceSurface
> dstSurface
= Factory::CreateDataSourceSurface(
840 srcSurface
->GetSize(), srcSurface
->GetFormat());
841 if (NS_WARN_IF(!dstSurface
)) {
845 DataSourceSurface::ScopedMap
srcMap(srcSurface
, DataSourceSurface::READ
);
846 if (!srcMap
.IsMapped()) {
848 << "Failed to map source surface for premultiplying alpha.";
852 DataSourceSurface::ScopedMap
dstMap(dstSurface
, DataSourceSurface::WRITE
);
853 if (!dstMap
.IsMapped()) {
855 << "Failed to map destination surface for premultiplying alpha.";
859 PremultiplyData(srcMap
.GetData(), srcMap
.GetStride(), mSurface
->GetFormat(),
860 dstMap
.GetData(), dstMap
.GetStride(), mSurface
->GetFormat(),
861 dstSurface
->GetSize());
863 mAlphaType
= gfxAlphaType::Premult
;
864 mSurface
= dstSurface
;
867 // Replace our surface with one optimized for the target we're about to draw
868 // to, under the assumption it'll likely be drawn again to that target.
869 // This call should be a no-op for already-optimized surfaces
870 mSurface
= aTarget
->OptimizeSourceSurface(mSurface
);
871 return do_AddRef(mSurface
);
874 already_AddRefed
<layers::Image
> ImageBitmap::TransferAsImage() {
875 RefPtr
<layers::Image
> image
= mData
;
877 return image
.forget();
880 UniquePtr
<ImageBitmapCloneData
> ImageBitmap::ToCloneData() const {
882 // A closed image cannot be cloned.
886 UniquePtr
<ImageBitmapCloneData
> result(new ImageBitmapCloneData());
887 result
->mPictureRect
= mPictureRect
;
888 result
->mAlphaType
= mAlphaType
;
889 RefPtr
<SourceSurface
> surface
= mData
->GetAsSourceSurface();
891 // It might just not be possible to get/map the surface. (e.g. from another
896 result
->mSurface
= surface
->GetDataSurface();
897 MOZ_ASSERT(result
->mSurface
);
898 result
->mWriteOnly
= mWriteOnly
;
904 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateFromSourceSurface(
905 nsIGlobalObject
* aGlobal
, gfx::SourceSurface
* aSource
, ErrorResult
& aRv
) {
906 RefPtr
<layers::Image
> data
= CreateImageFromSurface(aSource
);
907 RefPtr
<ImageBitmap
> ret
=
908 new ImageBitmap(aGlobal
, data
, false /* writeOnly */);
909 ret
->mAllocatedImageData
= true;
914 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateFromCloneData(
915 nsIGlobalObject
* aGlobal
, ImageBitmapCloneData
* aData
) {
916 RefPtr
<layers::Image
> data
= CreateImageFromSurface(aData
->mSurface
);
918 RefPtr
<ImageBitmap
> ret
=
919 new ImageBitmap(aGlobal
, data
, aData
->mWriteOnly
, aData
->mAlphaType
);
921 ret
->mAllocatedImageData
= true;
924 ret
->SetPictureRect(aData
->mPictureRect
, rv
);
929 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateFromOffscreenCanvas(
930 nsIGlobalObject
* aGlobal
, OffscreenCanvas
& aOffscreenCanvas
,
932 // Check write-only mode.
933 bool writeOnly
= aOffscreenCanvas
.IsWriteOnly();
934 uint32_t flags
= nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE
|
935 nsLayoutUtils::SFE_EXACT_SIZE_SURFACE
;
937 SurfaceFromElementResult res
=
938 nsLayoutUtils::SurfaceFromOffscreenCanvas(&aOffscreenCanvas
, flags
);
940 RefPtr
<SourceSurface
> surface
= res
.GetSourceSurface();
942 if (NS_WARN_IF(!surface
)) {
943 aRv
.Throw(NS_ERROR_NOT_AVAILABLE
);
947 RefPtr
<layers::Image
> data
= CreateImageFromSurface(surface
);
949 RefPtr
<ImageBitmap
> ret
= new ImageBitmap(aGlobal
, data
, writeOnly
);
951 ret
->mAllocatedImageData
= true;
957 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateImageBitmapInternal(
958 nsIGlobalObject
* aGlobal
, gfx::SourceSurface
* aSurface
,
959 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
960 const bool aWriteOnly
, const bool aAllocatedImageData
, const bool aMustCopy
,
961 const gfxAlphaType aAlphaType
, ErrorResult
& aRv
) {
962 bool needToReportMemoryAllocation
= aAllocatedImageData
;
963 const IntSize srcSize
= aSurface
->GetSize();
965 aCropRect
.valueOr(IntRect(0, 0, srcSize
.width
, srcSize
.height
));
967 RefPtr
<SourceSurface
> surface
= aSurface
;
968 RefPtr
<DataSourceSurface
> dataSurface
;
970 // handle alpha premultiplication if surface not of correct type
972 gfxAlphaType alphaType
= aAlphaType
;
973 bool mustCopy
= aMustCopy
;
974 bool requiresPremultiply
= false;
975 bool requiresUnpremultiply
= false;
977 if (!IsOpaque(surface
->GetFormat())) {
978 if (aAlphaType
== gfxAlphaType::Premult
&&
979 aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
980 requiresUnpremultiply
= true;
981 alphaType
= gfxAlphaType::NonPremult
;
982 if (!aAllocatedImageData
) {
985 } else if (aAlphaType
== gfxAlphaType::NonPremult
&&
986 aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::Premultiply
) {
987 requiresPremultiply
= true;
988 alphaType
= gfxAlphaType::Premult
;
989 if (!aAllocatedImageData
) {
996 * if we don't own the data and need to create a new buffer to flip Y.
998 * we need to crop and flip, where crop must come first.
1000 * Or the caller demands a copy (WebGL contexts).
1002 if ((aOptions
.mImageOrientation
== ImageOrientation::FlipY
&&
1003 (!aAllocatedImageData
|| aCropRect
.isSome())) ||
1005 dataSurface
= surface
->GetDataSurface();
1007 dataSurface
= CropAndCopyDataSourceSurface(dataSurface
, cropRect
);
1008 if (NS_WARN_IF(!dataSurface
)) {
1009 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1013 surface
= dataSurface
;
1014 cropRect
.SetRect(0, 0, dataSurface
->GetSize().width
,
1015 dataSurface
->GetSize().height
);
1016 needToReportMemoryAllocation
= true;
1019 // flip image in Y direction
1020 if (aOptions
.mImageOrientation
== ImageOrientation::FlipY
) {
1022 dataSurface
= surface
->GetDataSurface();
1025 surface
= FlipYDataSourceSurface(dataSurface
);
1026 if (NS_WARN_IF(!surface
)) {
1031 if (requiresPremultiply
) {
1033 dataSurface
= surface
->GetDataSurface();
1036 surface
= AlphaPremultiplyDataSourceSurface(dataSurface
, true);
1037 if (NS_WARN_IF(!surface
)) {
1042 // resize if required
1043 if (aOptions
.mResizeWidth
.WasPassed() || aOptions
.mResizeHeight
.WasPassed()) {
1045 dataSurface
= surface
->GetDataSurface();
1048 surface
= ScaleDataSourceSurface(dataSurface
, aOptions
);
1049 if (NS_WARN_IF(!surface
)) {
1050 aRv
.ThrowInvalidStateError("Failed to create resized image");
1054 needToReportMemoryAllocation
= true;
1055 cropRect
.SetRect(0, 0, surface
->GetSize().width
, surface
->GetSize().height
);
1058 if (requiresUnpremultiply
) {
1060 dataSurface
= surface
->GetDataSurface();
1063 surface
= AlphaPremultiplyDataSourceSurface(dataSurface
, false);
1064 if (NS_WARN_IF(!surface
)) {
1069 // Create an Image from the SourceSurface.
1070 RefPtr
<layers::Image
> data
= CreateImageFromSurface(surface
);
1071 RefPtr
<ImageBitmap
> ret
=
1072 new ImageBitmap(aGlobal
, data
, aWriteOnly
, alphaType
);
1074 if (needToReportMemoryAllocation
) {
1075 ret
->mAllocatedImageData
= true;
1078 // Set the picture rectangle.
1079 ret
->SetPictureRect(cropRect
, aRv
);
1081 return ret
.forget();
1085 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1086 nsIGlobalObject
* aGlobal
, HTMLImageElement
& aImageEl
,
1087 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1089 // Check if the image element is completely available or not.
1090 if (!aImageEl
.Complete()) {
1091 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1095 bool writeOnly
= true;
1096 gfxAlphaType alphaType
= gfxAlphaType::NonPremult
;
1098 // Get the SourceSurface out from the image element and then do security
1100 RefPtr
<SourceSurface
> surface
= GetSurfaceFromElement(
1101 aGlobal
, aImageEl
, &writeOnly
, aOptions
, &alphaType
, aRv
);
1103 if (NS_WARN_IF(aRv
.Failed())) {
1107 bool needToReportMemoryAllocation
= false;
1108 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1109 writeOnly
, needToReportMemoryAllocation
,
1110 false, alphaType
, aRv
);
1113 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1114 nsIGlobalObject
* aGlobal
, SVGImageElement
& aImageEl
,
1115 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1117 bool writeOnly
= true;
1118 gfxAlphaType alphaType
= gfxAlphaType::NonPremult
;
1120 // Get the SourceSurface out from the image element and then do security
1122 RefPtr
<SourceSurface
> surface
= GetSurfaceFromElement(
1123 aGlobal
, aImageEl
, &writeOnly
, aOptions
, &alphaType
, aRv
);
1125 if (NS_WARN_IF(aRv
.Failed())) {
1129 bool needToReportMemoryAllocation
= false;
1131 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1132 writeOnly
, needToReportMemoryAllocation
,
1133 false, alphaType
, aRv
);
1137 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1138 nsIGlobalObject
* aGlobal
, HTMLVideoElement
& aVideoEl
,
1139 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1141 aVideoEl
.LogVisibility(
1142 mozilla::dom::HTMLVideoElement::CallerAPI::CREATE_IMAGEBITMAP
);
1144 // Check network state.
1145 if (aVideoEl
.NetworkState() == NETWORK_EMPTY
) {
1146 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1150 // Check ready state.
1151 // Cannot be HTMLMediaElement::HAVE_NOTHING or
1152 // HTMLMediaElement::HAVE_METADATA.
1153 if (aVideoEl
.ReadyState() <= HAVE_METADATA
) {
1154 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1159 nsCOMPtr
<nsIPrincipal
> principal
= aVideoEl
.GetCurrentVideoPrincipal();
1160 bool hadCrossOriginRedirects
= aVideoEl
.HadCrossOriginRedirects();
1161 bool CORSUsed
= aVideoEl
.GetCORSMode() != CORS_NONE
;
1162 bool writeOnly
= CanvasUtils::CheckWriteOnlySecurity(CORSUsed
, principal
,
1163 hadCrossOriginRedirects
);
1165 // Create ImageBitmap.
1166 RefPtr
<layers::Image
> data
= aVideoEl
.GetCurrentImage();
1168 aRv
.Throw(NS_ERROR_NOT_AVAILABLE
);
1172 RefPtr
<SourceSurface
> surface
= data
->GetAsSourceSurface();
1174 // preserve original behavior in case of unavailble surface
1175 RefPtr
<ImageBitmap
> ret
= new ImageBitmap(aGlobal
, data
, writeOnly
);
1176 return ret
.forget();
1179 bool needToReportMemoryAllocation
= false;
1181 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1182 writeOnly
, needToReportMemoryAllocation
,
1183 false, gfxAlphaType::Premult
, aRv
);
1187 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1188 nsIGlobalObject
* aGlobal
, HTMLCanvasElement
& aCanvasEl
,
1189 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1191 if (aCanvasEl
.Width() == 0 || aCanvasEl
.Height() == 0) {
1192 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1196 bool writeOnly
= true;
1197 gfxAlphaType alphaType
= gfxAlphaType::NonPremult
;
1199 RefPtr
<SourceSurface
> surface
= GetSurfaceFromElement(
1200 aGlobal
, aCanvasEl
, &writeOnly
, aOptions
, &alphaType
, aRv
);
1202 if (NS_WARN_IF(aRv
.Failed())) {
1207 writeOnly
= aCanvasEl
.IsWriteOnly();
1210 // If the HTMLCanvasElement's rendering context is WebGL/WebGPU,
1211 // then the snapshot we got from the HTMLCanvasElement is
1212 // a DataSourceSurface which is a copy of the rendering context.
1213 // We handle cropping in this case.
1214 bool needToReportMemoryAllocation
= false;
1215 bool mustCopy
= false;
1217 if ((aCanvasEl
.GetCurrentContextType() == CanvasContextType::WebGL1
||
1218 aCanvasEl
.GetCurrentContextType() == CanvasContextType::WebGL2
||
1219 aCanvasEl
.GetCurrentContextType() == CanvasContextType::WebGPU
) &&
1220 aCropRect
.isSome()) {
1224 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1225 writeOnly
, needToReportMemoryAllocation
,
1226 mustCopy
, alphaType
, aRv
);
1230 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1231 nsIGlobalObject
* aGlobal
, OffscreenCanvas
& aOffscreenCanvas
,
1232 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1234 if (aOffscreenCanvas
.Width() == 0) {
1235 aRv
.ThrowInvalidStateError("Passed-in canvas has width 0");
1239 if (aOffscreenCanvas
.Height() == 0) {
1240 aRv
.ThrowInvalidStateError("Passed-in canvas has height 0");
1244 uint32_t flags
= nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE
|
1245 nsLayoutUtils::SFE_EXACT_SIZE_SURFACE
;
1247 // by default surfaces have premultiplied alpha
1248 // attempt to get non premultiplied if required
1249 if (aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
1250 flags
|= nsLayoutUtils::SFE_ALLOW_NON_PREMULT
;
1253 SurfaceFromElementResult res
=
1254 nsLayoutUtils::SurfaceFromOffscreenCanvas(&aOffscreenCanvas
, flags
);
1256 RefPtr
<SourceSurface
> surface
= res
.GetSourceSurface();
1257 if (NS_WARN_IF(!surface
)) {
1258 aRv
.ThrowInvalidStateError("Passed-in canvas failed to create snapshot");
1262 gfxAlphaType alphaType
= res
.mAlphaType
;
1263 bool writeOnly
= res
.mIsWriteOnly
;
1265 // If the OffscreenCanvas's rendering context is WebGL/WebGPU, then the
1266 // snapshot we got from the OffscreenCanvas is a DataSourceSurface which
1267 // is a copy of the rendering context. We handle cropping in this case.
1268 bool needToReportMemoryAllocation
= false;
1270 aCropRect
.isSome() &&
1271 (aOffscreenCanvas
.GetContextType() == CanvasContextType::WebGL1
||
1272 aOffscreenCanvas
.GetContextType() == CanvasContextType::WebGL2
||
1273 aOffscreenCanvas
.GetContextType() == CanvasContextType::WebGPU
);
1275 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1276 writeOnly
, needToReportMemoryAllocation
,
1277 mustCopy
, alphaType
, aRv
);
1281 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1282 nsIGlobalObject
* aGlobal
, ImageData
& aImageData
,
1283 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1285 // Copy data into SourceSurface.
1286 RootedSpiderMonkeyInterface
<Uint8ClampedArray
> array(RootingCx());
1287 if (!array
.Init(aImageData
.GetDataObject())) {
1288 aRv
.ThrowInvalidStateError(
1289 "Failed to extract Uint8ClampedArray from ImageData (security check "
1293 array
.ComputeState();
1294 const SurfaceFormat FORMAT
= SurfaceFormat::R8G8B8A8
;
1295 // ImageData's underlying data is not alpha-premultiplied.
1296 auto alphaType
= (aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::Premultiply
)
1297 ? gfxAlphaType::Premult
1298 : gfxAlphaType::NonPremult
;
1300 const uint32_t BYTES_PER_PIXEL
= BytesPerPixel(FORMAT
);
1301 const uint32_t imageWidth
= aImageData
.Width();
1302 const uint32_t imageHeight
= aImageData
.Height();
1303 const uint32_t imageStride
= imageWidth
* BYTES_PER_PIXEL
;
1304 const uint32_t dataLength
= array
.Length();
1305 const gfx::IntSize
imageSize(imageWidth
, imageHeight
);
1307 // Check the ImageData is neutered or not.
1308 if (imageWidth
== 0 || imageHeight
== 0) {
1309 aRv
.ThrowInvalidStateError("Passed-in image is empty");
1313 if ((imageWidth
* imageHeight
* BYTES_PER_PIXEL
) != dataLength
) {
1314 aRv
.ThrowInvalidStateError("Data size / image format mismatch");
1318 // Create and Crop the raw data into a layers::Image
1319 RefPtr
<layers::Image
> data
;
1321 // If the data could move during a GC, copy it out into a local buffer that
1322 // lives until a CreateImageFromRawData lower in the stack copies it.
1323 // Reassure the static analysis that we know what we're doing.
1324 size_t maxInline
= JS_MaxMovableTypedArraySize();
1325 uint8_t inlineDataBuffer
[maxInline
];
1326 uint8_t* fixedData
= array
.FixedData(inlineDataBuffer
, maxInline
);
1328 // Lie to the hazard analysis and say that we're done with everything that
1329 // `array` was using (safe because the data buffer is fixed, and the holding
1330 // JSObject is being kept alive elsewhere.)
1333 if (NS_IsMainThread()) {
1334 data
= CreateImageFromRawData(imageSize
, imageStride
, FORMAT
, fixedData
,
1335 dataLength
, aCropRect
, aOptions
);
1337 RefPtr
<CreateImageFromRawDataInMainThreadSyncTask
> task
=
1338 new CreateImageFromRawDataInMainThreadSyncTask(
1339 fixedData
, dataLength
, imageStride
, FORMAT
, imageSize
, aCropRect
,
1340 getter_AddRefs(data
), aOptions
);
1341 task
->Dispatch(Canceling
, aRv
);
1344 if (NS_WARN_IF(!data
)) {
1345 aRv
.ThrowInvalidStateError("Failed to create internal image");
1349 // Create an ImageBitmap.
1350 RefPtr
<ImageBitmap
> ret
=
1351 new ImageBitmap(aGlobal
, data
, false /* write-only */, alphaType
);
1353 ret
->mAllocatedImageData
= true;
1355 // The cropping information has been handled in the CreateImageFromRawData()
1358 return ret
.forget();
1362 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1363 nsIGlobalObject
* aGlobal
, CanvasRenderingContext2D
& aCanvasCtx
,
1364 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1366 nsCOMPtr
<nsPIDOMWindowInner
> win
= do_QueryInterface(aGlobal
);
1367 nsGlobalWindowInner
* window
= nsGlobalWindowInner::Cast(win
);
1368 if (NS_WARN_IF(!window
) || !window
->GetExtantDoc()) {
1369 aRv
.Throw(NS_ERROR_DOM_SECURITY_ERR
);
1373 window
->GetExtantDoc()->WarnOnceAbout(
1374 DeprecatedOperations::eCreateImageBitmapCanvasRenderingContext2D
);
1376 // Check write-only mode.
1378 aCanvasCtx
.GetCanvas()->IsWriteOnly() || aCanvasCtx
.IsWriteOnly();
1380 RefPtr
<SourceSurface
> surface
= aCanvasCtx
.GetSurfaceSnapshot();
1382 if (NS_WARN_IF(!surface
)) {
1383 aRv
.Throw(NS_ERROR_NOT_AVAILABLE
);
1387 const IntSize surfaceSize
= surface
->GetSize();
1388 if (surfaceSize
.width
== 0 || surfaceSize
.height
== 0) {
1389 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1393 bool needToReportMemoryAllocation
= true;
1395 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1396 writeOnly
, needToReportMemoryAllocation
,
1397 false, gfxAlphaType::Premult
, aRv
);
1401 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1402 nsIGlobalObject
* aGlobal
, ImageBitmap
& aImageBitmap
,
1403 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1405 if (!aImageBitmap
.mData
) {
1406 aRv
.Throw(NS_ERROR_DOM_INVALID_STATE_ERR
);
1410 IntRect cropRect
= aImageBitmap
.mPictureRect
;
1411 RefPtr
<SourceSurface
> surface
;
1413 bool needToReportMemoryAllocation
= false;
1415 if (aImageBitmap
.mSurface
) {
1416 // the source imageBitmap already has a cropped surface, just use this
1417 surface
= aImageBitmap
.mSurface
;
1418 cropRect
= aCropRect
.valueOr(cropRect
);
1420 RefPtr
<layers::Image
> data
= aImageBitmap
.mData
;
1421 surface
= data
->GetAsSourceSurface();
1422 if (NS_WARN_IF(!surface
)) {
1423 aRv
.Throw(NS_ERROR_NOT_AVAILABLE
);
1427 if (aCropRect
.isSome()) {
1428 // get new crop rect relative to original uncropped surface
1429 IntRect newCropRect
= aCropRect
.ref();
1430 newCropRect
= FixUpNegativeDimension(newCropRect
, aRv
);
1432 newCropRect
.MoveBy(cropRect
.X(), cropRect
.Y());
1434 if (cropRect
.Contains(newCropRect
)) {
1435 // new crop region within existing surface
1436 // safe to just crop this with new rect
1437 cropRect
= newCropRect
;
1439 // crop includes area outside original cropped region
1440 // create new surface cropped by original bitmap crop rect
1441 RefPtr
<DataSourceSurface
> dataSurface
= surface
->GetDataSurface();
1443 surface
= CropAndCopyDataSourceSurface(dataSurface
, cropRect
);
1444 if (NS_WARN_IF(!surface
)) {
1445 aRv
.Throw(NS_ERROR_NOT_AVAILABLE
);
1448 needToReportMemoryAllocation
= true;
1449 cropRect
= aCropRect
.ref();
1454 return CreateImageBitmapInternal(
1455 aGlobal
, surface
, Some(cropRect
), aOptions
, aImageBitmap
.mWriteOnly
,
1456 needToReportMemoryAllocation
, false, aImageBitmap
.mAlphaType
, aRv
);
1460 already_AddRefed
<ImageBitmap
> ImageBitmap::CreateInternal(
1461 nsIGlobalObject
* aGlobal
, VideoFrame
& aVideoFrame
,
1462 const Maybe
<IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1464 if (aVideoFrame
.CodedWidth() == 0) {
1465 aRv
.ThrowInvalidStateError("Passed-in video frame has width 0");
1469 if (aVideoFrame
.CodedHeight() == 0) {
1470 aRv
.ThrowInvalidStateError("Passed-in video frame has height 0");
1474 uint32_t flags
= nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE
;
1476 // by default surfaces have premultiplied alpha
1477 // attempt to get non premultiplied if required
1478 if (aOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
1479 flags
|= nsLayoutUtils::SFE_ALLOW_NON_PREMULT
;
1482 SurfaceFromElementResult res
=
1483 nsLayoutUtils::SurfaceFromVideoFrame(&aVideoFrame
, flags
);
1485 RefPtr
<SourceSurface
> surface
= res
.GetSourceSurface();
1486 if (NS_WARN_IF(!surface
)) {
1487 aRv
.ThrowInvalidStateError("Passed-in video frame has no surface data");
1491 gfxAlphaType alphaType
= res
.mAlphaType
;
1492 bool writeOnly
= res
.mIsWriteOnly
;
1493 bool needToReportMemoryAllocation
= false;
1494 bool mustCopy
= false;
1496 return CreateImageBitmapInternal(aGlobal
, surface
, aCropRect
, aOptions
,
1497 writeOnly
, needToReportMemoryAllocation
,
1498 mustCopy
, alphaType
, aRv
);
1501 class FulfillImageBitmapPromise
{
1503 FulfillImageBitmapPromise(Promise
* aPromise
, ImageBitmap
* aImageBitmap
)
1504 : mPromise(aPromise
), mImageBitmap(aImageBitmap
) {
1505 MOZ_ASSERT(aPromise
);
1508 void DoFulfillImageBitmapPromise() { mPromise
->MaybeResolve(mImageBitmap
); }
1511 RefPtr
<Promise
> mPromise
;
1512 RefPtr
<ImageBitmap
> mImageBitmap
;
1515 class FulfillImageBitmapPromiseTask final
: public Runnable
,
1516 public FulfillImageBitmapPromise
{
1518 FulfillImageBitmapPromiseTask(Promise
* aPromise
, ImageBitmap
* aImageBitmap
)
1519 : Runnable("dom::FulfillImageBitmapPromiseTask"),
1520 FulfillImageBitmapPromise(aPromise
, aImageBitmap
) {}
1522 NS_IMETHOD
Run() override
{
1523 DoFulfillImageBitmapPromise();
1528 class FulfillImageBitmapPromiseWorkerTask final
1529 : public WorkerSameThreadRunnable
,
1530 public FulfillImageBitmapPromise
{
1532 FulfillImageBitmapPromiseWorkerTask(Promise
* aPromise
,
1533 ImageBitmap
* aImageBitmap
)
1534 : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()),
1535 FulfillImageBitmapPromise(aPromise
, aImageBitmap
) {}
1537 bool WorkerRun(JSContext
* aCx
, WorkerPrivate
* aWorkerPrivate
) override
{
1538 DoFulfillImageBitmapPromise();
1543 static void AsyncFulfillImageBitmapPromise(Promise
* aPromise
,
1544 ImageBitmap
* aImageBitmap
) {
1545 if (NS_IsMainThread()) {
1546 nsCOMPtr
<nsIRunnable
> task
=
1547 new FulfillImageBitmapPromiseTask(aPromise
, aImageBitmap
);
1548 NS_DispatchToCurrentThread(task
); // Actually, to the main-thread.
1550 RefPtr
<FulfillImageBitmapPromiseWorkerTask
> task
=
1551 new FulfillImageBitmapPromiseWorkerTask(aPromise
, aImageBitmap
);
1552 task
->Dispatch(); // Actually, to the current worker-thread.
1556 class CreateImageBitmapFromBlobRunnable
;
1558 class CreateImageBitmapFromBlob final
: public DiscardableRunnable
,
1559 public imgIContainerCallback
,
1560 public nsIInputStreamCallback
{
1561 friend class CreateImageBitmapFromBlobRunnable
;
1564 NS_DECL_ISUPPORTS_INHERITED
1565 NS_DECL_IMGICONTAINERCALLBACK
1566 NS_DECL_NSIINPUTSTREAMCALLBACK
1568 static already_AddRefed
<CreateImageBitmapFromBlob
> Create(
1569 Promise
* aPromise
, nsIGlobalObject
* aGlobal
, Blob
& aBlob
,
1570 const Maybe
<IntRect
>& aCropRect
, nsIEventTarget
* aMainThreadEventTarget
,
1571 const ImageBitmapOptions
& aOptions
);
1573 NS_IMETHOD
Run() override
{
1574 MOZ_ASSERT(IsCurrentThread());
1576 nsresult rv
= StartMimeTypeAndDecodeAndCropBlob();
1577 if (NS_WARN_IF(NS_FAILED(rv
))) {
1578 MimeTypeAndDecodeAndCropBlobCompletedMainThread(nullptr, rv
);
1584 // Called by the WorkerRef.
1585 void WorkerShuttingDown();
1588 CreateImageBitmapFromBlob(Promise
* aPromise
, nsIGlobalObject
* aGlobal
,
1589 already_AddRefed
<nsIInputStream
> aInputStream
,
1590 const Maybe
<IntRect
>& aCropRect
,
1591 nsIEventTarget
* aMainThreadEventTarget
,
1592 const ImageBitmapOptions
& aOptions
)
1593 : DiscardableRunnable("dom::CreateImageBitmapFromBlob"),
1595 mMutex("dom::CreateImageBitmapFromBlob::mMutex"),
1597 mGlobalObject(aGlobal
),
1598 mInputStream(std::move(aInputStream
)),
1599 mCropRect(aCropRect
),
1600 mMainThreadEventTarget(aMainThreadEventTarget
),
1602 mThread(PR_GetCurrentThread()) {}
1604 virtual ~CreateImageBitmapFromBlob() = default;
1606 bool IsCurrentThread() const { return mThread
== PR_GetCurrentThread(); }
1608 // Called on the owning thread.
1609 nsresult
StartMimeTypeAndDecodeAndCropBlob();
1611 // Will be called when the decoding + cropping is completed on the
1612 // main-thread. This could the not the owning thread!
1613 void MimeTypeAndDecodeAndCropBlobCompletedMainThread(layers::Image
* aImage
,
1616 // Will be called when the decoding + cropping is completed on the owning
1618 void MimeTypeAndDecodeAndCropBlobCompletedOwningThread(layers::Image
* aImage
,
1621 // This is called on the main-thread only.
1622 nsresult
MimeTypeAndDecodeAndCropBlob();
1624 // This is called on the main-thread only.
1625 nsresult
DecodeAndCropBlob(const nsACString
& aMimeType
);
1627 // This is called on the main-thread only.
1628 nsresult
GetMimeTypeSync(nsACString
& aMimeType
);
1630 // This is called on the main-thread only.
1631 nsresult
GetMimeTypeAsync();
1633 Mutex mMutex MOZ_UNANNOTATED
;
1635 // The access to this object is protected by mutex but is always nullified on
1636 // the owning thread.
1637 RefPtr
<ThreadSafeWorkerRef
> mWorkerRef
;
1639 // Touched only on the owning thread.
1640 RefPtr
<Promise
> mPromise
;
1642 // Touched only on the owning thread.
1643 nsCOMPtr
<nsIGlobalObject
> mGlobalObject
;
1645 nsCOMPtr
<nsIInputStream
> mInputStream
;
1646 Maybe
<IntRect
> mCropRect
;
1647 nsCOMPtr
<nsIEventTarget
> mMainThreadEventTarget
;
1648 const ImageBitmapOptions mOptions
;
1652 NS_IMPL_ISUPPORTS_INHERITED(CreateImageBitmapFromBlob
, DiscardableRunnable
,
1653 imgIContainerCallback
, nsIInputStreamCallback
)
1655 class CreateImageBitmapFromBlobRunnable
: public WorkerRunnable
{
1657 explicit CreateImageBitmapFromBlobRunnable(WorkerPrivate
* aWorkerPrivate
,
1658 CreateImageBitmapFromBlob
* aTask
,
1659 layers::Image
* aImage
,
1661 : WorkerRunnable(aWorkerPrivate
),
1666 bool WorkerRun(JSContext
* aCx
, WorkerPrivate
* aWorkerPrivate
) override
{
1667 mTask
->MimeTypeAndDecodeAndCropBlobCompletedOwningThread(mImage
, mStatus
);
1672 RefPtr
<CreateImageBitmapFromBlob
> mTask
;
1673 RefPtr
<layers::Image
> mImage
;
1677 static void AsyncCreateImageBitmapFromBlob(Promise
* aPromise
,
1678 nsIGlobalObject
* aGlobal
,
1680 const Maybe
<IntRect
>& aCropRect
,
1681 const ImageBitmapOptions
& aOptions
) {
1682 // Let's identify the main-thread event target.
1683 nsCOMPtr
<nsIEventTarget
> mainThreadEventTarget
;
1684 if (NS_IsMainThread()) {
1685 mainThreadEventTarget
= aGlobal
->EventTargetFor(TaskCategory::Other
);
1687 WorkerPrivate
* workerPrivate
= GetCurrentThreadWorkerPrivate();
1688 MOZ_ASSERT(workerPrivate
);
1689 mainThreadEventTarget
= workerPrivate
->MainThreadEventTarget();
1692 RefPtr
<CreateImageBitmapFromBlob
> task
= CreateImageBitmapFromBlob::Create(
1693 aPromise
, aGlobal
, aBlob
, aCropRect
, mainThreadEventTarget
, aOptions
);
1694 if (NS_WARN_IF(!task
)) {
1695 aPromise
->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR
);
1699 NS_DispatchToCurrentThread(task
);
1703 already_AddRefed
<Promise
> ImageBitmap::Create(
1704 nsIGlobalObject
* aGlobal
, const ImageBitmapSource
& aSrc
,
1705 const Maybe
<gfx::IntRect
>& aCropRect
, const ImageBitmapOptions
& aOptions
,
1707 MOZ_ASSERT(aGlobal
);
1709 RefPtr
<Promise
> promise
= Promise::Create(aGlobal
, aRv
);
1711 if (NS_WARN_IF(aRv
.Failed())) {
1715 if (aCropRect
.isSome()) {
1716 if (aCropRect
->Width() == 0) {
1717 aRv
.ThrowRangeError(
1718 "The crop rect width passed to createImageBitmap must be nonzero");
1719 return promise
.forget();
1722 if (aCropRect
->Height() == 0) {
1723 aRv
.ThrowRangeError(
1724 "The crop rect height passed to createImageBitmap must be nonzero");
1725 return promise
.forget();
1729 if (aOptions
.mResizeWidth
.WasPassed() && aOptions
.mResizeWidth
.Value() == 0) {
1730 aRv
.ThrowInvalidStateError(
1731 "The resizeWidth passed to createImageBitmap must be nonzero");
1732 return promise
.forget();
1735 if (aOptions
.mResizeHeight
.WasPassed() &&
1736 aOptions
.mResizeHeight
.Value() == 0) {
1737 aRv
.ThrowInvalidStateError(
1738 "The resizeHeight passed to createImageBitmap must be nonzero");
1739 return promise
.forget();
1742 RefPtr
<ImageBitmap
> imageBitmap
;
1744 if (aSrc
.IsHTMLImageElement()) {
1747 "Creating ImageBitmap from HTMLImageElement off the main thread.");
1748 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsHTMLImageElement(),
1749 aCropRect
, aOptions
, aRv
);
1750 } else if (aSrc
.IsSVGImageElement()) {
1753 "Creating ImageBitmap from SVGImageElement off the main thread.");
1754 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsSVGImageElement(),
1755 aCropRect
, aOptions
, aRv
);
1756 } else if (aSrc
.IsHTMLVideoElement()) {
1759 "Creating ImageBitmap from HTMLVideoElement off the main thread.");
1760 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsHTMLVideoElement(),
1761 aCropRect
, aOptions
, aRv
);
1762 } else if (aSrc
.IsHTMLCanvasElement()) {
1765 "Creating ImageBitmap from HTMLCanvasElement off the main thread.");
1766 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsHTMLCanvasElement(),
1767 aCropRect
, aOptions
, aRv
);
1768 } else if (aSrc
.IsOffscreenCanvas()) {
1769 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsOffscreenCanvas(),
1770 aCropRect
, aOptions
, aRv
);
1771 } else if (aSrc
.IsImageData()) {
1772 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsImageData(), aCropRect
,
1774 } else if (aSrc
.IsCanvasRenderingContext2D()) {
1775 MOZ_ASSERT(NS_IsMainThread(),
1776 "Creating ImageBitmap from CanvasRenderingContext2D off the "
1778 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsCanvasRenderingContext2D(),
1779 aCropRect
, aOptions
, aRv
);
1780 } else if (aSrc
.IsImageBitmap()) {
1781 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsImageBitmap(), aCropRect
,
1783 } else if (aSrc
.IsBlob()) {
1784 AsyncCreateImageBitmapFromBlob(promise
, aGlobal
, aSrc
.GetAsBlob(),
1785 aCropRect
, aOptions
);
1786 return promise
.forget();
1787 } else if (aSrc
.IsVideoFrame()) {
1788 imageBitmap
= CreateInternal(aGlobal
, aSrc
.GetAsVideoFrame(), aCropRect
,
1791 MOZ_CRASH("Unsupported type!");
1795 if (!aRv
.Failed()) {
1796 AsyncFulfillImageBitmapPromise(promise
, imageBitmap
);
1799 return promise
.forget();
1803 JSObject
* ImageBitmap::ReadStructuredClone(
1804 JSContext
* aCx
, JSStructuredCloneReader
* aReader
, nsIGlobalObject
* aParent
,
1805 const nsTArray
<RefPtr
<DataSourceSurface
>>& aClonedSurfaces
,
1808 MOZ_ASSERT(aReader
);
1809 // aParent might be null.
1813 uint32_t picRectWidth_
;
1814 uint32_t picRectHeight_
;
1815 uint32_t alphaType_
;
1818 if (!JS_ReadUint32Pair(aReader
, &picRectX_
, &picRectY_
) ||
1819 !JS_ReadUint32Pair(aReader
, &picRectWidth_
, &picRectHeight_
) ||
1820 !JS_ReadUint32Pair(aReader
, &alphaType_
, &writeOnly
)) {
1824 int32_t picRectX
= BitwiseCast
<int32_t>(picRectX_
);
1825 int32_t picRectY
= BitwiseCast
<int32_t>(picRectY_
);
1826 int32_t picRectWidth
= BitwiseCast
<int32_t>(picRectWidth_
);
1827 int32_t picRectHeight
= BitwiseCast
<int32_t>(picRectHeight_
);
1828 const auto alphaType
= BitwiseCast
<gfxAlphaType
>(alphaType_
);
1830 // Create a new ImageBitmap.
1831 MOZ_ASSERT(!aClonedSurfaces
.IsEmpty());
1832 MOZ_ASSERT(aIndex
< aClonedSurfaces
.Length());
1834 // RefPtr<ImageBitmap> needs to go out of scope before toObjectOrNull() is
1835 // called because the static analysis thinks dereferencing XPCOM objects
1836 // can GC (because in some cases it can!), and a return statement with a
1837 // JSObject* type means that JSObject* is on the stack as a raw pointer
1838 // while destructors are running.
1839 JS::Rooted
<JS::Value
> value(aCx
);
1842 if (aIndex
>= aClonedSurfaces
.Length()) {
1846 RefPtr
<layers::Image
> img
= CreateImageFromSurface(aClonedSurfaces
[aIndex
]);
1847 RefPtr
<ImageBitmap
> imageBitmap
=
1848 new ImageBitmap(aParent
, img
, !!writeOnly
, alphaType
);
1851 imageBitmap
->SetPictureRect(
1852 IntRect(picRectX
, picRectY
, picRectWidth
, picRectHeight
), error
);
1853 if (NS_WARN_IF(error
.Failed())) {
1854 error
.SuppressException();
1858 if (!GetOrCreateDOMReflector(aCx
, imageBitmap
, &value
)) {
1862 imageBitmap
->mAllocatedImageData
= true;
1865 return &(value
.toObject());
1869 void ImageBitmap::WriteStructuredClone(
1870 JSStructuredCloneWriter
* aWriter
,
1871 nsTArray
<RefPtr
<DataSourceSurface
>>& aClonedSurfaces
,
1872 ImageBitmap
* aImageBitmap
, ErrorResult
& aRv
) {
1873 MOZ_ASSERT(aWriter
);
1874 MOZ_ASSERT(aImageBitmap
);
1876 if (aImageBitmap
->IsWriteOnly()) {
1877 return aRv
.ThrowDataCloneError("Cannot clone ImageBitmap, is write-only");
1880 if (!aImageBitmap
->mData
) {
1881 // A closed image cannot be cloned.
1882 return aRv
.ThrowDataCloneError("Cannot clone ImageBitmap, is closed");
1885 const uint32_t picRectX
= BitwiseCast
<uint32_t>(aImageBitmap
->mPictureRect
.x
);
1886 const uint32_t picRectY
= BitwiseCast
<uint32_t>(aImageBitmap
->mPictureRect
.y
);
1887 const uint32_t picRectWidth
=
1888 BitwiseCast
<uint32_t>(aImageBitmap
->mPictureRect
.width
);
1889 const uint32_t picRectHeight
=
1890 BitwiseCast
<uint32_t>(aImageBitmap
->mPictureRect
.height
);
1891 const uint32_t alphaType
= BitwiseCast
<uint32_t>(aImageBitmap
->mAlphaType
);
1893 // Indexing the cloned surfaces and send the index to the receiver.
1894 uint32_t index
= aClonedSurfaces
.Length();
1896 if (NS_WARN_IF(!JS_WriteUint32Pair(aWriter
, SCTAG_DOM_IMAGEBITMAP
, index
)) ||
1897 NS_WARN_IF(!JS_WriteUint32Pair(aWriter
, picRectX
, picRectY
)) ||
1898 NS_WARN_IF(!JS_WriteUint32Pair(aWriter
, picRectWidth
, picRectHeight
)) ||
1900 !JS_WriteUint32Pair(aWriter
, alphaType
, aImageBitmap
->mWriteOnly
))) {
1901 return aRv
.ThrowDataCloneError(
1902 "Cannot clone ImageBitmap, failed to write params");
1905 RefPtr
<SourceSurface
> surface
= aImageBitmap
->mData
->GetAsSourceSurface();
1906 if (NS_WARN_IF(!surface
)) {
1907 return aRv
.ThrowDataCloneError("Cannot clone ImageBitmap, no surface");
1910 RefPtr
<DataSourceSurface
> snapshot
= surface
->GetDataSurface();
1911 if (NS_WARN_IF(!snapshot
)) {
1912 return aRv
.ThrowDataCloneError("Cannot clone ImageBitmap, no data surface");
1915 RefPtr
<DataSourceSurface
> dstDataSurface
;
1917 // DataSourceSurfaceD2D1::GetStride() will call EnsureMapped implicitly and
1918 // won't Unmap after exiting function. So instead calling GetStride()
1919 // directly, using ScopedMap to get stride.
1920 DataSourceSurface::ScopedMap
map(snapshot
, DataSourceSurface::READ
);
1921 if (NS_WARN_IF(!map
.IsMapped())) {
1922 return aRv
.ThrowDataCloneError(
1923 "Cannot clone ImageBitmap, cannot map surface");
1926 dstDataSurface
= Factory::CreateDataSourceSurfaceWithStride(
1927 snapshot
->GetSize(), snapshot
->GetFormat(), map
.GetStride(), true);
1929 if (NS_WARN_IF(!dstDataSurface
)) {
1930 return aRv
.ThrowDataCloneError("Cannot clone ImageBitmap, out of memory");
1932 Factory::CopyDataSourceSurface(snapshot
, dstDataSurface
);
1933 aClonedSurfaces
.AppendElement(dstDataSurface
);
1936 size_t ImageBitmap::GetAllocatedSize() const {
1937 if (!mAllocatedImageData
) {
1941 // Calculate how many bytes are used.
1942 if (mData
->GetFormat() == mozilla::ImageFormat::PLANAR_YCBCR
) {
1943 return mData
->AsPlanarYCbCrImage()->GetDataSize();
1946 if (mData
->GetFormat() == mozilla::ImageFormat::NV_IMAGE
) {
1947 return mData
->AsNVImage()->GetBufferSize();
1950 RefPtr
<SourceSurface
> surface
= mData
->GetAsSourceSurface();
1951 if (NS_WARN_IF(!surface
)) {
1955 const int bytesPerPixel
= BytesPerPixel(surface
->GetFormat());
1956 return surface
->GetSize().height
* surface
->GetSize().width
* bytesPerPixel
;
1959 size_t BindingJSObjectMallocBytes(ImageBitmap
* aBitmap
) {
1960 return aBitmap
->GetAllocatedSize();
1964 already_AddRefed
<CreateImageBitmapFromBlob
> CreateImageBitmapFromBlob::Create(
1965 Promise
* aPromise
, nsIGlobalObject
* aGlobal
, Blob
& aBlob
,
1966 const Maybe
<IntRect
>& aCropRect
, nsIEventTarget
* aMainThreadEventTarget
,
1967 const ImageBitmapOptions
& aOptions
) {
1968 // Get the internal stream of the blob.
1969 nsCOMPtr
<nsIInputStream
> stream
;
1971 aBlob
.Impl()->CreateInputStream(getter_AddRefs(stream
), error
);
1972 if (NS_WARN_IF(error
.Failed())) {
1976 if (!NS_InputStreamIsBuffered(stream
)) {
1977 nsCOMPtr
<nsIInputStream
> bufferedStream
;
1978 nsresult rv
= NS_NewBufferedInputStream(getter_AddRefs(bufferedStream
),
1979 stream
.forget(), 4096);
1980 if (NS_WARN_IF(NS_FAILED(rv
))) {
1984 stream
= bufferedStream
;
1987 RefPtr
<CreateImageBitmapFromBlob
> task
= new CreateImageBitmapFromBlob(
1988 aPromise
, aGlobal
, stream
.forget(), aCropRect
, aMainThreadEventTarget
,
1991 // Nothing to do for the main-thread.
1992 if (NS_IsMainThread()) {
1993 return task
.forget();
1996 // Let's use a WorkerRef to keep the worker alive if this is not the
1998 WorkerPrivate
* workerPrivate
= GetCurrentThreadWorkerPrivate();
1999 MOZ_ASSERT(workerPrivate
);
2001 RefPtr
<StrongWorkerRef
> workerRef
=
2002 StrongWorkerRef::Create(workerPrivate
, "CreateImageBitmapFromBlob",
2003 [task
]() { task
->WorkerShuttingDown(); });
2004 if (NS_WARN_IF(!workerRef
)) {
2008 task
->mWorkerRef
= new ThreadSafeWorkerRef(workerRef
);
2009 return task
.forget();
2012 nsresult
CreateImageBitmapFromBlob::StartMimeTypeAndDecodeAndCropBlob() {
2013 MOZ_ASSERT(IsCurrentThread());
2016 if (!NS_IsMainThread()) {
2017 RefPtr
<CreateImageBitmapFromBlob
> self
= this;
2018 nsCOMPtr
<nsIRunnable
> r
= NS_NewRunnableFunction(
2019 "CreateImageBitmapFromBlob::MimeTypeAndDecodeAndCropBlob", [self
]() {
2020 nsresult rv
= self
->MimeTypeAndDecodeAndCropBlob();
2021 if (NS_WARN_IF(NS_FAILED(rv
))) {
2022 self
->MimeTypeAndDecodeAndCropBlobCompletedMainThread(nullptr, rv
);
2026 return mMainThreadEventTarget
->Dispatch(r
.forget());
2030 return MimeTypeAndDecodeAndCropBlob();
2033 nsresult
CreateImageBitmapFromBlob::MimeTypeAndDecodeAndCropBlob() {
2034 MOZ_ASSERT(NS_IsMainThread());
2036 nsAutoCString mimeType
;
2037 nsresult rv
= GetMimeTypeSync(mimeType
);
2038 if (rv
== NS_BASE_STREAM_WOULD_BLOCK
) {
2039 return GetMimeTypeAsync();
2042 if (NS_WARN_IF(NS_FAILED(rv
))) {
2046 return DecodeAndCropBlob(mimeType
);
2049 nsresult
CreateImageBitmapFromBlob::DecodeAndCropBlob(
2050 const nsACString
& aMimeType
) {
2051 // Get the Component object.
2052 nsCOMPtr
<imgITools
> imgtool
= do_GetService(NS_IMGTOOLS_CID
);
2053 if (NS_WARN_IF(!imgtool
)) {
2054 return NS_ERROR_FAILURE
;
2058 nsresult rv
= imgtool
->DecodeImageAsync(mInputStream
, aMimeType
, this,
2059 mMainThreadEventTarget
);
2060 if (NS_WARN_IF(NS_FAILED(rv
))) {
2067 static nsresult
sniff_cb(nsIInputStream
* aInputStream
, void* aClosure
,
2068 const char* aFromRawSegment
, uint32_t aToOffset
,
2069 uint32_t aCount
, uint32_t* aWriteCount
) {
2070 nsACString
* mimeType
= static_cast<nsACString
*>(aClosure
);
2071 MOZ_ASSERT(mimeType
);
2074 imgLoader::GetMimeTypeFromContent(aFromRawSegment
, aCount
, *mimeType
);
2079 // We don't want to consume data from the stream.
2080 return NS_ERROR_FAILURE
;
2083 nsresult
CreateImageBitmapFromBlob::GetMimeTypeSync(nsACString
& aMimeType
) {
2085 return mInputStream
->ReadSegments(sniff_cb
, &aMimeType
, 128, &dummy
);
2088 nsresult
CreateImageBitmapFromBlob::GetMimeTypeAsync() {
2089 nsCOMPtr
<nsIAsyncInputStream
> asyncInputStream
=
2090 do_QueryInterface(mInputStream
);
2091 if (NS_WARN_IF(!asyncInputStream
)) {
2092 // If the stream is not async, why are we here?
2093 return NS_ERROR_FAILURE
;
2096 return asyncInputStream
->AsyncWait(this, 0, 128, mMainThreadEventTarget
);
2100 CreateImageBitmapFromBlob::OnInputStreamReady(nsIAsyncInputStream
* aStream
) {
2101 // The stream should have data now. Let's start from scratch again.
2102 nsresult rv
= MimeTypeAndDecodeAndCropBlob();
2103 if (NS_WARN_IF(NS_FAILED(rv
))) {
2104 MimeTypeAndDecodeAndCropBlobCompletedMainThread(nullptr, rv
);
2111 CreateImageBitmapFromBlob::OnImageReady(imgIContainer
* aImgContainer
,
2113 MOZ_ASSERT(NS_IsMainThread());
2115 if (NS_FAILED(aStatus
)) {
2116 MimeTypeAndDecodeAndCropBlobCompletedMainThread(nullptr, aStatus
);
2120 MOZ_ASSERT(aImgContainer
);
2122 // Get the surface out.
2123 uint32_t frameFlags
=
2124 imgIContainer::FLAG_SYNC_DECODE
| imgIContainer::FLAG_ASYNC_NOTIFY
;
2125 uint32_t whichFrame
= imgIContainer::FRAME_FIRST
;
2127 if (mOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
2128 frameFlags
|= imgIContainer::FLAG_DECODE_NO_PREMULTIPLY_ALPHA
;
2131 if (mOptions
.mColorSpaceConversion
== ColorSpaceConversion::None
) {
2132 frameFlags
|= imgIContainer::FLAG_DECODE_NO_COLORSPACE_CONVERSION
;
2135 RefPtr
<SourceSurface
> surface
=
2136 aImgContainer
->GetFrame(whichFrame
, frameFlags
);
2138 if (NS_WARN_IF(!surface
)) {
2139 MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2140 nullptr, NS_ERROR_DOM_INVALID_STATE_ERR
);
2144 // Crop the source surface if needed.
2145 RefPtr
<SourceSurface
> croppedSurface
= surface
;
2146 RefPtr
<DataSourceSurface
> dataSurface
= surface
->GetDataSurface();
2148 // force a copy into unprotected memory as a side effect of
2149 // CropAndCopyDataSourceSurface
2150 bool copyRequired
= mCropRect
.isSome() ||
2151 mOptions
.mImageOrientation
== ImageOrientation::FlipY
;
2154 // The blob is just decoded into a RasterImage and not optimized yet, so the
2155 // _surface_ we get is a DataSourceSurface which wraps the RasterImage's
2158 // The _surface_ might already be optimized so that its type is not
2159 // SurfaceType::DATA. However, we could keep using the generic cropping and
2160 // copying since the decoded buffer is only used in this ImageBitmap so we
2161 // should crop it to save memory usage.
2163 // TODO: Bug1189632 is going to refactor this create-from-blob part to
2164 // decode the blob off the main thread. Re-check if we should do
2165 // cropping at this moment again there.
2168 mCropRect
.isSome() ? mCropRect
.ref() : dataSurface
->GetRect();
2170 croppedSurface
= CropAndCopyDataSourceSurface(dataSurface
, cropRect
);
2171 if (NS_WARN_IF(!croppedSurface
)) {
2172 MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2173 nullptr, NS_ERROR_DOM_INVALID_STATE_ERR
);
2177 dataSurface
= croppedSurface
->GetDataSurface();
2179 if (mCropRect
.isSome()) {
2180 mCropRect
->SetRect(0, 0, dataSurface
->GetSize().width
,
2181 dataSurface
->GetSize().height
);
2185 if (mOptions
.mImageOrientation
== ImageOrientation::FlipY
) {
2186 croppedSurface
= FlipYDataSourceSurface(dataSurface
);
2189 if (mOptions
.mResizeWidth
.WasPassed() || mOptions
.mResizeHeight
.WasPassed()) {
2190 dataSurface
= croppedSurface
->GetDataSurface();
2191 croppedSurface
= ScaleDataSourceSurface(dataSurface
, mOptions
);
2192 if (NS_WARN_IF(!croppedSurface
)) {
2193 MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2194 nullptr, NS_ERROR_DOM_INVALID_STATE_ERR
);
2197 if (mCropRect
.isSome()) {
2198 mCropRect
->SetRect(0, 0, croppedSurface
->GetSize().width
,
2199 croppedSurface
->GetSize().height
);
2203 if (NS_WARN_IF(!croppedSurface
)) {
2204 MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2205 nullptr, NS_ERROR_DOM_INVALID_STATE_ERR
);
2209 // Create an Image from the source surface.
2210 RefPtr
<layers::Image
> image
= CreateImageFromSurface(croppedSurface
);
2212 if (NS_WARN_IF(!image
)) {
2213 MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2214 nullptr, NS_ERROR_DOM_INVALID_STATE_ERR
);
2218 MimeTypeAndDecodeAndCropBlobCompletedMainThread(image
, NS_OK
);
2222 void CreateImageBitmapFromBlob::MimeTypeAndDecodeAndCropBlobCompletedMainThread(
2223 layers::Image
* aImage
, nsresult aStatus
) {
2224 MOZ_ASSERT(NS_IsMainThread());
2226 if (!IsCurrentThread()) {
2227 MutexAutoLock
lock(mMutex
);
2230 // The worker is already gone.
2234 RefPtr
<CreateImageBitmapFromBlobRunnable
> r
=
2235 new CreateImageBitmapFromBlobRunnable(mWorkerRef
->Private(), this,
2241 MimeTypeAndDecodeAndCropBlobCompletedOwningThread(aImage
, aStatus
);
2244 void CreateImageBitmapFromBlob::
2245 MimeTypeAndDecodeAndCropBlobCompletedOwningThread(layers::Image
* aImage
,
2247 MOZ_ASSERT(IsCurrentThread());
2250 // The worker is going to be released soon. No needs to continue.
2254 // Let's release what has to be released on the owning thread.
2255 auto raii
= MakeScopeExit([&] {
2256 // Doing this we also release the worker.
2257 mWorkerRef
= nullptr;
2260 mGlobalObject
= nullptr;
2263 if (NS_WARN_IF(NS_FAILED(aStatus
))) {
2264 mPromise
->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR
);
2268 gfxAlphaType alphaType
= gfxAlphaType::Premult
;
2270 if (mOptions
.mPremultiplyAlpha
== PremultiplyAlpha::None
) {
2271 alphaType
= gfxAlphaType::NonPremult
;
2274 // Create ImageBitmap object.
2275 RefPtr
<ImageBitmap
> imageBitmap
=
2276 new ImageBitmap(mGlobalObject
, aImage
, false /* write-only */, alphaType
);
2278 if (mCropRect
.isSome()) {
2280 imageBitmap
->SetPictureRect(mCropRect
.ref(), rv
);
2283 mPromise
->MaybeReject(std::move(rv
));
2288 imageBitmap
->mAllocatedImageData
= true;
2290 mPromise
->MaybeResolve(imageBitmap
);
2293 void CreateImageBitmapFromBlob::WorkerShuttingDown() {
2294 MOZ_ASSERT(IsCurrentThread());
2296 MutexAutoLock
lock(mMutex
);
2298 // Let's release all the non-thread-safe objects now.
2299 mWorkerRef
= nullptr;
2301 mGlobalObject
= nullptr;
2304 } // namespace mozilla::dom