1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "ImageEncoder.h"
8 #include "mozilla/dom/CanvasRenderingContext2D.h"
9 #include "mozilla/dom/GeneratePlaceholderCanvasData.h"
10 #include "mozilla/dom/MemoryBlobImpl.h"
11 #include "mozilla/dom/OffscreenCanvasDisplayHelper.h"
12 #include "mozilla/dom/WorkerPrivate.h"
13 #include "mozilla/gfx/2D.h"
14 #include "mozilla/gfx/DataSurfaceHelpers.h"
15 #include "mozilla/layers/CanvasRenderer.h"
16 #include "mozilla/RefPtr.h"
17 #include "mozilla/SyncRunnable.h"
18 #include "mozilla/Unused.h"
20 #include "nsComponentManagerUtils.h"
21 #include "nsThreadUtils.h"
22 #include "nsNetUtil.h"
23 #include "nsXPCOMCIDInternal.h"
24 #include "YCbCrUtils.h"
26 using namespace mozilla::gfx
;
28 namespace mozilla::dom
{
30 // This class should be placed inside GetBRGADataSourceSurfaceSync(). However,
31 // due to B2G ICS uses old complier (C++98/03) which forbids local class as
32 // template parameter, we need to move this class outside.
33 class SurfaceHelper
: public Runnable
{
35 explicit SurfaceHelper(already_AddRefed
<layers::Image
> aImage
)
36 : Runnable("SurfaceHelper"), mImage(aImage
) {}
38 // It retrieves a SourceSurface reference and convert color format on main
39 // thread and passes DataSourceSurface to caller thread.
40 NS_IMETHOD
Run() override
{
41 RefPtr
<gfx::SourceSurface
> surface
= mImage
->GetAsSourceSurface();
43 if (surface
->GetFormat() == gfx::SurfaceFormat::B8G8R8A8
) {
44 mDataSourceSurface
= surface
->GetDataSurface();
46 mDataSourceSurface
= gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(
47 surface
, gfx::SurfaceFormat::B8G8R8A8
);
50 // It guarantees the reference will be released on main thread.
51 NS_ReleaseOnMainThread("SurfaceHelper::surface", surface
.forget());
55 already_AddRefed
<gfx::DataSourceSurface
> GetDataSurfaceSafe() {
56 nsCOMPtr
<nsIEventTarget
> mainTarget
= GetMainThreadSerialEventTarget();
57 MOZ_ASSERT(mainTarget
);
58 SyncRunnable::DispatchToThread(mainTarget
, this, false);
60 return mDataSourceSurface
.forget();
64 RefPtr
<layers::Image
> mImage
;
65 RefPtr
<gfx::DataSourceSurface
> mDataSourceSurface
;
68 // This function returns a DataSourceSurface in B8G8R8A8 format.
69 // It uses SourceSurface to do format convert. Because most SourceSurface in
70 // image formats should be referenced or dereferenced on main thread, it uses a
71 // sync class SurfaceHelper to retrieve SourceSurface and convert to B8G8R8A8 on
73 already_AddRefed
<DataSourceSurface
> GetBRGADataSourceSurfaceSync(
74 already_AddRefed
<layers::Image
> aImage
) {
75 RefPtr
<SurfaceHelper
> helper
= new SurfaceHelper(std::move(aImage
));
76 return helper
->GetDataSurfaceSafe();
79 class EncodingCompleteEvent final
: public DiscardableRunnable
{
80 virtual ~EncodingCompleteEvent() = default;
83 explicit EncodingCompleteEvent(
84 EncodeCompleteCallback
* aEncodeCompleteCallback
)
85 : DiscardableRunnable("EncodingCompleteEvent"),
89 mEncodeCompleteCallback(aEncodeCompleteCallback
),
91 if (!NS_IsMainThread() && IsCurrentThreadRunningWorker()) {
92 mCreationEventTarget
= GetCurrentSerialEventTarget();
94 mCreationEventTarget
= GetMainThreadSerialEventTarget();
98 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. See
100 MOZ_CAN_RUN_SCRIPT_BOUNDARY
101 NS_IMETHOD
Run() override
{
104 // We want to null out mEncodeCompleteCallback no matter what.
105 RefPtr
<EncodeCompleteCallback
> callback(std::move(mEncodeCompleteCallback
));
107 RefPtr
<BlobImpl
> blobImpl
= new MemoryBlobImpl(mImgData
, mImgSize
, mType
);
108 rv
= callback
->ReceiveBlobImpl(blobImpl
.forget());
110 rv
= callback
->ReceiveBlobImpl(nullptr);
116 void SetMembers(void* aImgData
, uint64_t aImgSize
,
117 const nsAutoString
& aType
) {
123 void SetFailed() { mFailed
= true; }
125 nsIEventTarget
* GetCreationThreadEventTarget() {
126 return mCreationEventTarget
;
129 bool CanBeDeletedOnAnyThread() {
130 return !mEncodeCompleteCallback
||
131 mEncodeCompleteCallback
->CanBeDeletedOnAnyThread();
138 nsCOMPtr
<nsIEventTarget
> mCreationEventTarget
;
139 RefPtr
<EncodeCompleteCallback
> mEncodeCompleteCallback
;
143 class EncodingRunnable
: public Runnable
{
144 virtual ~EncodingRunnable() = default;
147 NS_INLINE_DECL_REFCOUNTING_INHERITED(EncodingRunnable
, Runnable
)
149 EncodingRunnable(const nsAString
& aType
, const nsAString
& aOptions
,
150 UniquePtr
<uint8_t[]> aImageBuffer
, layers::Image
* aImage
,
151 imgIEncoder
* aEncoder
,
152 EncodingCompleteEvent
* aEncodingCompleteEvent
,
153 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
154 bool aUsingCustomOptions
)
155 : Runnable("EncodingRunnable"),
158 mImageBuffer(std::move(aImageBuffer
)),
161 mEncodingCompleteEvent(aEncodingCompleteEvent
),
164 mUsePlaceholder(aUsePlaceholder
),
165 mUsingCustomOptions(aUsingCustomOptions
) {}
167 nsresult
ProcessImageData(uint64_t* aImgSize
, void** aImgData
) {
168 nsCOMPtr
<nsIInputStream
> stream
;
169 nsresult rv
= ImageEncoder::ExtractDataInternal(
170 mType
, mOptions
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
171 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
173 // If there are unrecognized custom parse options, we should fall back to
174 // the default values for the encoder without any options at all.
175 if (rv
== NS_ERROR_INVALID_ARG
&& mUsingCustomOptions
) {
176 rv
= ImageEncoder::ExtractDataInternal(
177 mType
, u
""_ns
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
178 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
180 NS_ENSURE_SUCCESS(rv
, rv
);
182 rv
= NS_ReadInputStreamToBuffer(stream
, aImgData
, -1, aImgSize
);
183 NS_ENSURE_SUCCESS(rv
, rv
);
188 NS_IMETHOD
Run() override
{
190 void* imgData
= nullptr;
192 nsresult rv
= ProcessImageData(&imgSize
, &imgData
);
194 mEncodingCompleteEvent
->SetFailed();
196 mEncodingCompleteEvent
->SetMembers(imgData
, imgSize
, mType
);
198 rv
= mEncodingCompleteEvent
->GetCreationThreadEventTarget()->Dispatch(
199 mEncodingCompleteEvent
, nsIThread::DISPATCH_NORMAL
);
201 if (!mEncodingCompleteEvent
->CanBeDeletedOnAnyThread()) {
202 // Better to leak than to crash.
203 Unused
<< mEncodingCompleteEvent
.forget();
213 nsAutoString mOptions
;
214 UniquePtr
<uint8_t[]> mImageBuffer
;
215 RefPtr
<layers::Image
> mImage
;
216 nsCOMPtr
<imgIEncoder
> mEncoder
;
217 RefPtr
<EncodingCompleteEvent
> mEncodingCompleteEvent
;
219 const nsIntSize mSize
;
220 bool mUsePlaceholder
;
221 bool mUsingCustomOptions
;
225 nsresult
ImageEncoder::ExtractData(
226 nsAString
& aType
, const nsAString
& aOptions
, const nsIntSize aSize
,
227 bool aUsePlaceholder
, nsICanvasRenderingContextInternal
* aContext
,
228 OffscreenCanvasDisplayHelper
* aOffscreenDisplay
, nsIInputStream
** aStream
) {
229 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
231 return NS_IMAGELIB_ERROR_NO_ENCODER
;
234 return ExtractDataInternal(aType
, aOptions
, nullptr, 0, aSize
,
235 aUsePlaceholder
, nullptr, aContext
,
236 aOffscreenDisplay
, aStream
, encoder
);
240 nsresult
ImageEncoder::ExtractDataFromLayersImageAsync(
241 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
242 layers::Image
* aImage
, bool aUsePlaceholder
,
243 EncodeCompleteCallback
* aEncodeCallback
) {
244 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
246 return NS_IMAGELIB_ERROR_NO_ENCODER
;
249 RefPtr
<EncodingCompleteEvent
> completeEvent
=
250 new EncodingCompleteEvent(aEncodeCallback
);
252 nsIntSize
size(aImage
->GetSize().width
, aImage
->GetSize().height
);
253 nsCOMPtr
<nsIRunnable
> event
=
254 new EncodingRunnable(aType
, aOptions
, nullptr, aImage
, encoder
,
255 completeEvent
, imgIEncoder::INPUT_FORMAT_HOSTARGB
,
256 size
, aUsePlaceholder
, aUsingCustomOptions
);
257 return NS_DispatchBackgroundTask(event
.forget());
261 nsresult
ImageEncoder::ExtractDataAsync(
262 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
263 UniquePtr
<uint8_t[]> aImageBuffer
, int32_t aFormat
, const nsIntSize aSize
,
264 bool aUsePlaceholder
, EncodeCompleteCallback
* aEncodeCallback
) {
265 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
267 return NS_IMAGELIB_ERROR_NO_ENCODER
;
270 RefPtr
<EncodingCompleteEvent
> completeEvent
=
271 new EncodingCompleteEvent(aEncodeCallback
);
273 nsCOMPtr
<nsIRunnable
> event
= new EncodingRunnable(
274 aType
, aOptions
, std::move(aImageBuffer
), nullptr, encoder
, completeEvent
,
275 aFormat
, aSize
, aUsePlaceholder
, aUsingCustomOptions
);
276 return NS_DispatchBackgroundTask(event
.forget());
280 nsresult
ImageEncoder::GetInputStream(int32_t aWidth
, int32_t aHeight
,
281 uint8_t* aImageBuffer
, int32_t aFormat
,
282 imgIEncoder
* aEncoder
,
283 const nsAString
& aEncoderOptions
,
284 nsIInputStream
** aStream
) {
286 aEncoder
->InitFromData(aImageBuffer
, aWidth
* aHeight
* 4, aWidth
,
287 aHeight
, aWidth
* 4, aFormat
, aEncoderOptions
);
288 NS_ENSURE_SUCCESS(rv
, rv
);
290 nsCOMPtr
<imgIEncoder
> encoder(aEncoder
);
291 encoder
.forget(aStream
);
296 nsresult
ImageEncoder::ExtractDataInternal(
297 const nsAString
& aType
, const nsAString
& aOptions
, uint8_t* aImageBuffer
,
298 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
299 layers::Image
* aImage
, nsICanvasRenderingContextInternal
* aContext
,
300 OffscreenCanvasDisplayHelper
* aOffscreenDisplay
, nsIInputStream
** aStream
,
301 imgIEncoder
* aEncoder
) {
302 if (aSize
.IsEmpty()) {
303 return NS_ERROR_INVALID_ARG
;
306 nsCOMPtr
<nsIInputStream
> imgStream
;
310 if (aImageBuffer
&& !aUsePlaceholder
) {
311 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
312 return NS_ERROR_INVALID_ARG
;
315 rv
= ImageEncoder::GetInputStream(aSize
.width
, aSize
.height
, aImageBuffer
,
316 aFormat
, aEncoder
, aOptions
,
317 getter_AddRefs(imgStream
));
318 } else if (aContext
&& !aUsePlaceholder
) {
319 NS_ConvertUTF16toUTF8
encoderType(aType
);
320 rv
= aContext
->GetInputStream(encoderType
.get(), aOptions
,
321 getter_AddRefs(imgStream
));
322 } else if (aOffscreenDisplay
&& !aUsePlaceholder
) {
323 const NS_ConvertUTF16toUTF8
encoderType(aType
);
324 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
325 return NS_ERROR_INVALID_ARG
;
328 const RefPtr
<SourceSurface
> snapshot
=
329 aOffscreenDisplay
->GetSurfaceSnapshot();
331 return NS_ERROR_OUT_OF_MEMORY
;
334 const RefPtr
<DataSourceSurface
> data
= snapshot
->GetDataSurface();
336 return NS_ERROR_OUT_OF_MEMORY
;
340 DataSourceSurface::MappedSurface map
;
341 if (!data
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
342 return NS_ERROR_INVALID_ARG
;
344 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
345 aSize
.width
, aSize
.height
, aSize
.width
* 4,
346 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
349 if (NS_SUCCEEDED(rv
)) {
350 imgStream
= aEncoder
;
352 } else if (aImage
&& !aUsePlaceholder
) {
353 // It is safe to convert PlanarYCbCr format from YUV to RGB off-main-thread.
354 // Other image formats could have problem to convert format off-main-thread.
355 // So here it uses a help function GetBRGADataSourceSurfaceSync() to convert
356 // format on main thread.
357 if (aImage
->GetFormat() == ImageFormat::PLANAR_YCBCR
) {
358 nsTArray
<uint8_t> data
;
359 layers::PlanarYCbCrImage
* ycbcrImage
=
360 static_cast<layers::PlanarYCbCrImage
*>(aImage
);
361 gfxImageFormat format
= SurfaceFormat::A8R8G8B8_UINT32
;
362 uint32_t stride
= GetAlignedStride
<16>(aSize
.width
, 4);
363 size_t length
= BufferSizeFromStrideAndHeight(stride
, aSize
.height
);
365 return NS_ERROR_INVALID_ARG
;
367 data
.SetCapacity(length
);
369 ConvertYCbCrToRGB(*ycbcrImage
->GetData(), format
, aSize
, data
.Elements(),
372 rv
= aEncoder
->InitFromData(data
.Elements(),
373 aSize
.width
* aSize
.height
* 4, aSize
.width
,
374 aSize
.height
, aSize
.width
* 4,
375 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
377 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
378 return NS_ERROR_INVALID_ARG
;
381 RefPtr
<gfx::DataSourceSurface
> dataSurface
;
382 RefPtr
<layers::Image
> image(aImage
);
383 dataSurface
= GetBRGADataSourceSurfaceSync(image
.forget());
385 DataSourceSurface::MappedSurface map
;
386 if (!dataSurface
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
387 return NS_ERROR_INVALID_ARG
;
389 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
390 aSize
.width
, aSize
.height
, aSize
.width
* 4,
391 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
392 dataSurface
->Unmap();
395 if (NS_SUCCEEDED(rv
)) {
396 imgStream
= aEncoder
;
399 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
400 return NS_ERROR_INVALID_ARG
;
403 // no context, so we have to encode an empty image
404 // note that if we didn't have a current context, the spec says we're
405 // supposed to just return transparent black pixels of the canvas
407 RefPtr
<DataSourceSurface
> emptyCanvas
=
408 Factory::CreateDataSourceSurfaceWithStride(
409 IntSize(aSize
.width
, aSize
.height
), SurfaceFormat::B8G8R8A8
,
410 4 * aSize
.width
, true);
411 if (NS_WARN_IF(!emptyCanvas
)) {
412 return NS_ERROR_INVALID_ARG
;
415 DataSourceSurface::MappedSurface map
;
416 if (!emptyCanvas
->Map(DataSourceSurface::MapType::WRITE
, &map
)) {
417 return NS_ERROR_INVALID_ARG
;
419 if (aUsePlaceholder
) {
420 auto size
= 4 * aSize
.width
* aSize
.height
;
421 auto* data
= map
.mData
;
422 GeneratePlaceholderCanvasData(size
, data
);
424 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
425 aSize
.width
, aSize
.height
, aSize
.width
* 4,
426 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
427 emptyCanvas
->Unmap();
428 if (NS_SUCCEEDED(rv
)) {
429 imgStream
= aEncoder
;
432 NS_ENSURE_SUCCESS(rv
, rv
);
434 imgStream
.forget(aStream
);
439 already_AddRefed
<imgIEncoder
> ImageEncoder::GetImageEncoder(nsAString
& aType
) {
440 // Get an image encoder for the media type.
441 nsCString
encoderCID("@mozilla.org/image/encoder;2?type=");
442 NS_ConvertUTF16toUTF8
encoderType(aType
);
443 encoderCID
+= encoderType
;
444 nsCOMPtr
<imgIEncoder
> encoder
= do_CreateInstance(encoderCID
.get());
446 if (!encoder
&& aType
!= u
"image/png"_ns
) {
447 // Unable to create an encoder instance of the specified type. Falling back
449 aType
.AssignLiteral("image/png");
450 nsCString
PNGEncoderCID("@mozilla.org/image/encoder;2?type=image/png");
451 encoder
= do_CreateInstance(PNGEncoderCID
.get());
454 return encoder
.forget();
457 } // namespace mozilla::dom