1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "ImageEncoder.h"
8 #include "mozilla/dom/CanvasRenderingContext2D.h"
9 #include "mozilla/dom/GeneratePlaceholderCanvasData.h"
10 #include "mozilla/dom/MemoryBlobImpl.h"
11 #include "mozilla/dom/OffscreenCanvasDisplayHelper.h"
12 #include "mozilla/dom/WorkerPrivate.h"
13 #include "mozilla/gfx/2D.h"
14 #include "mozilla/gfx/DataSurfaceHelpers.h"
15 #include "mozilla/layers/CanvasRenderer.h"
16 #include "mozilla/RefPtr.h"
17 #include "mozilla/SyncRunnable.h"
18 #include "mozilla/Unused.h"
20 #include "nsComponentManagerUtils.h"
21 #include "nsThreadUtils.h"
22 #include "nsNetUtil.h"
23 #include "nsXPCOMCIDInternal.h"
24 #include "YCbCrUtils.h"
26 using namespace mozilla::gfx
;
28 namespace mozilla::dom
{
30 // This class should be placed inside GetBRGADataSourceSurfaceSync(). However,
31 // due to B2G ICS uses old complier (C++98/03) which forbids local class as
32 // template parameter, we need to move this class outside.
33 class SurfaceHelper
: public Runnable
{
35 explicit SurfaceHelper(already_AddRefed
<layers::Image
> aImage
)
36 : Runnable("SurfaceHelper"), mImage(aImage
) {}
38 // It retrieves a SourceSurface reference and convert color format on main
39 // thread and passes DataSourceSurface to caller thread.
40 NS_IMETHOD
Run() override
{
41 RefPtr
<gfx::SourceSurface
> surface
= mImage
->GetAsSourceSurface();
43 if (surface
->GetFormat() == gfx::SurfaceFormat::B8G8R8A8
) {
44 mDataSourceSurface
= surface
->GetDataSurface();
46 mDataSourceSurface
= gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(
47 surface
, gfx::SurfaceFormat::B8G8R8A8
);
50 // It guarantees the reference will be released on main thread.
51 NS_ReleaseOnMainThread("SurfaceHelper::surface", surface
.forget());
55 already_AddRefed
<gfx::DataSourceSurface
> GetDataSurfaceSafe() {
56 nsCOMPtr
<nsIEventTarget
> mainTarget
= GetMainThreadSerialEventTarget();
57 MOZ_ASSERT(mainTarget
);
58 SyncRunnable::DispatchToThread(mainTarget
, this, false);
60 return mDataSourceSurface
.forget();
64 RefPtr
<layers::Image
> mImage
;
65 RefPtr
<gfx::DataSourceSurface
> mDataSourceSurface
;
68 // This function returns a DataSourceSurface in B8G8R8A8 format.
69 // It uses SourceSurface to do format convert. Because most SourceSurface in
70 // image formats should be referenced or dereferenced on main thread, it uses a
71 // sync class SurfaceHelper to retrieve SourceSurface and convert to B8G8R8A8 on
73 already_AddRefed
<DataSourceSurface
> GetBRGADataSourceSurfaceSync(
74 already_AddRefed
<layers::Image
> aImage
) {
75 RefPtr
<SurfaceHelper
> helper
= new SurfaceHelper(std::move(aImage
));
76 return helper
->GetDataSurfaceSafe();
79 class EncodingCompleteEvent final
: public DiscardableRunnable
{
80 virtual ~EncodingCompleteEvent() = default;
83 explicit EncodingCompleteEvent(
84 EncodeCompleteCallback
* aEncodeCompleteCallback
)
85 : DiscardableRunnable("EncodingCompleteEvent"),
88 mEncodeCompleteCallback(aEncodeCompleteCallback
),
90 if (!NS_IsMainThread() && IsCurrentThreadRunningWorker()) {
91 mCreationEventTarget
= GetCurrentSerialEventTarget();
93 mCreationEventTarget
= GetMainThreadSerialEventTarget();
97 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. See
99 MOZ_CAN_RUN_SCRIPT_BOUNDARY
100 NS_IMETHOD
Run() override
{
103 // We want to null out mEncodeCompleteCallback no matter what.
104 RefPtr
<EncodeCompleteCallback
> callback(std::move(mEncodeCompleteCallback
));
106 RefPtr
<BlobImpl
> blobImpl
= new MemoryBlobImpl(mImgData
, mImgSize
, mType
);
107 rv
= callback
->ReceiveBlobImpl(blobImpl
.forget());
109 rv
= callback
->ReceiveBlobImpl(nullptr);
115 void SetMembers(void* aImgData
, uint64_t aImgSize
,
116 const nsAutoString
& aType
) {
122 void SetFailed() { mFailed
= true; }
124 nsIEventTarget
* GetCreationThreadEventTarget() {
125 return mCreationEventTarget
;
128 bool CanBeDeletedOnAnyThread() {
129 return !mEncodeCompleteCallback
||
130 mEncodeCompleteCallback
->CanBeDeletedOnAnyThread();
137 nsCOMPtr
<nsIEventTarget
> mCreationEventTarget
;
138 RefPtr
<EncodeCompleteCallback
> mEncodeCompleteCallback
;
142 class EncodingRunnable
: public Runnable
{
143 virtual ~EncodingRunnable() = default;
146 NS_INLINE_DECL_REFCOUNTING_INHERITED(EncodingRunnable
, Runnable
)
148 EncodingRunnable(const nsAString
& aType
, const nsAString
& aOptions
,
149 UniquePtr
<uint8_t[]> aImageBuffer
, layers::Image
* aImage
,
150 imgIEncoder
* aEncoder
,
151 EncodingCompleteEvent
* aEncodingCompleteEvent
,
152 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
153 bool aUsingCustomOptions
)
154 : Runnable("EncodingRunnable"),
157 mImageBuffer(std::move(aImageBuffer
)),
160 mEncodingCompleteEvent(aEncodingCompleteEvent
),
163 mUsePlaceholder(aUsePlaceholder
),
164 mUsingCustomOptions(aUsingCustomOptions
) {}
166 nsresult
ProcessImageData(uint64_t* aImgSize
, void** aImgData
) {
167 nsCOMPtr
<nsIInputStream
> stream
;
168 nsresult rv
= ImageEncoder::ExtractDataInternal(
169 mType
, mOptions
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
170 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
172 // If there are unrecognized custom parse options, we should fall back to
173 // the default values for the encoder without any options at all.
174 if (rv
== NS_ERROR_INVALID_ARG
&& mUsingCustomOptions
) {
175 rv
= ImageEncoder::ExtractDataInternal(
176 mType
, u
""_ns
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
177 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
179 NS_ENSURE_SUCCESS(rv
, rv
);
181 rv
= NS_ReadInputStreamToBuffer(stream
, aImgData
, -1, aImgSize
);
182 NS_ENSURE_SUCCESS(rv
, rv
);
187 NS_IMETHOD
Run() override
{
189 void* imgData
= nullptr;
191 nsresult rv
= ProcessImageData(&imgSize
, &imgData
);
193 mEncodingCompleteEvent
->SetFailed();
195 mEncodingCompleteEvent
->SetMembers(imgData
, imgSize
, mType
);
197 rv
= mEncodingCompleteEvent
->GetCreationThreadEventTarget()->Dispatch(
198 mEncodingCompleteEvent
, nsIThread::DISPATCH_NORMAL
);
200 if (!mEncodingCompleteEvent
->CanBeDeletedOnAnyThread()) {
201 // Better to leak than to crash.
202 Unused
<< mEncodingCompleteEvent
.forget();
212 nsAutoString mOptions
;
213 UniquePtr
<uint8_t[]> mImageBuffer
;
214 RefPtr
<layers::Image
> mImage
;
215 nsCOMPtr
<imgIEncoder
> mEncoder
;
216 RefPtr
<EncodingCompleteEvent
> mEncodingCompleteEvent
;
218 const nsIntSize mSize
;
219 bool mUsePlaceholder
;
220 bool mUsingCustomOptions
;
224 nsresult
ImageEncoder::ExtractData(
225 nsAString
& aType
, const nsAString
& aOptions
, const nsIntSize aSize
,
226 bool aUsePlaceholder
, nsICanvasRenderingContextInternal
* aContext
,
227 OffscreenCanvasDisplayHelper
* aOffscreenDisplay
, nsIInputStream
** aStream
) {
228 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
230 return NS_IMAGELIB_ERROR_NO_ENCODER
;
233 return ExtractDataInternal(aType
, aOptions
, nullptr, 0, aSize
,
234 aUsePlaceholder
, nullptr, aContext
,
235 aOffscreenDisplay
, aStream
, encoder
);
239 nsresult
ImageEncoder::ExtractDataFromLayersImageAsync(
240 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
241 layers::Image
* aImage
, bool aUsePlaceholder
,
242 EncodeCompleteCallback
* aEncodeCallback
) {
243 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
245 return NS_IMAGELIB_ERROR_NO_ENCODER
;
248 RefPtr
<EncodingCompleteEvent
> completeEvent
=
249 new EncodingCompleteEvent(aEncodeCallback
);
251 nsIntSize
size(aImage
->GetSize().width
, aImage
->GetSize().height
);
252 nsCOMPtr
<nsIRunnable
> event
=
253 new EncodingRunnable(aType
, aOptions
, nullptr, aImage
, encoder
,
254 completeEvent
, imgIEncoder::INPUT_FORMAT_HOSTARGB
,
255 size
, aUsePlaceholder
, aUsingCustomOptions
);
256 return NS_DispatchBackgroundTask(event
.forget());
260 nsresult
ImageEncoder::ExtractDataAsync(
261 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
262 UniquePtr
<uint8_t[]> aImageBuffer
, int32_t aFormat
, const nsIntSize aSize
,
263 bool aUsePlaceholder
, EncodeCompleteCallback
* aEncodeCallback
) {
264 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
266 return NS_IMAGELIB_ERROR_NO_ENCODER
;
269 RefPtr
<EncodingCompleteEvent
> completeEvent
=
270 new EncodingCompleteEvent(aEncodeCallback
);
272 nsCOMPtr
<nsIRunnable
> event
= new EncodingRunnable(
273 aType
, aOptions
, std::move(aImageBuffer
), nullptr, encoder
, completeEvent
,
274 aFormat
, aSize
, aUsePlaceholder
, aUsingCustomOptions
);
275 return NS_DispatchBackgroundTask(event
.forget());
279 nsresult
ImageEncoder::GetInputStream(int32_t aWidth
, int32_t aHeight
,
280 uint8_t* aImageBuffer
, int32_t aFormat
,
281 imgIEncoder
* aEncoder
,
282 const nsAString
& aEncoderOptions
,
283 nsIInputStream
** aStream
) {
285 aEncoder
->InitFromData(aImageBuffer
, aWidth
* aHeight
* 4, aWidth
,
286 aHeight
, aWidth
* 4, aFormat
, aEncoderOptions
);
287 NS_ENSURE_SUCCESS(rv
, rv
);
289 nsCOMPtr
<imgIEncoder
> encoder(aEncoder
);
290 encoder
.forget(aStream
);
295 nsresult
ImageEncoder::ExtractDataInternal(
296 const nsAString
& aType
, const nsAString
& aOptions
, uint8_t* aImageBuffer
,
297 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
298 layers::Image
* aImage
, nsICanvasRenderingContextInternal
* aContext
,
299 OffscreenCanvasDisplayHelper
* aOffscreenDisplay
, nsIInputStream
** aStream
,
300 imgIEncoder
* aEncoder
) {
301 if (aSize
.IsEmpty()) {
302 return NS_ERROR_INVALID_ARG
;
305 nsCOMPtr
<nsIInputStream
> imgStream
;
309 if (aImageBuffer
&& !aUsePlaceholder
) {
310 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
311 return NS_ERROR_INVALID_ARG
;
314 rv
= ImageEncoder::GetInputStream(aSize
.width
, aSize
.height
, aImageBuffer
,
315 aFormat
, aEncoder
, aOptions
,
316 getter_AddRefs(imgStream
));
317 } else if (aContext
&& !aUsePlaceholder
) {
318 NS_ConvertUTF16toUTF8
encoderType(aType
);
319 rv
= aContext
->GetInputStream(encoderType
.get(), aOptions
,
320 getter_AddRefs(imgStream
));
321 } else if (aOffscreenDisplay
&& !aUsePlaceholder
) {
322 const NS_ConvertUTF16toUTF8
encoderType(aType
);
323 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
324 return NS_ERROR_INVALID_ARG
;
327 const RefPtr
<SourceSurface
> snapshot
=
328 aOffscreenDisplay
->GetSurfaceSnapshot();
330 return NS_ERROR_OUT_OF_MEMORY
;
333 const RefPtr
<DataSourceSurface
> data
= snapshot
->GetDataSurface();
335 return NS_ERROR_OUT_OF_MEMORY
;
339 DataSourceSurface::MappedSurface map
;
340 if (!data
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
341 return NS_ERROR_INVALID_ARG
;
343 auto size
= data
->GetSize();
344 rv
= aEncoder
->InitFromData(map
.mData
, size
.width
* size
.height
* 4,
345 size
.width
, size
.height
, size
.width
* 4,
346 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
349 if (NS_SUCCEEDED(rv
)) {
350 imgStream
= aEncoder
;
352 } else if (aImage
&& !aUsePlaceholder
) {
353 // It is safe to convert PlanarYCbCr format from YUV to RGB off-main-thread.
354 // Other image formats could have problem to convert format off-main-thread.
355 // So here it uses a help function GetBRGADataSourceSurfaceSync() to convert
356 // format on main thread.
357 if (aImage
->GetFormat() == ImageFormat::PLANAR_YCBCR
) {
358 nsTArray
<uint8_t> data
;
359 layers::PlanarYCbCrImage
* ycbcrImage
=
360 static_cast<layers::PlanarYCbCrImage
*>(aImage
);
361 gfxImageFormat format
= SurfaceFormat::A8R8G8B8_UINT32
;
362 uint32_t stride
= GetAlignedStride
<16>(aSize
.width
, 4);
363 size_t length
= BufferSizeFromStrideAndHeight(stride
, aSize
.height
);
365 return NS_ERROR_INVALID_ARG
;
367 data
.SetCapacity(length
);
369 ConvertYCbCrToRGB(*ycbcrImage
->GetData(), format
, aSize
, data
.Elements(),
372 rv
= aEncoder
->InitFromData(data
.Elements(),
373 aSize
.width
* aSize
.height
* 4, aSize
.width
,
374 aSize
.height
, aSize
.width
* 4,
375 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
377 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
378 return NS_ERROR_INVALID_ARG
;
381 RefPtr
<gfx::DataSourceSurface
> dataSurface
;
382 RefPtr
<layers::Image
> image(aImage
);
383 dataSurface
= GetBRGADataSourceSurfaceSync(image
.forget());
385 DataSourceSurface::MappedSurface map
;
386 if (!dataSurface
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
387 return NS_ERROR_INVALID_ARG
;
389 auto size
= dataSurface
->GetSize();
390 rv
= aEncoder
->InitFromData(map
.mData
, size
.width
* size
.height
* 4,
391 size
.width
, size
.height
, size
.width
* 4,
392 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
393 dataSurface
->Unmap();
396 if (NS_SUCCEEDED(rv
)) {
397 imgStream
= aEncoder
;
400 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
401 return NS_ERROR_INVALID_ARG
;
404 // no context, so we have to encode an empty image
405 // note that if we didn't have a current context, the spec says we're
406 // supposed to just return transparent black pixels of the canvas
408 RefPtr
<DataSourceSurface
> emptyCanvas
=
409 Factory::CreateDataSourceSurfaceWithStride(
410 IntSize(aSize
.width
, aSize
.height
), SurfaceFormat::B8G8R8A8
,
411 4 * aSize
.width
, true);
412 if (NS_WARN_IF(!emptyCanvas
)) {
413 return NS_ERROR_INVALID_ARG
;
416 DataSourceSurface::MappedSurface map
;
417 if (!emptyCanvas
->Map(DataSourceSurface::MapType::WRITE
, &map
)) {
418 return NS_ERROR_INVALID_ARG
;
420 if (aUsePlaceholder
) {
421 auto size
= 4 * aSize
.width
* aSize
.height
;
422 auto* data
= map
.mData
;
423 GeneratePlaceholderCanvasData(size
, data
);
425 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
426 aSize
.width
, aSize
.height
, aSize
.width
* 4,
427 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
428 emptyCanvas
->Unmap();
429 if (NS_SUCCEEDED(rv
)) {
430 imgStream
= aEncoder
;
433 NS_ENSURE_SUCCESS(rv
, rv
);
435 imgStream
.forget(aStream
);
440 already_AddRefed
<imgIEncoder
> ImageEncoder::GetImageEncoder(nsAString
& aType
) {
441 // Get an image encoder for the media type.
442 nsCString
encoderCID("@mozilla.org/image/encoder;2?type=");
443 NS_ConvertUTF16toUTF8
encoderType(aType
);
444 encoderCID
+= encoderType
;
445 nsCOMPtr
<imgIEncoder
> encoder
= do_CreateInstance(encoderCID
.get());
447 if (!encoder
&& aType
!= u
"image/png"_ns
) {
448 // Unable to create an encoder instance of the specified type. Falling back
450 aType
.AssignLiteral("image/png");
451 nsCString
PNGEncoderCID("@mozilla.org/image/encoder;2?type=image/png");
452 encoder
= do_CreateInstance(PNGEncoderCID
.get());
455 return encoder
.forget();
458 } // namespace mozilla::dom