1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "ImageEncoder.h"
8 #include "mozilla/dom/CanvasRenderingContext2D.h"
9 #include "mozilla/dom/GeneratePlaceholderCanvasData.h"
10 #include "mozilla/dom/MemoryBlobImpl.h"
11 #include "mozilla/dom/WorkerPrivate.h"
12 #include "mozilla/gfx/2D.h"
13 #include "mozilla/gfx/DataSurfaceHelpers.h"
14 #include "mozilla/layers/CanvasRenderer.h"
15 #include "mozilla/RefPtr.h"
16 #include "mozilla/SyncRunnable.h"
17 #include "mozilla/Unused.h"
19 #include "nsComponentManagerUtils.h"
20 #include "nsThreadUtils.h"
21 #include "nsNetUtil.h"
22 #include "nsXPCOMCIDInternal.h"
23 #include "YCbCrUtils.h"
25 using namespace mozilla::gfx
;
27 namespace mozilla::dom
{
29 // This class should be placed inside GetBRGADataSourceSurfaceSync(). However,
30 // due to B2G ICS uses old complier (C++98/03) which forbids local class as
31 // template parameter, we need to move this class outside.
32 class SurfaceHelper
: public Runnable
{
34 explicit SurfaceHelper(already_AddRefed
<layers::Image
> aImage
)
35 : Runnable("SurfaceHelper"), mImage(aImage
) {}
37 // It retrieves a SourceSurface reference and convert color format on main
38 // thread and passes DataSourceSurface to caller thread.
39 NS_IMETHOD
Run() override
{
40 RefPtr
<gfx::SourceSurface
> surface
= mImage
->GetAsSourceSurface();
42 if (surface
->GetFormat() == gfx::SurfaceFormat::B8G8R8A8
) {
43 mDataSourceSurface
= surface
->GetDataSurface();
45 mDataSourceSurface
= gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(
46 surface
, gfx::SurfaceFormat::B8G8R8A8
);
49 // It guarantees the reference will be released on main thread.
50 NS_ReleaseOnMainThread("SurfaceHelper::surface", surface
.forget());
54 already_AddRefed
<gfx::DataSourceSurface
> GetDataSurfaceSafe() {
55 nsCOMPtr
<nsIEventTarget
> mainTarget
= GetMainThreadSerialEventTarget();
56 MOZ_ASSERT(mainTarget
);
57 SyncRunnable::DispatchToThread(mainTarget
, this, false);
59 return mDataSourceSurface
.forget();
63 RefPtr
<layers::Image
> mImage
;
64 RefPtr
<gfx::DataSourceSurface
> mDataSourceSurface
;
67 // This function returns a DataSourceSurface in B8G8R8A8 format.
68 // It uses SourceSurface to do format convert. Because most SourceSurface in
69 // image formats should be referenced or dereferenced on main thread, it uses a
70 // sync class SurfaceHelper to retrieve SourceSurface and convert to B8G8R8A8 on
72 already_AddRefed
<DataSourceSurface
> GetBRGADataSourceSurfaceSync(
73 already_AddRefed
<layers::Image
> aImage
) {
74 RefPtr
<SurfaceHelper
> helper
= new SurfaceHelper(std::move(aImage
));
75 return helper
->GetDataSurfaceSafe();
78 class EncodingCompleteEvent final
: public DiscardableRunnable
{
79 virtual ~EncodingCompleteEvent() = default;
82 explicit EncodingCompleteEvent(
83 EncodeCompleteCallback
* aEncodeCompleteCallback
)
84 : DiscardableRunnable("EncodingCompleteEvent"),
88 mEncodeCompleteCallback(aEncodeCompleteCallback
),
90 if (!NS_IsMainThread() && IsCurrentThreadRunningWorker()) {
91 mCreationEventTarget
= GetCurrentSerialEventTarget();
93 mCreationEventTarget
= GetMainThreadSerialEventTarget();
97 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. See
99 MOZ_CAN_RUN_SCRIPT_BOUNDARY
100 NS_IMETHOD
Run() override
{
103 // We want to null out mEncodeCompleteCallback no matter what.
104 RefPtr
<EncodeCompleteCallback
> callback(std::move(mEncodeCompleteCallback
));
106 RefPtr
<BlobImpl
> blobImpl
= new MemoryBlobImpl(mImgData
, mImgSize
, mType
);
107 rv
= callback
->ReceiveBlobImpl(blobImpl
.forget());
109 rv
= callback
->ReceiveBlobImpl(nullptr);
115 void SetMembers(void* aImgData
, uint64_t aImgSize
,
116 const nsAutoString
& aType
) {
122 void SetFailed() { mFailed
= true; }
124 nsIEventTarget
* GetCreationThreadEventTarget() {
125 return mCreationEventTarget
;
128 bool CanBeDeletedOnAnyThread() {
129 return !mEncodeCompleteCallback
||
130 mEncodeCompleteCallback
->CanBeDeletedOnAnyThread();
137 nsCOMPtr
<nsIEventTarget
> mCreationEventTarget
;
138 RefPtr
<EncodeCompleteCallback
> mEncodeCompleteCallback
;
142 class EncodingRunnable
: public Runnable
{
143 virtual ~EncodingRunnable() = default;
146 NS_INLINE_DECL_REFCOUNTING_INHERITED(EncodingRunnable
, Runnable
)
148 EncodingRunnable(const nsAString
& aType
, const nsAString
& aOptions
,
149 UniquePtr
<uint8_t[]> aImageBuffer
, layers::Image
* aImage
,
150 imgIEncoder
* aEncoder
,
151 EncodingCompleteEvent
* aEncodingCompleteEvent
,
152 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
153 bool aUsingCustomOptions
)
154 : Runnable("EncodingRunnable"),
157 mImageBuffer(std::move(aImageBuffer
)),
160 mEncodingCompleteEvent(aEncodingCompleteEvent
),
163 mUsePlaceholder(aUsePlaceholder
),
164 mUsingCustomOptions(aUsingCustomOptions
) {}
166 nsresult
ProcessImageData(uint64_t* aImgSize
, void** aImgData
) {
167 nsCOMPtr
<nsIInputStream
> stream
;
168 nsresult rv
= ImageEncoder::ExtractDataInternal(
169 mType
, mOptions
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
170 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
172 // If there are unrecognized custom parse options, we should fall back to
173 // the default values for the encoder without any options at all.
174 if (rv
== NS_ERROR_INVALID_ARG
&& mUsingCustomOptions
) {
175 rv
= ImageEncoder::ExtractDataInternal(
176 mType
, u
""_ns
, mImageBuffer
.get(), mFormat
, mSize
, mUsePlaceholder
,
177 mImage
, nullptr, nullptr, getter_AddRefs(stream
), mEncoder
);
179 NS_ENSURE_SUCCESS(rv
, rv
);
181 rv
= NS_ReadInputStreamToBuffer(stream
, aImgData
, -1, aImgSize
);
182 NS_ENSURE_SUCCESS(rv
, rv
);
187 NS_IMETHOD
Run() override
{
189 void* imgData
= nullptr;
191 nsresult rv
= ProcessImageData(&imgSize
, &imgData
);
193 mEncodingCompleteEvent
->SetFailed();
195 mEncodingCompleteEvent
->SetMembers(imgData
, imgSize
, mType
);
197 rv
= mEncodingCompleteEvent
->GetCreationThreadEventTarget()->Dispatch(
198 mEncodingCompleteEvent
, nsIThread::DISPATCH_NORMAL
);
200 if (!mEncodingCompleteEvent
->CanBeDeletedOnAnyThread()) {
201 // Better to leak than to crash.
202 Unused
<< mEncodingCompleteEvent
.forget();
212 nsAutoString mOptions
;
213 UniquePtr
<uint8_t[]> mImageBuffer
;
214 RefPtr
<layers::Image
> mImage
;
215 nsCOMPtr
<imgIEncoder
> mEncoder
;
216 RefPtr
<EncodingCompleteEvent
> mEncodingCompleteEvent
;
218 const nsIntSize mSize
;
219 bool mUsePlaceholder
;
220 bool mUsingCustomOptions
;
224 nsresult
ImageEncoder::ExtractData(nsAString
& aType
, const nsAString
& aOptions
,
225 const nsIntSize aSize
, bool aUsePlaceholder
,
226 nsICanvasRenderingContextInternal
* aContext
,
227 layers::CanvasRenderer
* aRenderer
,
228 nsIInputStream
** aStream
) {
229 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
231 return NS_IMAGELIB_ERROR_NO_ENCODER
;
234 return ExtractDataInternal(aType
, aOptions
, nullptr, 0, aSize
,
235 aUsePlaceholder
, nullptr, aContext
, aRenderer
,
240 nsresult
ImageEncoder::ExtractDataFromLayersImageAsync(
241 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
242 layers::Image
* aImage
, bool aUsePlaceholder
,
243 EncodeCompleteCallback
* aEncodeCallback
) {
244 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
246 return NS_IMAGELIB_ERROR_NO_ENCODER
;
249 RefPtr
<EncodingCompleteEvent
> completeEvent
=
250 new EncodingCompleteEvent(aEncodeCallback
);
252 nsIntSize
size(aImage
->GetSize().width
, aImage
->GetSize().height
);
253 nsCOMPtr
<nsIRunnable
> event
=
254 new EncodingRunnable(aType
, aOptions
, nullptr, aImage
, encoder
,
255 completeEvent
, imgIEncoder::INPUT_FORMAT_HOSTARGB
,
256 size
, aUsePlaceholder
, aUsingCustomOptions
);
257 return NS_DispatchBackgroundTask(event
.forget());
261 nsresult
ImageEncoder::ExtractDataAsync(
262 nsAString
& aType
, const nsAString
& aOptions
, bool aUsingCustomOptions
,
263 UniquePtr
<uint8_t[]> aImageBuffer
, int32_t aFormat
, const nsIntSize aSize
,
264 bool aUsePlaceholder
, EncodeCompleteCallback
* aEncodeCallback
) {
265 nsCOMPtr
<imgIEncoder
> encoder
= ImageEncoder::GetImageEncoder(aType
);
267 return NS_IMAGELIB_ERROR_NO_ENCODER
;
270 RefPtr
<EncodingCompleteEvent
> completeEvent
=
271 new EncodingCompleteEvent(aEncodeCallback
);
273 nsCOMPtr
<nsIRunnable
> event
= new EncodingRunnable(
274 aType
, aOptions
, std::move(aImageBuffer
), nullptr, encoder
, completeEvent
,
275 aFormat
, aSize
, aUsePlaceholder
, aUsingCustomOptions
);
276 return NS_DispatchBackgroundTask(event
.forget());
280 nsresult
ImageEncoder::GetInputStream(int32_t aWidth
, int32_t aHeight
,
281 uint8_t* aImageBuffer
, int32_t aFormat
,
282 imgIEncoder
* aEncoder
,
283 const nsAString
& aEncoderOptions
,
284 nsIInputStream
** aStream
) {
286 aEncoder
->InitFromData(aImageBuffer
, aWidth
* aHeight
* 4, aWidth
,
287 aHeight
, aWidth
* 4, aFormat
, aEncoderOptions
);
288 NS_ENSURE_SUCCESS(rv
, rv
);
290 nsCOMPtr
<imgIEncoder
> encoder(aEncoder
);
291 encoder
.forget(aStream
);
296 nsresult
ImageEncoder::ExtractDataInternal(
297 const nsAString
& aType
, const nsAString
& aOptions
, uint8_t* aImageBuffer
,
298 int32_t aFormat
, const nsIntSize aSize
, bool aUsePlaceholder
,
299 layers::Image
* aImage
, nsICanvasRenderingContextInternal
* aContext
,
300 layers::CanvasRenderer
* aRenderer
, nsIInputStream
** aStream
,
301 imgIEncoder
* aEncoder
) {
302 if (aSize
.IsEmpty()) {
303 return NS_ERROR_INVALID_ARG
;
306 nsCOMPtr
<nsIInputStream
> imgStream
;
310 if (aImageBuffer
&& !aUsePlaceholder
) {
311 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
312 return NS_ERROR_INVALID_ARG
;
315 rv
= ImageEncoder::GetInputStream(aSize
.width
, aSize
.height
, aImageBuffer
,
316 aFormat
, aEncoder
, aOptions
,
317 getter_AddRefs(imgStream
));
318 } else if (aContext
&& !aUsePlaceholder
) {
319 NS_ConvertUTF16toUTF8
encoderType(aType
);
320 rv
= aContext
->GetInputStream(encoderType
.get(), aOptions
,
321 getter_AddRefs(imgStream
));
322 } else if (aRenderer
&& !aUsePlaceholder
) {
323 MOZ_CRASH("unused?");
324 const NS_ConvertUTF16toUTF8
encoderType(aType
);
325 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
326 return NS_ERROR_INVALID_ARG
;
329 const auto snapshot
= aRenderer
->BorrowSnapshot();
331 return NS_ERROR_OUT_OF_MEMORY
;
334 const RefPtr
<DataSourceSurface
> data
= snapshot
->mSurf
->GetDataSurface();
336 return NS_ERROR_OUT_OF_MEMORY
;
340 DataSourceSurface::MappedSurface map
;
341 if (!data
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
342 return NS_ERROR_INVALID_ARG
;
344 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
345 aSize
.width
, aSize
.height
, aSize
.width
* 4,
346 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
349 } else if (aImage
&& !aUsePlaceholder
) {
350 // It is safe to convert PlanarYCbCr format from YUV to RGB off-main-thread.
351 // Other image formats could have problem to convert format off-main-thread.
352 // So here it uses a help function GetBRGADataSourceSurfaceSync() to convert
353 // format on main thread.
354 if (aImage
->GetFormat() == ImageFormat::PLANAR_YCBCR
) {
355 nsTArray
<uint8_t> data
;
356 layers::PlanarYCbCrImage
* ycbcrImage
=
357 static_cast<layers::PlanarYCbCrImage
*>(aImage
);
358 gfxImageFormat format
= SurfaceFormat::A8R8G8B8_UINT32
;
359 uint32_t stride
= GetAlignedStride
<16>(aSize
.width
, 4);
360 size_t length
= BufferSizeFromStrideAndHeight(stride
, aSize
.height
);
362 return NS_ERROR_INVALID_ARG
;
364 data
.SetCapacity(length
);
366 ConvertYCbCrToRGB(*ycbcrImage
->GetData(), format
, aSize
, data
.Elements(),
369 rv
= aEncoder
->InitFromData(data
.Elements(),
370 aSize
.width
* aSize
.height
* 4, aSize
.width
,
371 aSize
.height
, aSize
.width
* 4,
372 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
374 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
375 return NS_ERROR_INVALID_ARG
;
378 RefPtr
<gfx::DataSourceSurface
> dataSurface
;
379 RefPtr
<layers::Image
> image(aImage
);
380 dataSurface
= GetBRGADataSourceSurfaceSync(image
.forget());
382 DataSourceSurface::MappedSurface map
;
383 if (!dataSurface
->Map(gfx::DataSourceSurface::MapType::READ
, &map
)) {
384 return NS_ERROR_INVALID_ARG
;
386 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
387 aSize
.width
, aSize
.height
, aSize
.width
* 4,
388 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
389 dataSurface
->Unmap();
392 if (NS_SUCCEEDED(rv
)) {
393 imgStream
= aEncoder
;
396 if (BufferSizeFromDimensions(aSize
.width
, aSize
.height
, 4) == 0) {
397 return NS_ERROR_INVALID_ARG
;
400 // no context, so we have to encode an empty image
401 // note that if we didn't have a current context, the spec says we're
402 // supposed to just return transparent black pixels of the canvas
404 RefPtr
<DataSourceSurface
> emptyCanvas
=
405 Factory::CreateDataSourceSurfaceWithStride(
406 IntSize(aSize
.width
, aSize
.height
), SurfaceFormat::B8G8R8A8
,
407 4 * aSize
.width
, true);
408 if (NS_WARN_IF(!emptyCanvas
)) {
409 return NS_ERROR_INVALID_ARG
;
412 DataSourceSurface::MappedSurface map
;
413 if (!emptyCanvas
->Map(DataSourceSurface::MapType::WRITE
, &map
)) {
414 return NS_ERROR_INVALID_ARG
;
416 if (aUsePlaceholder
) {
417 auto size
= 4 * aSize
.width
* aSize
.height
;
418 auto* data
= map
.mData
;
419 GeneratePlaceholderCanvasData(size
, data
);
421 rv
= aEncoder
->InitFromData(map
.mData
, aSize
.width
* aSize
.height
* 4,
422 aSize
.width
, aSize
.height
, aSize
.width
* 4,
423 imgIEncoder::INPUT_FORMAT_HOSTARGB
, aOptions
);
424 emptyCanvas
->Unmap();
425 if (NS_SUCCEEDED(rv
)) {
426 imgStream
= aEncoder
;
429 NS_ENSURE_SUCCESS(rv
, rv
);
431 imgStream
.forget(aStream
);
436 already_AddRefed
<imgIEncoder
> ImageEncoder::GetImageEncoder(nsAString
& aType
) {
437 // Get an image encoder for the media type.
438 nsCString
encoderCID("@mozilla.org/image/encoder;2?type=");
439 NS_ConvertUTF16toUTF8
encoderType(aType
);
440 encoderCID
+= encoderType
;
441 nsCOMPtr
<imgIEncoder
> encoder
= do_CreateInstance(encoderCID
.get());
443 if (!encoder
&& aType
!= u
"image/png"_ns
) {
444 // Unable to create an encoder instance of the specified type. Falling back
446 aType
.AssignLiteral("image/png");
447 nsCString
PNGEncoderCID("@mozilla.org/image/encoder;2?type=image/png");
448 encoder
= do_CreateInstance(PNGEncoderCID
.get());
451 return encoder
.forget();
454 } // namespace mozilla::dom