dom/base/ImageEncoder.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "ImageEncoder.h"
   8 #include "mozilla/dom/CanvasRenderingContext2D.h"
   9 #include "mozilla/dom/GeneratePlaceholderCanvasData.h"
  10 #include "mozilla/dom/MemoryBlobImpl.h"
  11 #include "mozilla/dom/OffscreenCanvasDisplayHelper.h"
  12 #include "mozilla/dom/WorkerPrivate.h"
  13 #include "mozilla/gfx/2D.h"
  14 #include "mozilla/gfx/DataSurfaceHelpers.h"
  15 #include "mozilla/layers/CanvasRenderer.h"
  16 #include "mozilla/RefPtr.h"
  17 #include "mozilla/SyncRunnable.h"
  18 #include "mozilla/Unused.h"
  19 #include "gfxUtils.h"
  20 #include "nsComponentManagerUtils.h"
  21 #include "nsThreadUtils.h"
  22 #include "nsNetUtil.h"
  23 #include "nsXPCOMCIDInternal.h"
  24 #include "YCbCrUtils.h"
  25
  26 using namespace mozilla::gfx;
  27
  28 namespace mozilla::dom {
  29
  30 // This class should be placed inside GetBRGADataSourceSurfaceSync(). However,
  31 // due to B2G ICS uses old complier (C++98/03) which forbids local class as
  32 // template parameter, we need to move this class outside.
  33 class SurfaceHelper : public Runnable {
  34  public:
  35   explicit SurfaceHelper(already_AddRefed<layers::Image> aImage)
  36       : Runnable("SurfaceHelper"), mImage(aImage) {}
  37
  38   // It retrieves a SourceSurface reference and convert color format on main
  39   // thread and passes DataSourceSurface to caller thread.
  40   NS_IMETHOD Run() override {
  41     RefPtr<gfx::SourceSurface> surface = mImage->GetAsSourceSurface();
  42
  43     if (surface->GetFormat() == gfx::SurfaceFormat::B8G8R8A8) {
  44       mDataSourceSurface = surface->GetDataSurface();
  45     } else {
  46       mDataSourceSurface = gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(
  47           surface, gfx::SurfaceFormat::B8G8R8A8);
  48     }
  49
  50     // It guarantees the reference will be released on main thread.
  51     NS_ReleaseOnMainThread("SurfaceHelper::surface", surface.forget());
  52     return NS_OK;
  53   }
  54
  55   already_AddRefed<gfx::DataSourceSurface> GetDataSurfaceSafe() {
  56     nsCOMPtr<nsIEventTarget> mainTarget = GetMainThreadSerialEventTarget();
  57     MOZ_ASSERT(mainTarget);
  58     SyncRunnable::DispatchToThread(mainTarget, this, false);
  59
  60     return mDataSourceSurface.forget();
  61   }
  62
  63  private:
  64   RefPtr<layers::Image> mImage;
  65   RefPtr<gfx::DataSourceSurface> mDataSourceSurface;
  66 };
  67
  68 // This function returns a DataSourceSurface in B8G8R8A8 format.
  69 // It uses SourceSurface to do format convert. Because most SourceSurface in
  70 // image formats should be referenced or dereferenced on main thread, it uses a
  71 // sync class SurfaceHelper to retrieve SourceSurface and convert to B8G8R8A8 on
  72 // main thread.
  73 already_AddRefed<DataSourceSurface> GetBRGADataSourceSurfaceSync(
  74     already_AddRefed<layers::Image> aImage) {
  75   RefPtr<SurfaceHelper> helper = new SurfaceHelper(std::move(aImage));
  76   return helper->GetDataSurfaceSafe();
  77 }
  78
  79 class EncodingCompleteEvent final : public DiscardableRunnable {
  80   virtual ~EncodingCompleteEvent() = default;
  81
  82  public:
  83   explicit EncodingCompleteEvent(
  84       EncodeCompleteCallback* aEncodeCompleteCallback)
  85       : DiscardableRunnable("EncodingCompleteEvent"),
  86         mImgSize(0),
  87         mImgData(nullptr),
  88         mEncodeCompleteCallback(aEncodeCompleteCallback),
  89         mFailed(false) {
  90     if (!NS_IsMainThread() && IsCurrentThreadRunningWorker()) {
  91       mCreationEventTarget = GetCurrentSerialEventTarget();
  92     } else {
  93       mCreationEventTarget = GetMainThreadSerialEventTarget();
  94     }
  95   }
  96
  97   // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.  See
  98   // bug 1535398.
  99   MOZ_CAN_RUN_SCRIPT_BOUNDARY
 100   NS_IMETHOD Run() override {
 101     nsresult rv = NS_OK;
 102
 103     // We want to null out mEncodeCompleteCallback no matter what.
 104     RefPtr<EncodeCompleteCallback> callback(std::move(mEncodeCompleteCallback));
 105     if (!mFailed) {
 106       RefPtr<BlobImpl> blobImpl = new MemoryBlobImpl(mImgData, mImgSize, mType);
 107       rv = callback->ReceiveBlobImpl(blobImpl.forget());
 108     } else {
 109       rv = callback->ReceiveBlobImpl(nullptr);
 110     }
 111
 112     return rv;
 113   }
 114
 115   void SetMembers(void* aImgData, uint64_t aImgSize,
 116                   const nsAutoString& aType) {
 117     mImgData = aImgData;
 118     mImgSize = aImgSize;
 119     mType = aType;
 120   }
 121
 122   void SetFailed() { mFailed = true; }
 123
 124   nsIEventTarget* GetCreationThreadEventTarget() {
 125     return mCreationEventTarget;
 126   }
 127
 128   bool CanBeDeletedOnAnyThread() {
 129     return !mEncodeCompleteCallback ||
 130            mEncodeCompleteCallback->CanBeDeletedOnAnyThread();
 131   }
 132
 133  private:
 134   uint64_t mImgSize;
 135   nsAutoString mType;
 136   void* mImgData;
 137   nsCOMPtr<nsIEventTarget> mCreationEventTarget;
 138   RefPtr<EncodeCompleteCallback> mEncodeCompleteCallback;
 139   bool mFailed;
 140 };
 141
 142 class EncodingRunnable : public Runnable {
 143   virtual ~EncodingRunnable() = default;
 144
 145  public:
 146   NS_INLINE_DECL_REFCOUNTING_INHERITED(EncodingRunnable, Runnable)
 147
 148   EncodingRunnable(const nsAString& aType, const nsAString& aOptions,
 149                    UniquePtr<uint8_t[]> aImageBuffer, layers::Image* aImage,
 150                    imgIEncoder* aEncoder,
 151                    EncodingCompleteEvent* aEncodingCompleteEvent,
 152                    int32_t aFormat, const nsIntSize aSize, bool aUsePlaceholder,
 153                    bool aUsingCustomOptions)
 154       : Runnable("EncodingRunnable"),
 155         mType(aType),
 156         mOptions(aOptions),
 157         mImageBuffer(std::move(aImageBuffer)),
 158         mImage(aImage),
 159         mEncoder(aEncoder),
 160         mEncodingCompleteEvent(aEncodingCompleteEvent),
 161         mFormat(aFormat),
 162         mSize(aSize),
 163         mUsePlaceholder(aUsePlaceholder),
 164         mUsingCustomOptions(aUsingCustomOptions) {}
 165
 166   nsresult ProcessImageData(uint64_t* aImgSize, void** aImgData) {
 167     nsCOMPtr<nsIInputStream> stream;
 168     nsresult rv = ImageEncoder::ExtractDataInternal(
 169         mType, mOptions, mImageBuffer.get(), mFormat, mSize, mUsePlaceholder,
 170         mImage, nullptr, nullptr, getter_AddRefs(stream), mEncoder);
 171
 172     // If there are unrecognized custom parse options, we should fall back to
 173     // the default values for the encoder without any options at all.
 174     if (rv == NS_ERROR_INVALID_ARG && mUsingCustomOptions) {
 175       rv = ImageEncoder::ExtractDataInternal(
 176           mType, u""_ns, mImageBuffer.get(), mFormat, mSize, mUsePlaceholder,
 177           mImage, nullptr, nullptr, getter_AddRefs(stream), mEncoder);
 178     }
 179     NS_ENSURE_SUCCESS(rv, rv);
 180
 181     rv = NS_ReadInputStreamToBuffer(stream, aImgData, -1, aImgSize);
 182     NS_ENSURE_SUCCESS(rv, rv);
 183
 184     return rv;
 185   }
 186
 187   NS_IMETHOD Run() override {
 188     uint64_t imgSize;
 189     void* imgData = nullptr;
 190
 191     nsresult rv = ProcessImageData(&imgSize, &imgData);
 192     if (NS_FAILED(rv)) {
 193       mEncodingCompleteEvent->SetFailed();
 194     } else {
 195       mEncodingCompleteEvent->SetMembers(imgData, imgSize, mType);
 196     }
 197     rv = mEncodingCompleteEvent->GetCreationThreadEventTarget()->Dispatch(
 198         mEncodingCompleteEvent, nsIThread::DISPATCH_NORMAL);
 199     if (NS_FAILED(rv)) {
 200       if (!mEncodingCompleteEvent->CanBeDeletedOnAnyThread()) {
 201         // Better to leak than to crash.
 202         Unused << mEncodingCompleteEvent.forget();
 203       }
 204       return rv;
 205     }
 206
 207     return rv;
 208   }
 209
 210  private:
 211   nsAutoString mType;
 212   nsAutoString mOptions;
 213   UniquePtr<uint8_t[]> mImageBuffer;
 214   RefPtr<layers::Image> mImage;
 215   nsCOMPtr<imgIEncoder> mEncoder;
 216   RefPtr<EncodingCompleteEvent> mEncodingCompleteEvent;
 217   int32_t mFormat;
 218   const nsIntSize mSize;
 219   bool mUsePlaceholder;
 220   bool mUsingCustomOptions;
 221 };
 222
 223 /* static */
 224 nsresult ImageEncoder::ExtractData(
 225     nsAString& aType, const nsAString& aOptions, const nsIntSize aSize,
 226     bool aUsePlaceholder, nsICanvasRenderingContextInternal* aContext,
 227     OffscreenCanvasDisplayHelper* aOffscreenDisplay, nsIInputStream** aStream) {
 228   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 229   if (!encoder) {
 230     return NS_IMAGELIB_ERROR_NO_ENCODER;
 231   }
 232
 233   return ExtractDataInternal(aType, aOptions, nullptr, 0, aSize,
 234                              aUsePlaceholder, nullptr, aContext,
 235                              aOffscreenDisplay, aStream, encoder);
 236 }
 237
 238 /* static */
 239 nsresult ImageEncoder::ExtractDataFromLayersImageAsync(
 240     nsAString& aType, const nsAString& aOptions, bool aUsingCustomOptions,
 241     layers::Image* aImage, bool aUsePlaceholder,
 242     EncodeCompleteCallback* aEncodeCallback) {
 243   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 244   if (!encoder) {
 245     return NS_IMAGELIB_ERROR_NO_ENCODER;
 246   }
 247
 248   RefPtr<EncodingCompleteEvent> completeEvent =
 249       new EncodingCompleteEvent(aEncodeCallback);
 250
 251   nsIntSize size(aImage->GetSize().width, aImage->GetSize().height);
 252   nsCOMPtr<nsIRunnable> event =
 253       new EncodingRunnable(aType, aOptions, nullptr, aImage, encoder,
 254                            completeEvent, imgIEncoder::INPUT_FORMAT_HOSTARGB,
 255                            size, aUsePlaceholder, aUsingCustomOptions);
 256   return NS_DispatchBackgroundTask(event.forget());
 257 }
 258
 259 /* static */
 260 nsresult ImageEncoder::ExtractDataAsync(
 261     nsAString& aType, const nsAString& aOptions, bool aUsingCustomOptions,
 262     UniquePtr<uint8_t[]> aImageBuffer, int32_t aFormat, const nsIntSize aSize,
 263     bool aUsePlaceholder, EncodeCompleteCallback* aEncodeCallback) {
 264   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 265   if (!encoder) {
 266     return NS_IMAGELIB_ERROR_NO_ENCODER;
 267   }
 268
 269   RefPtr<EncodingCompleteEvent> completeEvent =
 270       new EncodingCompleteEvent(aEncodeCallback);
 271
 272   nsCOMPtr<nsIRunnable> event = new EncodingRunnable(
 273       aType, aOptions, std::move(aImageBuffer), nullptr, encoder, completeEvent,
 274       aFormat, aSize, aUsePlaceholder, aUsingCustomOptions);
 275   return NS_DispatchBackgroundTask(event.forget());
 276 }
 277
 278 /*static*/
 279 nsresult ImageEncoder::GetInputStream(int32_t aWidth, int32_t aHeight,
 280                                       uint8_t* aImageBuffer, int32_t aFormat,
 281                                       imgIEncoder* aEncoder,
 282                                       const nsAString& aEncoderOptions,
 283                                       nsIInputStream** aStream) {
 284   nsresult rv =
 285       aEncoder->InitFromData(aImageBuffer, aWidth * aHeight * 4, aWidth,
 286                              aHeight, aWidth * 4, aFormat, aEncoderOptions);
 287   NS_ENSURE_SUCCESS(rv, rv);
 288
 289   nsCOMPtr<imgIEncoder> encoder(aEncoder);
 290   encoder.forget(aStream);
 291   return NS_OK;
 292 }
 293
 294 /* static */
 295 nsresult ImageEncoder::ExtractDataInternal(
 296     const nsAString& aType, const nsAString& aOptions, uint8_t* aImageBuffer,
 297     int32_t aFormat, const nsIntSize aSize, bool aUsePlaceholder,
 298     layers::Image* aImage, nsICanvasRenderingContextInternal* aContext,
 299     OffscreenCanvasDisplayHelper* aOffscreenDisplay, nsIInputStream** aStream,
 300     imgIEncoder* aEncoder) {
 301   if (aSize.IsEmpty()) {
 302     return NS_ERROR_INVALID_ARG;
 303   }
 304
 305   nsCOMPtr<nsIInputStream> imgStream;
 306
 307   // get image bytes
 308   nsresult rv;
 309   if (aImageBuffer && !aUsePlaceholder) {
 310     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 311       return NS_ERROR_INVALID_ARG;
 312     }
 313
 314     rv = ImageEncoder::GetInputStream(aSize.width, aSize.height, aImageBuffer,
 315                                       aFormat, aEncoder, aOptions,
 316                                       getter_AddRefs(imgStream));
 317   } else if (aContext && !aUsePlaceholder) {
 318     NS_ConvertUTF16toUTF8 encoderType(aType);
 319     rv = aContext->GetInputStream(encoderType.get(), aOptions,
 320                                   getter_AddRefs(imgStream));
 321   } else if (aOffscreenDisplay && !aUsePlaceholder) {
 322     const NS_ConvertUTF16toUTF8 encoderType(aType);
 323     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 324       return NS_ERROR_INVALID_ARG;
 325     }
 326
 327     const RefPtr<SourceSurface> snapshot =
 328         aOffscreenDisplay->GetSurfaceSnapshot();
 329     if (!snapshot) {
 330       return NS_ERROR_OUT_OF_MEMORY;
 331     }
 332
 333     const RefPtr<DataSourceSurface> data = snapshot->GetDataSurface();
 334     if (!data) {
 335       return NS_ERROR_OUT_OF_MEMORY;
 336     }
 337
 338     {
 339       DataSourceSurface::MappedSurface map;
 340       if (!data->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
 341         return NS_ERROR_INVALID_ARG;
 342       }
 343       auto size = data->GetSize();
 344       rv = aEncoder->InitFromData(map.mData, size.width * size.height * 4,
 345                                   size.width, size.height, size.width * 4,
 346                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 347       data->Unmap();
 348     }
 349     if (NS_SUCCEEDED(rv)) {
 350       imgStream = aEncoder;
 351     }
 352   } else if (aImage && !aUsePlaceholder) {
 353     // It is safe to convert PlanarYCbCr format from YUV to RGB off-main-thread.
 354     // Other image formats could have problem to convert format off-main-thread.
 355     // So here it uses a help function GetBRGADataSourceSurfaceSync() to convert
 356     // format on main thread.
 357     if (aImage->GetFormat() == ImageFormat::PLANAR_YCBCR) {
 358       nsTArray<uint8_t> data;
 359       layers::PlanarYCbCrImage* ycbcrImage =
 360           static_cast<layers::PlanarYCbCrImage*>(aImage);
 361       gfxImageFormat format = SurfaceFormat::A8R8G8B8_UINT32;
 362       uint32_t stride = GetAlignedStride<16>(aSize.width, 4);
 363       size_t length = BufferSizeFromStrideAndHeight(stride, aSize.height);
 364       if (length == 0) {
 365         return NS_ERROR_INVALID_ARG;
 366       }
 367       data.SetCapacity(length);
 368
 369       ConvertYCbCrToRGB(*ycbcrImage->GetData(), format, aSize, data.Elements(),
 370                         stride);
 371
 372       rv = aEncoder->InitFromData(data.Elements(),
 373                                   aSize.width * aSize.height * 4, aSize.width,
 374                                   aSize.height, aSize.width * 4,
 375                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 376     } else {
 377       if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 378         return NS_ERROR_INVALID_ARG;
 379       }
 380
 381       RefPtr<gfx::DataSourceSurface> dataSurface;
 382       RefPtr<layers::Image> image(aImage);
 383       dataSurface = GetBRGADataSourceSurfaceSync(image.forget());
 384
 385       DataSourceSurface::MappedSurface map;
 386       if (!dataSurface->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
 387         return NS_ERROR_INVALID_ARG;
 388       }
 389       auto size = dataSurface->GetSize();
 390       rv = aEncoder->InitFromData(map.mData, size.width * size.height * 4,
 391                                   size.width, size.height, size.width * 4,
 392                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 393       dataSurface->Unmap();
 394     }
 395
 396     if (NS_SUCCEEDED(rv)) {
 397       imgStream = aEncoder;
 398     }
 399   } else {
 400     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 401       return NS_ERROR_INVALID_ARG;
 402     }
 403
 404     // no context, so we have to encode an empty image
 405     // note that if we didn't have a current context, the spec says we're
 406     // supposed to just return transparent black pixels of the canvas
 407     // dimensions.
 408     RefPtr<DataSourceSurface> emptyCanvas =
 409         Factory::CreateDataSourceSurfaceWithStride(
 410             IntSize(aSize.width, aSize.height), SurfaceFormat::B8G8R8A8,
 411             4 * aSize.width, true);
 412     if (NS_WARN_IF(!emptyCanvas)) {
 413       return NS_ERROR_INVALID_ARG;
 414     }
 415
 416     DataSourceSurface::MappedSurface map;
 417     if (!emptyCanvas->Map(DataSourceSurface::MapType::WRITE, &map)) {
 418       return NS_ERROR_INVALID_ARG;
 419     }
 420     if (aUsePlaceholder) {
 421       auto size = 4 * aSize.width * aSize.height;
 422       auto* data = map.mData;
 423       GeneratePlaceholderCanvasData(size, data);
 424     }
 425     rv = aEncoder->InitFromData(map.mData, aSize.width * aSize.height * 4,
 426                                 aSize.width, aSize.height, aSize.width * 4,
 427                                 imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 428     emptyCanvas->Unmap();
 429     if (NS_SUCCEEDED(rv)) {
 430       imgStream = aEncoder;
 431     }
 432   }
 433   NS_ENSURE_SUCCESS(rv, rv);
 434
 435   imgStream.forget(aStream);
 436   return rv;
 437 }
 438
 439 /* static */
 440 already_AddRefed<imgIEncoder> ImageEncoder::GetImageEncoder(nsAString& aType) {
 441   // Get an image encoder for the media type.
 442   nsCString encoderCID("@mozilla.org/image/encoder;2?type=");
 443   NS_ConvertUTF16toUTF8 encoderType(aType);
 444   encoderCID += encoderType;
 445   nsCOMPtr<imgIEncoder> encoder = do_CreateInstance(encoderCID.get());
 446
 447   if (!encoder && aType != u"image/png"_ns) {
 448     // Unable to create an encoder instance of the specified type. Falling back
 449     // to PNG.
 450     aType.AssignLiteral("image/png");
 451     nsCString PNGEncoderCID("@mozilla.org/image/encoder;2?type=image/png");
 452     encoder = do_CreateInstance(PNGEncoderCID.get());
 453   }
 454
 455   return encoder.forget();
 456 }
 457
 458 }  // namespace mozilla::dom