dom/base/ImageEncoder.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "ImageEncoder.h"
   8 #include "mozilla/dom/CanvasRenderingContext2D.h"
   9 #include "mozilla/dom/GeneratePlaceholderCanvasData.h"
  10 #include "mozilla/dom/MemoryBlobImpl.h"
  11 #include "mozilla/dom/OffscreenCanvasDisplayHelper.h"
  12 #include "mozilla/dom/WorkerPrivate.h"
  13 #include "mozilla/gfx/2D.h"
  14 #include "mozilla/gfx/DataSurfaceHelpers.h"
  15 #include "mozilla/layers/CanvasRenderer.h"
  16 #include "mozilla/RefPtr.h"
  17 #include "mozilla/SyncRunnable.h"
  18 #include "mozilla/Unused.h"
  19 #include "gfxUtils.h"
  20 #include "nsComponentManagerUtils.h"
  21 #include "nsThreadUtils.h"
  22 #include "nsNetUtil.h"
  23 #include "nsXPCOMCIDInternal.h"
  24 #include "YCbCrUtils.h"
  25
  26 using namespace mozilla::gfx;
  27
  28 namespace mozilla::dom {
  29
  30 // This class should be placed inside GetBRGADataSourceSurfaceSync(). However,
  31 // due to B2G ICS uses old complier (C++98/03) which forbids local class as
  32 // template parameter, we need to move this class outside.
  33 class SurfaceHelper : public Runnable {
  34  public:
  35   explicit SurfaceHelper(already_AddRefed<layers::Image> aImage)
  36       : Runnable("SurfaceHelper"), mImage(aImage) {}
  37
  38   // It retrieves a SourceSurface reference and convert color format on main
  39   // thread and passes DataSourceSurface to caller thread.
  40   NS_IMETHOD Run() override {
  41     RefPtr<gfx::SourceSurface> surface = mImage->GetAsSourceSurface();
  42
  43     if (surface->GetFormat() == gfx::SurfaceFormat::B8G8R8A8) {
  44       mDataSourceSurface = surface->GetDataSurface();
  45     } else {
  46       mDataSourceSurface = gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(
  47           surface, gfx::SurfaceFormat::B8G8R8A8);
  48     }
  49
  50     // It guarantees the reference will be released on main thread.
  51     NS_ReleaseOnMainThread("SurfaceHelper::surface", surface.forget());
  52     return NS_OK;
  53   }
  54
  55   already_AddRefed<gfx::DataSourceSurface> GetDataSurfaceSafe() {
  56     nsCOMPtr<nsIEventTarget> mainTarget = GetMainThreadSerialEventTarget();
  57     MOZ_ASSERT(mainTarget);
  58     SyncRunnable::DispatchToThread(mainTarget, this, false);
  59
  60     return mDataSourceSurface.forget();
  61   }
  62
  63  private:
  64   RefPtr<layers::Image> mImage;
  65   RefPtr<gfx::DataSourceSurface> mDataSourceSurface;
  66 };
  67
  68 // This function returns a DataSourceSurface in B8G8R8A8 format.
  69 // It uses SourceSurface to do format convert. Because most SourceSurface in
  70 // image formats should be referenced or dereferenced on main thread, it uses a
  71 // sync class SurfaceHelper to retrieve SourceSurface and convert to B8G8R8A8 on
  72 // main thread.
  73 already_AddRefed<DataSourceSurface> GetBRGADataSourceSurfaceSync(
  74     already_AddRefed<layers::Image> aImage) {
  75   RefPtr<SurfaceHelper> helper = new SurfaceHelper(std::move(aImage));
  76   return helper->GetDataSurfaceSafe();
  77 }
  78
  79 class EncodingCompleteEvent final : public DiscardableRunnable {
  80   virtual ~EncodingCompleteEvent() = default;
  81
  82  public:
  83   explicit EncodingCompleteEvent(
  84       EncodeCompleteCallback* aEncodeCompleteCallback)
  85       : DiscardableRunnable("EncodingCompleteEvent"),
  86         mImgSize(0),
  87         mType(),
  88         mImgData(nullptr),
  89         mEncodeCompleteCallback(aEncodeCompleteCallback),
  90         mFailed(false) {
  91     if (!NS_IsMainThread() && IsCurrentThreadRunningWorker()) {
  92       mCreationEventTarget = GetCurrentSerialEventTarget();
  93     } else {
  94       mCreationEventTarget = GetMainThreadSerialEventTarget();
  95     }
  96   }
  97
  98   // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.  See
  99   // bug 1535398.
 100   MOZ_CAN_RUN_SCRIPT_BOUNDARY
 101   NS_IMETHOD Run() override {
 102     nsresult rv = NS_OK;
 103
 104     // We want to null out mEncodeCompleteCallback no matter what.
 105     RefPtr<EncodeCompleteCallback> callback(std::move(mEncodeCompleteCallback));
 106     if (!mFailed) {
 107       RefPtr<BlobImpl> blobImpl = new MemoryBlobImpl(mImgData, mImgSize, mType);
 108       rv = callback->ReceiveBlobImpl(blobImpl.forget());
 109     } else {
 110       rv = callback->ReceiveBlobImpl(nullptr);
 111     }
 112
 113     return rv;
 114   }
 115
 116   void SetMembers(void* aImgData, uint64_t aImgSize,
 117                   const nsAutoString& aType) {
 118     mImgData = aImgData;
 119     mImgSize = aImgSize;
 120     mType = aType;
 121   }
 122
 123   void SetFailed() { mFailed = true; }
 124
 125   nsIEventTarget* GetCreationThreadEventTarget() {
 126     return mCreationEventTarget;
 127   }
 128
 129   bool CanBeDeletedOnAnyThread() {
 130     return !mEncodeCompleteCallback ||
 131            mEncodeCompleteCallback->CanBeDeletedOnAnyThread();
 132   }
 133
 134  private:
 135   uint64_t mImgSize;
 136   nsAutoString mType;
 137   void* mImgData;
 138   nsCOMPtr<nsIEventTarget> mCreationEventTarget;
 139   RefPtr<EncodeCompleteCallback> mEncodeCompleteCallback;
 140   bool mFailed;
 141 };
 142
 143 class EncodingRunnable : public Runnable {
 144   virtual ~EncodingRunnable() = default;
 145
 146  public:
 147   NS_INLINE_DECL_REFCOUNTING_INHERITED(EncodingRunnable, Runnable)
 148
 149   EncodingRunnable(const nsAString& aType, const nsAString& aOptions,
 150                    UniquePtr<uint8_t[]> aImageBuffer, layers::Image* aImage,
 151                    imgIEncoder* aEncoder,
 152                    EncodingCompleteEvent* aEncodingCompleteEvent,
 153                    int32_t aFormat, const nsIntSize aSize, bool aUsePlaceholder,
 154                    bool aUsingCustomOptions)
 155       : Runnable("EncodingRunnable"),
 156         mType(aType),
 157         mOptions(aOptions),
 158         mImageBuffer(std::move(aImageBuffer)),
 159         mImage(aImage),
 160         mEncoder(aEncoder),
 161         mEncodingCompleteEvent(aEncodingCompleteEvent),
 162         mFormat(aFormat),
 163         mSize(aSize),
 164         mUsePlaceholder(aUsePlaceholder),
 165         mUsingCustomOptions(aUsingCustomOptions) {}
 166
 167   nsresult ProcessImageData(uint64_t* aImgSize, void** aImgData) {
 168     nsCOMPtr<nsIInputStream> stream;
 169     nsresult rv = ImageEncoder::ExtractDataInternal(
 170         mType, mOptions, mImageBuffer.get(), mFormat, mSize, mUsePlaceholder,
 171         mImage, nullptr, nullptr, getter_AddRefs(stream), mEncoder);
 172
 173     // If there are unrecognized custom parse options, we should fall back to
 174     // the default values for the encoder without any options at all.
 175     if (rv == NS_ERROR_INVALID_ARG && mUsingCustomOptions) {
 176       rv = ImageEncoder::ExtractDataInternal(
 177           mType, u""_ns, mImageBuffer.get(), mFormat, mSize, mUsePlaceholder,
 178           mImage, nullptr, nullptr, getter_AddRefs(stream), mEncoder);
 179     }
 180     NS_ENSURE_SUCCESS(rv, rv);
 181
 182     rv = NS_ReadInputStreamToBuffer(stream, aImgData, -1, aImgSize);
 183     NS_ENSURE_SUCCESS(rv, rv);
 184
 185     return rv;
 186   }
 187
 188   NS_IMETHOD Run() override {
 189     uint64_t imgSize;
 190     void* imgData = nullptr;
 191
 192     nsresult rv = ProcessImageData(&imgSize, &imgData);
 193     if (NS_FAILED(rv)) {
 194       mEncodingCompleteEvent->SetFailed();
 195     } else {
 196       mEncodingCompleteEvent->SetMembers(imgData, imgSize, mType);
 197     }
 198     rv = mEncodingCompleteEvent->GetCreationThreadEventTarget()->Dispatch(
 199         mEncodingCompleteEvent, nsIThread::DISPATCH_NORMAL);
 200     if (NS_FAILED(rv)) {
 201       if (!mEncodingCompleteEvent->CanBeDeletedOnAnyThread()) {
 202         // Better to leak than to crash.
 203         Unused << mEncodingCompleteEvent.forget();
 204       }
 205       return rv;
 206     }
 207
 208     return rv;
 209   }
 210
 211  private:
 212   nsAutoString mType;
 213   nsAutoString mOptions;
 214   UniquePtr<uint8_t[]> mImageBuffer;
 215   RefPtr<layers::Image> mImage;
 216   nsCOMPtr<imgIEncoder> mEncoder;
 217   RefPtr<EncodingCompleteEvent> mEncodingCompleteEvent;
 218   int32_t mFormat;
 219   const nsIntSize mSize;
 220   bool mUsePlaceholder;
 221   bool mUsingCustomOptions;
 222 };
 223
 224 /* static */
 225 nsresult ImageEncoder::ExtractData(
 226     nsAString& aType, const nsAString& aOptions, const nsIntSize aSize,
 227     bool aUsePlaceholder, nsICanvasRenderingContextInternal* aContext,
 228     OffscreenCanvasDisplayHelper* aOffscreenDisplay, nsIInputStream** aStream) {
 229   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 230   if (!encoder) {
 231     return NS_IMAGELIB_ERROR_NO_ENCODER;
 232   }
 233
 234   return ExtractDataInternal(aType, aOptions, nullptr, 0, aSize,
 235                              aUsePlaceholder, nullptr, aContext,
 236                              aOffscreenDisplay, aStream, encoder);
 237 }
 238
 239 /* static */
 240 nsresult ImageEncoder::ExtractDataFromLayersImageAsync(
 241     nsAString& aType, const nsAString& aOptions, bool aUsingCustomOptions,
 242     layers::Image* aImage, bool aUsePlaceholder,
 243     EncodeCompleteCallback* aEncodeCallback) {
 244   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 245   if (!encoder) {
 246     return NS_IMAGELIB_ERROR_NO_ENCODER;
 247   }
 248
 249   RefPtr<EncodingCompleteEvent> completeEvent =
 250       new EncodingCompleteEvent(aEncodeCallback);
 251
 252   nsIntSize size(aImage->GetSize().width, aImage->GetSize().height);
 253   nsCOMPtr<nsIRunnable> event =
 254       new EncodingRunnable(aType, aOptions, nullptr, aImage, encoder,
 255                            completeEvent, imgIEncoder::INPUT_FORMAT_HOSTARGB,
 256                            size, aUsePlaceholder, aUsingCustomOptions);
 257   return NS_DispatchBackgroundTask(event.forget());
 258 }
 259
 260 /* static */
 261 nsresult ImageEncoder::ExtractDataAsync(
 262     nsAString& aType, const nsAString& aOptions, bool aUsingCustomOptions,
 263     UniquePtr<uint8_t[]> aImageBuffer, int32_t aFormat, const nsIntSize aSize,
 264     bool aUsePlaceholder, EncodeCompleteCallback* aEncodeCallback) {
 265   nsCOMPtr<imgIEncoder> encoder = ImageEncoder::GetImageEncoder(aType);
 266   if (!encoder) {
 267     return NS_IMAGELIB_ERROR_NO_ENCODER;
 268   }
 269
 270   RefPtr<EncodingCompleteEvent> completeEvent =
 271       new EncodingCompleteEvent(aEncodeCallback);
 272
 273   nsCOMPtr<nsIRunnable> event = new EncodingRunnable(
 274       aType, aOptions, std::move(aImageBuffer), nullptr, encoder, completeEvent,
 275       aFormat, aSize, aUsePlaceholder, aUsingCustomOptions);
 276   return NS_DispatchBackgroundTask(event.forget());
 277 }
 278
 279 /*static*/
 280 nsresult ImageEncoder::GetInputStream(int32_t aWidth, int32_t aHeight,
 281                                       uint8_t* aImageBuffer, int32_t aFormat,
 282                                       imgIEncoder* aEncoder,
 283                                       const nsAString& aEncoderOptions,
 284                                       nsIInputStream** aStream) {
 285   nsresult rv =
 286       aEncoder->InitFromData(aImageBuffer, aWidth * aHeight * 4, aWidth,
 287                              aHeight, aWidth * 4, aFormat, aEncoderOptions);
 288   NS_ENSURE_SUCCESS(rv, rv);
 289
 290   nsCOMPtr<imgIEncoder> encoder(aEncoder);
 291   encoder.forget(aStream);
 292   return NS_OK;
 293 }
 294
 295 /* static */
 296 nsresult ImageEncoder::ExtractDataInternal(
 297     const nsAString& aType, const nsAString& aOptions, uint8_t* aImageBuffer,
 298     int32_t aFormat, const nsIntSize aSize, bool aUsePlaceholder,
 299     layers::Image* aImage, nsICanvasRenderingContextInternal* aContext,
 300     OffscreenCanvasDisplayHelper* aOffscreenDisplay, nsIInputStream** aStream,
 301     imgIEncoder* aEncoder) {
 302   if (aSize.IsEmpty()) {
 303     return NS_ERROR_INVALID_ARG;
 304   }
 305
 306   nsCOMPtr<nsIInputStream> imgStream;
 307
 308   // get image bytes
 309   nsresult rv;
 310   if (aImageBuffer && !aUsePlaceholder) {
 311     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 312       return NS_ERROR_INVALID_ARG;
 313     }
 314
 315     rv = ImageEncoder::GetInputStream(aSize.width, aSize.height, aImageBuffer,
 316                                       aFormat, aEncoder, aOptions,
 317                                       getter_AddRefs(imgStream));
 318   } else if (aContext && !aUsePlaceholder) {
 319     NS_ConvertUTF16toUTF8 encoderType(aType);
 320     rv = aContext->GetInputStream(encoderType.get(), aOptions,
 321                                   getter_AddRefs(imgStream));
 322   } else if (aOffscreenDisplay && !aUsePlaceholder) {
 323     const NS_ConvertUTF16toUTF8 encoderType(aType);
 324     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 325       return NS_ERROR_INVALID_ARG;
 326     }
 327
 328     const RefPtr<SourceSurface> snapshot =
 329         aOffscreenDisplay->GetSurfaceSnapshot();
 330     if (!snapshot) {
 331       return NS_ERROR_OUT_OF_MEMORY;
 332     }
 333
 334     const RefPtr<DataSourceSurface> data = snapshot->GetDataSurface();
 335     if (!data) {
 336       return NS_ERROR_OUT_OF_MEMORY;
 337     }
 338
 339     {
 340       DataSourceSurface::MappedSurface map;
 341       if (!data->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
 342         return NS_ERROR_INVALID_ARG;
 343       }
 344       rv = aEncoder->InitFromData(map.mData, aSize.width * aSize.height * 4,
 345                                   aSize.width, aSize.height, aSize.width * 4,
 346                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 347       data->Unmap();
 348     }
 349     if (NS_SUCCEEDED(rv)) {
 350       imgStream = aEncoder;
 351     }
 352   } else if (aImage && !aUsePlaceholder) {
 353     // It is safe to convert PlanarYCbCr format from YUV to RGB off-main-thread.
 354     // Other image formats could have problem to convert format off-main-thread.
 355     // So here it uses a help function GetBRGADataSourceSurfaceSync() to convert
 356     // format on main thread.
 357     if (aImage->GetFormat() == ImageFormat::PLANAR_YCBCR) {
 358       nsTArray<uint8_t> data;
 359       layers::PlanarYCbCrImage* ycbcrImage =
 360           static_cast<layers::PlanarYCbCrImage*>(aImage);
 361       gfxImageFormat format = SurfaceFormat::A8R8G8B8_UINT32;
 362       uint32_t stride = GetAlignedStride<16>(aSize.width, 4);
 363       size_t length = BufferSizeFromStrideAndHeight(stride, aSize.height);
 364       if (length == 0) {
 365         return NS_ERROR_INVALID_ARG;
 366       }
 367       data.SetCapacity(length);
 368
 369       ConvertYCbCrToRGB(*ycbcrImage->GetData(), format, aSize, data.Elements(),
 370                         stride);
 371
 372       rv = aEncoder->InitFromData(data.Elements(),
 373                                   aSize.width * aSize.height * 4, aSize.width,
 374                                   aSize.height, aSize.width * 4,
 375                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 376     } else {
 377       if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 378         return NS_ERROR_INVALID_ARG;
 379       }
 380
 381       RefPtr<gfx::DataSourceSurface> dataSurface;
 382       RefPtr<layers::Image> image(aImage);
 383       dataSurface = GetBRGADataSourceSurfaceSync(image.forget());
 384
 385       DataSourceSurface::MappedSurface map;
 386       if (!dataSurface->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
 387         return NS_ERROR_INVALID_ARG;
 388       }
 389       rv = aEncoder->InitFromData(map.mData, aSize.width * aSize.height * 4,
 390                                   aSize.width, aSize.height, aSize.width * 4,
 391                                   imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 392       dataSurface->Unmap();
 393     }
 394
 395     if (NS_SUCCEEDED(rv)) {
 396       imgStream = aEncoder;
 397     }
 398   } else {
 399     if (BufferSizeFromDimensions(aSize.width, aSize.height, 4) == 0) {
 400       return NS_ERROR_INVALID_ARG;
 401     }
 402
 403     // no context, so we have to encode an empty image
 404     // note that if we didn't have a current context, the spec says we're
 405     // supposed to just return transparent black pixels of the canvas
 406     // dimensions.
 407     RefPtr<DataSourceSurface> emptyCanvas =
 408         Factory::CreateDataSourceSurfaceWithStride(
 409             IntSize(aSize.width, aSize.height), SurfaceFormat::B8G8R8A8,
 410             4 * aSize.width, true);
 411     if (NS_WARN_IF(!emptyCanvas)) {
 412       return NS_ERROR_INVALID_ARG;
 413     }
 414
 415     DataSourceSurface::MappedSurface map;
 416     if (!emptyCanvas->Map(DataSourceSurface::MapType::WRITE, &map)) {
 417       return NS_ERROR_INVALID_ARG;
 418     }
 419     if (aUsePlaceholder) {
 420       auto size = 4 * aSize.width * aSize.height;
 421       auto* data = map.mData;
 422       GeneratePlaceholderCanvasData(size, data);
 423     }
 424     rv = aEncoder->InitFromData(map.mData, aSize.width * aSize.height * 4,
 425                                 aSize.width, aSize.height, aSize.width * 4,
 426                                 imgIEncoder::INPUT_FORMAT_HOSTARGB, aOptions);
 427     emptyCanvas->Unmap();
 428     if (NS_SUCCEEDED(rv)) {
 429       imgStream = aEncoder;
 430     }
 431   }
 432   NS_ENSURE_SUCCESS(rv, rv);
 433
 434   imgStream.forget(aStream);
 435   return rv;
 436 }
 437
 438 /* static */
 439 already_AddRefed<imgIEncoder> ImageEncoder::GetImageEncoder(nsAString& aType) {
 440   // Get an image encoder for the media type.
 441   nsCString encoderCID("@mozilla.org/image/encoder;2?type=");
 442   NS_ConvertUTF16toUTF8 encoderType(aType);
 443   encoderCID += encoderType;
 444   nsCOMPtr<imgIEncoder> encoder = do_CreateInstance(encoderCID.get());
 445
 446   if (!encoder && aType != u"image/png"_ns) {
 447     // Unable to create an encoder instance of the specified type. Falling back
 448     // to PNG.
 449     aType.AssignLiteral("image/png");
 450     nsCString PNGEncoderCID("@mozilla.org/image/encoder;2?type=image/png");
 451     encoder = do_CreateInstance(PNGEncoderCID.get());
 452   }
 453
 454   return encoder.forget();
 455 }
 456
 457 }  // namespace mozilla::dom