dom/canvas/OffscreenCanvasDisplayHelper.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "OffscreenCanvasDisplayHelper.h"
   8 #include "mozilla/dom/Document.h"
   9 #include "mozilla/dom/WorkerRef.h"
  10 #include "mozilla/dom/WorkerRunnable.h"
  11 #include "mozilla/gfx/2D.h"
  12 #include "mozilla/gfx/CanvasManagerChild.h"
  13 #include "mozilla/gfx/DataSurfaceHelpers.h"
  14 #include "mozilla/gfx/Swizzle.h"
  15 #include "mozilla/layers/ImageBridgeChild.h"
  16 #include "mozilla/layers/PersistentBufferProvider.h"
  17 #include "mozilla/layers/TextureClientSharedSurface.h"
  18 #include "mozilla/layers/TextureWrapperImage.h"
  19 #include "mozilla/StaticPrefs_gfx.h"
  20 #include "mozilla/SVGObserverUtils.h"
  21 #include "nsICanvasRenderingContextInternal.h"
  22 #include "nsRFPService.h"
  23
  24 namespace mozilla::dom {
  25
  26 OffscreenCanvasDisplayHelper::OffscreenCanvasDisplayHelper(
  27     HTMLCanvasElement* aCanvasElement, uint32_t aWidth, uint32_t aHeight)
  28     : mMutex("mozilla::dom::OffscreenCanvasDisplayHelper"),
  29       mCanvasElement(aCanvasElement),
  30       mImageProducerID(layers::ImageContainer::AllocateProducerID()) {
  31   mData.mSize.width = aWidth;
  32   mData.mSize.height = aHeight;
  33 }
  34
  35 OffscreenCanvasDisplayHelper::~OffscreenCanvasDisplayHelper() = default;
  36
  37 void OffscreenCanvasDisplayHelper::DestroyElement() {
  38   MOZ_ASSERT(NS_IsMainThread());
  39
  40   MutexAutoLock lock(mMutex);
  41   if (mImageContainer) {
  42     mImageContainer->ClearAllImages();
  43     mImageContainer = nullptr;
  44   }
  45   mFrontBufferSurface = nullptr;
  46   mCanvasElement = nullptr;
  47 }
  48
  49 void OffscreenCanvasDisplayHelper::DestroyCanvas() {
  50   if (auto* cm = gfx::CanvasManagerChild::Get()) {
  51     cm->EndCanvasTransaction();
  52   }
  53
  54   MutexAutoLock lock(mMutex);
  55   if (mImageContainer) {
  56     mImageContainer->ClearAllImages();
  57     mImageContainer = nullptr;
  58   }
  59   mFrontBufferSurface = nullptr;
  60   mOffscreenCanvas = nullptr;
  61   mWorkerRef = nullptr;
  62 }
  63
  64 CanvasContextType OffscreenCanvasDisplayHelper::GetContextType() const {
  65   MutexAutoLock lock(mMutex);
  66   return mType;
  67 }
  68
  69 RefPtr<layers::ImageContainer> OffscreenCanvasDisplayHelper::GetImageContainer()
  70     const {
  71   MutexAutoLock lock(mMutex);
  72   return mImageContainer;
  73 }
  74
  75 void OffscreenCanvasDisplayHelper::UpdateContext(
  76     OffscreenCanvas* aOffscreenCanvas, RefPtr<ThreadSafeWorkerRef>&& aWorkerRef,
  77     CanvasContextType aType, const Maybe<int32_t>& aChildId) {
  78   RefPtr<layers::ImageContainer> imageContainer =
  79       MakeRefPtr<layers::ImageContainer>(layers::ImageContainer::ASYNCHRONOUS);
  80
  81   MutexAutoLock lock(mMutex);
  82
  83   mOffscreenCanvas = aOffscreenCanvas;
  84   mWorkerRef = std::move(aWorkerRef);
  85   mType = aType;
  86   mContextChildId = aChildId;
  87   mImageContainer = std::move(imageContainer);
  88
  89   if (aChildId) {
  90     mContextManagerId = Some(gfx::CanvasManagerChild::Get()->Id());
  91   } else {
  92     mContextManagerId.reset();
  93   }
  94
  95   MaybeQueueInvalidateElement();
  96 }
  97
  98 void OffscreenCanvasDisplayHelper::FlushForDisplay() {
  99   MOZ_ASSERT(NS_IsMainThread());
 100
 101   MutexAutoLock lock(mMutex);
 102
 103   // Without an OffscreenCanvas object bound to us, we either have not drawn
 104   // using the canvas at all, or we have already destroyed it.
 105   if (!mOffscreenCanvas) {
 106     return;
 107   }
 108
 109   // We assign/destroy the worker ref at the same time as the OffscreenCanvas
 110   // ref, so we can only have the canvas without a worker ref if it exists on
 111   // the main thread.
 112   if (!mWorkerRef) {
 113     // We queue to ensure that we have the same asynchronous update behaviour
 114     // for a main thread and a worker based OffscreenCanvas.
 115     mOffscreenCanvas->QueueCommitToCompositor();
 116     return;
 117   }
 118
 119   class FlushWorkerRunnable final : public WorkerRunnable {
 120    public:
 121     FlushWorkerRunnable(WorkerPrivate* aWorkerPrivate,
 122                         OffscreenCanvasDisplayHelper* aDisplayHelper)
 123         : WorkerRunnable(aWorkerPrivate, "FlushWorkerRunnable"),
 124           mDisplayHelper(aDisplayHelper) {}
 125
 126     bool WorkerRun(JSContext*, WorkerPrivate*) override {
 127       // The OffscreenCanvas can only be freed on the worker thread, so we
 128       // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
 129       // destructor. We just need to make sure we don't call into
 130       // OffscreenCanvas while holding the lock since it calls back into the
 131       // OffscreenCanvasDisplayHelper.
 132       RefPtr<OffscreenCanvas> canvas;
 133       {
 134         MutexAutoLock lock(mDisplayHelper->mMutex);
 135         canvas = mDisplayHelper->mOffscreenCanvas;
 136       }
 137
 138       if (canvas) {
 139         canvas->CommitFrameToCompositor();
 140       }
 141       return true;
 142     }
 143
 144    private:
 145     RefPtr<OffscreenCanvasDisplayHelper> mDisplayHelper;
 146   };
 147
 148   // Otherwise we are calling from the main thread during painting to a canvas
 149   // on a worker thread.
 150   auto task = MakeRefPtr<FlushWorkerRunnable>(mWorkerRef->Private(), this);
 151   task->Dispatch();
 152 }
 153
 154 bool OffscreenCanvasDisplayHelper::CommitFrameToCompositor(
 155     nsICanvasRenderingContextInternal* aContext,
 156     layers::TextureType aTextureType,
 157     const Maybe<OffscreenCanvasDisplayData>& aData) {
 158   auto endTransaction = MakeScopeExit([&]() {
 159     if (auto* cm = gfx::CanvasManagerChild::Get()) {
 160       cm->EndCanvasTransaction();
 161     }
 162   });
 163
 164   MutexAutoLock lock(mMutex);
 165
 166   gfx::SurfaceFormat format = gfx::SurfaceFormat::B8G8R8A8;
 167   layers::TextureFlags flags = layers::TextureFlags::IMMUTABLE;
 168
 169   if (!mCanvasElement) {
 170     // Our weak reference to the canvas element has been cleared, so we cannot
 171     // present directly anymore.
 172     return false;
 173   }
 174
 175   if (aData) {
 176     mData = aData.ref();
 177     MaybeQueueInvalidateElement();
 178   }
 179
 180   if (mData.mOwnerId.isSome()) {
 181     // No need to update the ImageContainer as the presentation itself is
 182     // handled in the compositor process.
 183     return true;
 184   }
 185
 186   if (!mImageContainer) {
 187     return false;
 188   }
 189
 190   if (mData.mIsOpaque) {
 191     flags |= layers::TextureFlags::IS_OPAQUE;
 192     format = gfx::SurfaceFormat::B8G8R8X8;
 193   } else if (!mData.mIsAlphaPremult) {
 194     flags |= layers::TextureFlags::NON_PREMULTIPLIED;
 195   }
 196
 197   switch (mData.mOriginPos) {
 198     case gl::OriginPos::BottomLeft:
 199       flags |= layers::TextureFlags::ORIGIN_BOTTOM_LEFT;
 200       break;
 201     case gl::OriginPos::TopLeft:
 202       break;
 203     default:
 204       MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
 205       break;
 206   }
 207
 208   auto imageBridge = layers::ImageBridgeChild::GetSingleton();
 209   if (!imageBridge) {
 210     return false;
 211   }
 212
 213   bool paintCallbacks = mData.mDoPaintCallbacks;
 214   bool hasRemoteTextureDesc = false;
 215   RefPtr<layers::Image> image;
 216   RefPtr<layers::TextureClient> texture;
 217   RefPtr<gfx::SourceSurface> surface;
 218   Maybe<layers::SurfaceDescriptor> desc;
 219   RefPtr<layers::FwdTransactionTracker> tracker;
 220
 221   {
 222     MutexAutoUnlock unlock(mMutex);
 223     if (paintCallbacks) {
 224       aContext->OnBeforePaintTransaction();
 225     }
 226
 227     desc = aContext->PresentFrontBuffer(nullptr, aTextureType);
 228     if (desc) {
 229       hasRemoteTextureDesc =
 230           desc->type() ==
 231           layers::SurfaceDescriptor::TSurfaceDescriptorRemoteTexture;
 232       if (hasRemoteTextureDesc) {
 233         tracker = aContext->UseCompositableForwarder(imageBridge);
 234         if (tracker) {
 235           flags |= layers::TextureFlags::WAIT_FOR_REMOTE_TEXTURE_OWNER;
 236         }
 237       }
 238     } else {
 239       if (layers::PersistentBufferProvider* provider =
 240               aContext->GetBufferProvider()) {
 241         texture = provider->GetTextureClient();
 242       }
 243
 244       if (!texture) {
 245         surface =
 246             aContext->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
 247         if (surface && surface->GetType() == gfx::SurfaceType::WEBGL) {
 248           // Ensure we can map in the surface. If we get a SourceSurfaceWebgl
 249           // surface, then it may not be backed by raw pixels yet. We need to
 250           // map it on the owning thread rather than the ImageBridge thread.
 251           gfx::DataSourceSurface::ScopedMap map(
 252               static_cast<gfx::DataSourceSurface*>(surface.get()),
 253               gfx::DataSourceSurface::READ);
 254           if (!map.IsMapped()) {
 255             surface = nullptr;
 256           }
 257         }
 258       }
 259     }
 260
 261     if (paintCallbacks) {
 262       aContext->OnDidPaintTransaction();
 263     }
 264   }
 265
 266   // We save any current surface because we might need it in GetSnapshot. If we
 267   // are on a worker thread and not WebGL, then this will be the only way we can
 268   // access the pixel data on the main thread.
 269   mFrontBufferSurface = surface;
 270
 271   // We do not use the ImageContainer plumbing with remote textures, so if we
 272   // have that, we can just early return here.
 273   if (hasRemoteTextureDesc) {
 274     const auto& textureDesc = desc->get_SurfaceDescriptorRemoteTexture();
 275     imageBridge->UpdateCompositable(mImageContainer, textureDesc.textureId(),
 276                                     textureDesc.ownerId(), mData.mSize, flags,
 277                                     tracker);
 278     return true;
 279   }
 280
 281   if (surface) {
 282     auto surfaceImage = MakeRefPtr<layers::SourceSurfaceImage>(surface);
 283     surfaceImage->SetTextureFlags(flags);
 284     image = surfaceImage;
 285   } else {
 286     if (desc && !texture) {
 287       texture = layers::SharedSurfaceTextureData::CreateTextureClient(
 288           *desc, format, mData.mSize, flags, imageBridge);
 289     }
 290     if (texture) {
 291       image = new layers::TextureWrapperImage(
 292           texture, gfx::IntRect(gfx::IntPoint(0, 0), texture->GetSize()));
 293     }
 294   }
 295
 296   if (image) {
 297     AutoTArray<layers::ImageContainer::NonOwningImage, 1> imageList;
 298     imageList.AppendElement(layers::ImageContainer::NonOwningImage(
 299         image, TimeStamp(), mLastFrameID++, mImageProducerID));
 300     mImageContainer->SetCurrentImages(imageList);
 301   } else {
 302     mImageContainer->ClearAllImages();
 303   }
 304
 305   return true;
 306 }
 307
 308 void OffscreenCanvasDisplayHelper::MaybeQueueInvalidateElement() {
 309   if (!mPendingInvalidate) {
 310     mPendingInvalidate = true;
 311     NS_DispatchToMainThread(NS_NewRunnableFunction(
 312         "OffscreenCanvasDisplayHelper::InvalidateElement",
 313         [self = RefPtr{this}] { self->InvalidateElement(); }));
 314   }
 315 }
 316
 317 void OffscreenCanvasDisplayHelper::InvalidateElement() {
 318   MOZ_ASSERT(NS_IsMainThread());
 319
 320   HTMLCanvasElement* canvasElement;
 321   gfx::IntSize size;
 322
 323   {
 324     MutexAutoLock lock(mMutex);
 325     MOZ_ASSERT(mPendingInvalidate);
 326     mPendingInvalidate = false;
 327     canvasElement = mCanvasElement;
 328     size = mData.mSize;
 329   }
 330
 331   if (canvasElement) {
 332     SVGObserverUtils::InvalidateDirectRenderingObservers(canvasElement);
 333     canvasElement->InvalidateCanvasPlaceholder(size.width, size.height);
 334     canvasElement->InvalidateCanvasContent(nullptr);
 335   }
 336 }
 337
 338 already_AddRefed<gfx::SourceSurface>
 339 OffscreenCanvasDisplayHelper::TransformSurface(gfx::SourceSurface* aSurface,
 340                                                bool aHasAlpha,
 341                                                bool aIsAlphaPremult,
 342                                                gl::OriginPos aOriginPos) const {
 343   if (!aSurface) {
 344     return nullptr;
 345   }
 346
 347   if (aOriginPos == gl::OriginPos::TopLeft && (!aHasAlpha || aIsAlphaPremult)) {
 348     // If we don't need to y-flip, and it is either opaque or premultiplied,
 349     // we can just return the same surface.
 350     return do_AddRef(aSurface);
 351   }
 352
 353   // Otherwise we need to copy and apply the necessary transformations.
 354   RefPtr<gfx::DataSourceSurface> srcSurface = aSurface->GetDataSurface();
 355   if (!srcSurface) {
 356     return nullptr;
 357   }
 358
 359   const auto size = srcSurface->GetSize();
 360   const auto format = srcSurface->GetFormat();
 361
 362   RefPtr<gfx::DataSourceSurface> dstSurface =
 363       gfx::Factory::CreateDataSourceSurface(size, format, /* aZero */ false);
 364   if (!dstSurface) {
 365     return nullptr;
 366   }
 367
 368   gfx::DataSourceSurface::ScopedMap srcMap(srcSurface,
 369                                            gfx::DataSourceSurface::READ);
 370   gfx::DataSourceSurface::ScopedMap dstMap(dstSurface,
 371                                            gfx::DataSourceSurface::WRITE);
 372   if (!srcMap.IsMapped() || !dstMap.IsMapped()) {
 373     return nullptr;
 374   }
 375
 376   bool success;
 377   switch (aOriginPos) {
 378     case gl::OriginPos::BottomLeft:
 379       if (aHasAlpha && !aIsAlphaPremult) {
 380         success = gfx::PremultiplyYFlipData(
 381             srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(),
 382             dstMap.GetStride(), format, size);
 383       } else {
 384         success = gfx::SwizzleYFlipData(srcMap.GetData(), srcMap.GetStride(),
 385                                         format, dstMap.GetData(),
 386                                         dstMap.GetStride(), format, size);
 387       }
 388       break;
 389     case gl::OriginPos::TopLeft:
 390       if (aHasAlpha && !aIsAlphaPremult) {
 391         success = gfx::PremultiplyData(srcMap.GetData(), srcMap.GetStride(),
 392                                        format, dstMap.GetData(),
 393                                        dstMap.GetStride(), format, size);
 394       } else {
 395         success = gfx::SwizzleData(srcMap.GetData(), srcMap.GetStride(), format,
 396                                    dstMap.GetData(), dstMap.GetStride(), format,
 397                                    size);
 398       }
 399       break;
 400     default:
 401       MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
 402       success = false;
 403       break;
 404   }
 405
 406   if (!success) {
 407     return nullptr;
 408   }
 409
 410   return dstSurface.forget();
 411 }
 412
 413 already_AddRefed<gfx::SourceSurface>
 414 OffscreenCanvasDisplayHelper::GetSurfaceSnapshot() {
 415   MOZ_ASSERT(NS_IsMainThread());
 416
 417   class SnapshotWorkerRunnable final : public MainThreadWorkerRunnable {
 418    public:
 419     SnapshotWorkerRunnable(WorkerPrivate* aWorkerPrivate,
 420                            OffscreenCanvasDisplayHelper* aDisplayHelper)
 421         : MainThreadWorkerRunnable(aWorkerPrivate, "SnapshotWorkerRunnable"),
 422           mMonitor("SnapshotWorkerRunnable::mMonitor"),
 423           mDisplayHelper(aDisplayHelper) {}
 424
 425     bool WorkerRun(JSContext*, WorkerPrivate*) override {
 426       // The OffscreenCanvas can only be freed on the worker thread, so we
 427       // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
 428       // destructor. We just need to make sure we don't call into
 429       // OffscreenCanvas while holding the lock since it calls back into the
 430       // OffscreenCanvasDisplayHelper.
 431       RefPtr<OffscreenCanvas> canvas;
 432       {
 433         MutexAutoLock lock(mDisplayHelper->mMutex);
 434         canvas = mDisplayHelper->mOffscreenCanvas;
 435       }
 436
 437       // Now that we are on the correct thread, we can extract the snapshot. If
 438       // it is a Skia surface, perform a copy to threading issues.
 439       RefPtr<gfx::SourceSurface> surface;
 440       if (canvas) {
 441         if (auto* context = canvas->GetContext()) {
 442           surface =
 443               context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
 444           if (surface && surface->GetType() == gfx::SurfaceType::SKIA) {
 445             surface = gfx::Factory::CopyDataSourceSurface(
 446                 static_cast<gfx::DataSourceSurface*>(surface.get()));
 447           }
 448         }
 449       }
 450
 451       MonitorAutoLock lock(mMonitor);
 452       mSurface = std::move(surface);
 453       mComplete = true;
 454       lock.NotifyAll();
 455       return true;
 456     }
 457
 458     already_AddRefed<gfx::SourceSurface> Wait(int32_t aTimeoutMs) {
 459       MonitorAutoLock lock(mMonitor);
 460
 461       TimeDuration timeout = TimeDuration::FromMilliseconds(aTimeoutMs);
 462       while (!mComplete) {
 463         if (lock.Wait(timeout) == CVStatus::Timeout) {
 464           return nullptr;
 465         }
 466       }
 467
 468       return mSurface.forget();
 469     }
 470
 471    private:
 472     Monitor mMonitor;
 473     RefPtr<OffscreenCanvasDisplayHelper> mDisplayHelper;
 474     RefPtr<gfx::SourceSurface> mSurface MOZ_GUARDED_BY(mMonitor);
 475     bool mComplete MOZ_GUARDED_BY(mMonitor) = false;
 476   };
 477
 478   bool hasAlpha;
 479   bool isAlphaPremult;
 480   gl::OriginPos originPos;
 481   HTMLCanvasElement* canvasElement;
 482   RefPtr<gfx::SourceSurface> surface;
 483   RefPtr<SnapshotWorkerRunnable> workerRunnable;
 484
 485   {
 486     MutexAutoLock lock(mMutex);
 487 #ifdef MOZ_WIDGET_ANDROID
 488     // On Android, we cannot both display a GL context and read back the pixels.
 489     if (mCanvasElement) {
 490       return nullptr;
 491     }
 492 #endif
 493
 494     hasAlpha = !mData.mIsOpaque;
 495     isAlphaPremult = mData.mIsAlphaPremult;
 496     originPos = mData.mOriginPos;
 497     canvasElement = mCanvasElement;
 498     if (mWorkerRef) {
 499       workerRunnable =
 500           MakeRefPtr<SnapshotWorkerRunnable>(mWorkerRef->Private(), this);
 501       workerRunnable->Dispatch();
 502     }
 503   }
 504
 505   if (workerRunnable) {
 506     // We transferred to a DOM worker, so we need to do the readback on the
 507     // owning thread and wait for the result.
 508     surface = workerRunnable->Wait(
 509         StaticPrefs::gfx_offscreencanvas_snapshot_timeout_ms());
 510   } else if (canvasElement) {
 511     // If we have a context, it is owned by the main thread.
 512     const auto* offscreenCanvas = canvasElement->GetOffscreenCanvas();
 513     if (nsICanvasRenderingContextInternal* context =
 514             offscreenCanvas->GetContext()) {
 515       surface =
 516           context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
 517     }
 518   }
 519
 520   return TransformSurface(surface, hasAlpha, isAlphaPremult, originPos);
 521 }
 522
 523 already_AddRefed<layers::Image> OffscreenCanvasDisplayHelper::GetAsImage() {
 524   MOZ_ASSERT(NS_IsMainThread());
 525
 526   RefPtr<gfx::SourceSurface> surface = GetSurfaceSnapshot();
 527   if (!surface) {
 528     return nullptr;
 529   }
 530   return MakeAndAddRef<layers::SourceSurfaceImage>(surface);
 531 }
 532
 533 UniquePtr<uint8_t[]> OffscreenCanvasDisplayHelper::GetImageBuffer(
 534     int32_t* aOutFormat, gfx::IntSize* aOutImageSize) {
 535   RefPtr<gfx::SourceSurface> surface = GetSurfaceSnapshot();
 536   if (!surface) {
 537     return nullptr;
 538   }
 539
 540   RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
 541   if (!dataSurface) {
 542     return nullptr;
 543   }
 544
 545   *aOutFormat = imgIEncoder::INPUT_FORMAT_HOSTARGB;
 546   *aOutImageSize = dataSurface->GetSize();
 547
 548   UniquePtr<uint8_t[]> imageBuffer = gfx::SurfaceToPackedBGRA(dataSurface);
 549   if (!imageBuffer) {
 550     return nullptr;
 551   }
 552
 553   bool resistFingerprinting;
 554   nsICookieJarSettings* cookieJarSettings = nullptr;
 555   {
 556     MutexAutoLock lock(mMutex);
 557     if (mCanvasElement) {
 558       Document* doc = mCanvasElement->OwnerDoc();
 559       resistFingerprinting =
 560           doc->ShouldResistFingerprinting(RFPTarget::CanvasRandomization);
 561       if (resistFingerprinting) {
 562         cookieJarSettings = doc->CookieJarSettings();
 563       }
 564     } else {
 565       resistFingerprinting = nsContentUtils::ShouldResistFingerprinting(
 566           "Fallback", RFPTarget::CanvasRandomization);
 567     }
 568   }
 569
 570   if (resistFingerprinting) {
 571     nsRFPService::RandomizePixels(
 572         cookieJarSettings, imageBuffer.get(), dataSurface->GetSize().width,
 573         dataSurface->GetSize().height,
 574         dataSurface->GetSize().width * dataSurface->GetSize().height * 4,
 575         gfx::SurfaceFormat::A8R8G8B8_UINT32);
 576   }
 577   return imageBuffer;
 578 }
 579
 580 }  // namespace mozilla::dom