1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "OffscreenCanvasDisplayHelper.h"
8 #include "mozilla/dom/Document.h"
9 #include "mozilla/dom/WorkerRef.h"
10 #include "mozilla/dom/WorkerRunnable.h"
11 #include "mozilla/gfx/2D.h"
12 #include "mozilla/gfx/CanvasManagerChild.h"
13 #include "mozilla/gfx/DataSurfaceHelpers.h"
14 #include "mozilla/gfx/Swizzle.h"
15 #include "mozilla/layers/ImageBridgeChild.h"
16 #include "mozilla/layers/PersistentBufferProvider.h"
17 #include "mozilla/layers/TextureClientSharedSurface.h"
18 #include "mozilla/layers/TextureWrapperImage.h"
19 #include "mozilla/StaticPrefs_gfx.h"
20 #include "mozilla/SVGObserverUtils.h"
21 #include "nsICanvasRenderingContextInternal.h"
22 #include "nsRFPService.h"
24 namespace mozilla::dom
{
26 OffscreenCanvasDisplayHelper::OffscreenCanvasDisplayHelper(
27 HTMLCanvasElement
* aCanvasElement
, uint32_t aWidth
, uint32_t aHeight
)
28 : mMutex("mozilla::dom::OffscreenCanvasDisplayHelper"),
29 mCanvasElement(aCanvasElement
),
30 mImageProducerID(layers::ImageContainer::AllocateProducerID()) {
31 mData
.mSize
.width
= aWidth
;
32 mData
.mSize
.height
= aHeight
;
35 OffscreenCanvasDisplayHelper::~OffscreenCanvasDisplayHelper() = default;
37 void OffscreenCanvasDisplayHelper::DestroyElement() {
38 MOZ_ASSERT(NS_IsMainThread());
40 MutexAutoLock
lock(mMutex
);
41 if (mImageContainer
) {
42 mImageContainer
->ClearAllImages();
43 mImageContainer
= nullptr;
45 mFrontBufferSurface
= nullptr;
46 mCanvasElement
= nullptr;
49 void OffscreenCanvasDisplayHelper::DestroyCanvas() {
50 if (auto* cm
= gfx::CanvasManagerChild::Get()) {
51 cm
->EndCanvasTransaction();
54 MutexAutoLock
lock(mMutex
);
55 if (mImageContainer
) {
56 mImageContainer
->ClearAllImages();
57 mImageContainer
= nullptr;
59 mFrontBufferSurface
= nullptr;
60 mOffscreenCanvas
= nullptr;
64 bool OffscreenCanvasDisplayHelper::CanElementCaptureStream() const {
65 MutexAutoLock
lock(mMutex
);
69 bool OffscreenCanvasDisplayHelper::UsingElementCaptureStream() const {
70 MutexAutoLock
lock(mMutex
);
72 if (NS_WARN_IF(!NS_IsMainThread())) {
73 MOZ_ASSERT_UNREACHABLE("Should not call off main-thread!");
74 return !!mCanvasElement
;
77 return mCanvasElement
&& mCanvasElement
->UsingCaptureStream();
80 CanvasContextType
OffscreenCanvasDisplayHelper::GetContextType() const {
81 MutexAutoLock
lock(mMutex
);
85 RefPtr
<layers::ImageContainer
> OffscreenCanvasDisplayHelper::GetImageContainer()
87 MutexAutoLock
lock(mMutex
);
88 return mImageContainer
;
91 void OffscreenCanvasDisplayHelper::UpdateContext(
92 OffscreenCanvas
* aOffscreenCanvas
, RefPtr
<ThreadSafeWorkerRef
>&& aWorkerRef
,
93 CanvasContextType aType
, const Maybe
<int32_t>& aChildId
) {
94 RefPtr
<layers::ImageContainer
> imageContainer
=
95 MakeRefPtr
<layers::ImageContainer
>(
96 layers::ImageUsageType::OffscreenCanvas
,
97 layers::ImageContainer::ASYNCHRONOUS
);
99 MutexAutoLock
lock(mMutex
);
101 mOffscreenCanvas
= aOffscreenCanvas
;
102 mWorkerRef
= std::move(aWorkerRef
);
104 mContextChildId
= aChildId
;
105 mImageContainer
= std::move(imageContainer
);
108 mContextManagerId
= Some(gfx::CanvasManagerChild::Get()->Id());
110 mContextManagerId
.reset();
113 MaybeQueueInvalidateElement();
116 void OffscreenCanvasDisplayHelper::FlushForDisplay() {
117 MOZ_ASSERT(NS_IsMainThread());
119 MutexAutoLock
lock(mMutex
);
121 // Without an OffscreenCanvas object bound to us, we either have not drawn
122 // using the canvas at all, or we have already destroyed it.
123 if (!mOffscreenCanvas
) {
127 // We assign/destroy the worker ref at the same time as the OffscreenCanvas
128 // ref, so we can only have the canvas without a worker ref if it exists on
131 // We queue to ensure that we have the same asynchronous update behaviour
132 // for a main thread and a worker based OffscreenCanvas.
133 mOffscreenCanvas
->QueueCommitToCompositor();
137 class FlushWorkerRunnable final
: public WorkerThreadRunnable
{
139 FlushWorkerRunnable(WorkerPrivate
* aWorkerPrivate
,
140 OffscreenCanvasDisplayHelper
* aDisplayHelper
)
141 : WorkerThreadRunnable("FlushWorkerRunnable"),
142 mDisplayHelper(aDisplayHelper
) {}
144 bool WorkerRun(JSContext
*, WorkerPrivate
*) override
{
145 // The OffscreenCanvas can only be freed on the worker thread, so we
146 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
147 // destructor. We just need to make sure we don't call into
148 // OffscreenCanvas while holding the lock since it calls back into the
149 // OffscreenCanvasDisplayHelper.
150 RefPtr
<OffscreenCanvas
> canvas
;
152 MutexAutoLock
lock(mDisplayHelper
->mMutex
);
153 canvas
= mDisplayHelper
->mOffscreenCanvas
;
157 canvas
->CommitFrameToCompositor();
163 RefPtr
<OffscreenCanvasDisplayHelper
> mDisplayHelper
;
166 // Otherwise we are calling from the main thread during painting to a canvas
167 // on a worker thread.
168 auto task
= MakeRefPtr
<FlushWorkerRunnable
>(mWorkerRef
->Private(), this);
169 task
->Dispatch(mWorkerRef
->Private());
172 bool OffscreenCanvasDisplayHelper::CommitFrameToCompositor(
173 nsICanvasRenderingContextInternal
* aContext
,
174 layers::TextureType aTextureType
,
175 const Maybe
<OffscreenCanvasDisplayData
>& aData
) {
176 auto endTransaction
= MakeScopeExit([&]() {
177 if (auto* cm
= gfx::CanvasManagerChild::Get()) {
178 cm
->EndCanvasTransaction();
182 MutexAutoLock
lock(mMutex
);
184 gfx::SurfaceFormat format
= gfx::SurfaceFormat::B8G8R8A8
;
185 layers::TextureFlags flags
= layers::TextureFlags::IMMUTABLE
;
187 if (!mCanvasElement
) {
188 // Our weak reference to the canvas element has been cleared, so we cannot
189 // present directly anymore.
195 MaybeQueueInvalidateElement();
198 if (mData
.mOwnerId
.isSome()) {
199 // No need to update the ImageContainer as the presentation itself is
200 // handled in the compositor process.
204 if (!mImageContainer
) {
208 if (mData
.mIsOpaque
) {
209 flags
|= layers::TextureFlags::IS_OPAQUE
;
210 format
= gfx::SurfaceFormat::B8G8R8X8
;
211 } else if (!mData
.mIsAlphaPremult
) {
212 flags
|= layers::TextureFlags::NON_PREMULTIPLIED
;
215 switch (mData
.mOriginPos
) {
216 case gl::OriginPos::BottomLeft
:
217 flags
|= layers::TextureFlags::ORIGIN_BOTTOM_LEFT
;
219 case gl::OriginPos::TopLeft
:
222 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
226 auto imageBridge
= layers::ImageBridgeChild::GetSingleton();
231 bool paintCallbacks
= mData
.mDoPaintCallbacks
;
232 bool hasRemoteTextureDesc
= false;
233 RefPtr
<layers::Image
> image
;
234 RefPtr
<layers::TextureClient
> texture
;
235 RefPtr
<gfx::SourceSurface
> surface
;
236 Maybe
<layers::SurfaceDescriptor
> desc
;
237 RefPtr
<layers::FwdTransactionTracker
> tracker
;
240 MutexAutoUnlock
unlock(mMutex
);
241 if (paintCallbacks
) {
242 aContext
->OnBeforePaintTransaction();
245 desc
= aContext
->PresentFrontBuffer(nullptr, aTextureType
);
247 hasRemoteTextureDesc
=
249 layers::SurfaceDescriptor::TSurfaceDescriptorRemoteTexture
;
250 if (hasRemoteTextureDesc
) {
251 tracker
= aContext
->UseCompositableForwarder(imageBridge
);
253 flags
|= layers::TextureFlags::WAIT_FOR_REMOTE_TEXTURE_OWNER
;
257 if (layers::PersistentBufferProvider
* provider
=
258 aContext
->GetBufferProvider()) {
259 texture
= provider
->GetTextureClient();
264 aContext
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
265 if (surface
&& surface
->GetType() == gfx::SurfaceType::WEBGL
) {
266 // Ensure we can map in the surface. If we get a SourceSurfaceWebgl
267 // surface, then it may not be backed by raw pixels yet. We need to
268 // map it on the owning thread rather than the ImageBridge thread.
269 gfx::DataSourceSurface::ScopedMap
map(
270 static_cast<gfx::DataSourceSurface
*>(surface
.get()),
271 gfx::DataSourceSurface::READ
);
272 if (!map
.IsMapped()) {
279 if (paintCallbacks
) {
280 aContext
->OnDidPaintTransaction();
284 // We save any current surface because we might need it in GetSnapshot. If we
285 // are on a worker thread and not WebGL, then this will be the only way we can
286 // access the pixel data on the main thread.
287 mFrontBufferSurface
= surface
;
289 // We do not use the ImageContainer plumbing with remote textures, so if we
290 // have that, we can just early return here.
291 if (hasRemoteTextureDesc
) {
292 const auto& textureDesc
= desc
->get_SurfaceDescriptorRemoteTexture();
293 imageBridge
->UpdateCompositable(mImageContainer
, textureDesc
.textureId(),
294 textureDesc
.ownerId(), mData
.mSize
, flags
,
300 auto surfaceImage
= MakeRefPtr
<layers::SourceSurfaceImage
>(surface
);
301 surfaceImage
->SetTextureFlags(flags
);
302 image
= surfaceImage
;
304 if (desc
&& !texture
) {
305 texture
= layers::SharedSurfaceTextureData::CreateTextureClient(
306 *desc
, format
, mData
.mSize
, flags
, imageBridge
);
309 image
= new layers::TextureWrapperImage(
310 texture
, gfx::IntRect(gfx::IntPoint(0, 0), texture
->GetSize()));
315 AutoTArray
<layers::ImageContainer::NonOwningImage
, 1> imageList
;
316 imageList
.AppendElement(layers::ImageContainer::NonOwningImage(
317 image
, TimeStamp(), mLastFrameID
++, mImageProducerID
));
318 mImageContainer
->SetCurrentImages(imageList
);
320 mImageContainer
->ClearAllImages();
326 void OffscreenCanvasDisplayHelper::MaybeQueueInvalidateElement() {
327 if (!mPendingInvalidate
) {
328 mPendingInvalidate
= true;
329 NS_DispatchToMainThread(NS_NewRunnableFunction(
330 "OffscreenCanvasDisplayHelper::InvalidateElement",
331 [self
= RefPtr
{this}] { self
->InvalidateElement(); }));
335 void OffscreenCanvasDisplayHelper::InvalidateElement() {
336 MOZ_ASSERT(NS_IsMainThread());
338 HTMLCanvasElement
* canvasElement
;
342 MutexAutoLock
lock(mMutex
);
343 MOZ_ASSERT(mPendingInvalidate
);
344 mPendingInvalidate
= false;
345 canvasElement
= mCanvasElement
;
350 SVGObserverUtils::InvalidateDirectRenderingObservers(canvasElement
);
351 canvasElement
->InvalidateCanvasPlaceholder(size
.width
, size
.height
);
352 canvasElement
->InvalidateCanvasContent(nullptr);
356 already_AddRefed
<gfx::SourceSurface
>
357 OffscreenCanvasDisplayHelper::TransformSurface(gfx::SourceSurface
* aSurface
,
359 bool aIsAlphaPremult
,
360 gl::OriginPos aOriginPos
) const {
365 if (aOriginPos
== gl::OriginPos::TopLeft
&& (!aHasAlpha
|| aIsAlphaPremult
)) {
366 // If we don't need to y-flip, and it is either opaque or premultiplied,
367 // we can just return the same surface.
368 return do_AddRef(aSurface
);
371 // Otherwise we need to copy and apply the necessary transformations.
372 RefPtr
<gfx::DataSourceSurface
> srcSurface
= aSurface
->GetDataSurface();
377 const auto size
= srcSurface
->GetSize();
378 const auto format
= srcSurface
->GetFormat();
380 RefPtr
<gfx::DataSourceSurface
> dstSurface
=
381 gfx::Factory::CreateDataSourceSurface(size
, format
, /* aZero */ false);
386 gfx::DataSourceSurface::ScopedMap
srcMap(srcSurface
,
387 gfx::DataSourceSurface::READ
);
388 gfx::DataSourceSurface::ScopedMap
dstMap(dstSurface
,
389 gfx::DataSourceSurface::WRITE
);
390 if (!srcMap
.IsMapped() || !dstMap
.IsMapped()) {
395 switch (aOriginPos
) {
396 case gl::OriginPos::BottomLeft
:
397 if (aHasAlpha
&& !aIsAlphaPremult
) {
398 success
= gfx::PremultiplyYFlipData(
399 srcMap
.GetData(), srcMap
.GetStride(), format
, dstMap
.GetData(),
400 dstMap
.GetStride(), format
, size
);
402 success
= gfx::SwizzleYFlipData(srcMap
.GetData(), srcMap
.GetStride(),
403 format
, dstMap
.GetData(),
404 dstMap
.GetStride(), format
, size
);
407 case gl::OriginPos::TopLeft
:
408 if (aHasAlpha
&& !aIsAlphaPremult
) {
409 success
= gfx::PremultiplyData(srcMap
.GetData(), srcMap
.GetStride(),
410 format
, dstMap
.GetData(),
411 dstMap
.GetStride(), format
, size
);
413 success
= gfx::SwizzleData(srcMap
.GetData(), srcMap
.GetStride(), format
,
414 dstMap
.GetData(), dstMap
.GetStride(), format
,
419 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
428 return dstSurface
.forget();
431 already_AddRefed
<gfx::SourceSurface
>
432 OffscreenCanvasDisplayHelper::GetSurfaceSnapshot() {
433 MOZ_ASSERT(NS_IsMainThread());
435 class SnapshotWorkerRunnable final
: public MainThreadWorkerRunnable
{
437 SnapshotWorkerRunnable(WorkerPrivate
* aWorkerPrivate
,
438 OffscreenCanvasDisplayHelper
* aDisplayHelper
)
439 : MainThreadWorkerRunnable("SnapshotWorkerRunnable"),
440 mMonitor("SnapshotWorkerRunnable::mMonitor"),
441 mDisplayHelper(aDisplayHelper
) {}
443 bool WorkerRun(JSContext
*, WorkerPrivate
*) override
{
444 // The OffscreenCanvas can only be freed on the worker thread, so we
445 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
446 // destructor. We just need to make sure we don't call into
447 // OffscreenCanvas while holding the lock since it calls back into the
448 // OffscreenCanvasDisplayHelper.
449 RefPtr
<OffscreenCanvas
> canvas
;
451 MutexAutoLock
lock(mDisplayHelper
->mMutex
);
452 canvas
= mDisplayHelper
->mOffscreenCanvas
;
455 // Now that we are on the correct thread, we can extract the snapshot. If
456 // it is a Skia surface, perform a copy to threading issues.
457 RefPtr
<gfx::SourceSurface
> surface
;
459 if (auto* context
= canvas
->GetContext()) {
461 context
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
462 if (surface
&& surface
->GetType() == gfx::SurfaceType::SKIA
) {
463 surface
= gfx::Factory::CopyDataSourceSurface(
464 static_cast<gfx::DataSourceSurface
*>(surface
.get()));
469 MonitorAutoLock
lock(mMonitor
);
470 mSurface
= std::move(surface
);
476 already_AddRefed
<gfx::SourceSurface
> Wait(int32_t aTimeoutMs
) {
477 MonitorAutoLock
lock(mMonitor
);
479 TimeDuration timeout
= TimeDuration::FromMilliseconds(aTimeoutMs
);
481 if (lock
.Wait(timeout
) == CVStatus::Timeout
) {
486 return mSurface
.forget();
491 RefPtr
<OffscreenCanvasDisplayHelper
> mDisplayHelper
;
492 RefPtr
<gfx::SourceSurface
> mSurface
MOZ_GUARDED_BY(mMonitor
);
493 bool mComplete
MOZ_GUARDED_BY(mMonitor
) = false;
498 gl::OriginPos originPos
;
499 HTMLCanvasElement
* canvasElement
;
500 RefPtr
<gfx::SourceSurface
> surface
;
501 RefPtr
<SnapshotWorkerRunnable
> workerRunnable
;
504 MutexAutoLock
lock(mMutex
);
505 #ifdef MOZ_WIDGET_ANDROID
506 // On Android, we cannot both display a GL context and read back the pixels.
507 if (mCanvasElement
) {
512 hasAlpha
= !mData
.mIsOpaque
;
513 isAlphaPremult
= mData
.mIsAlphaPremult
;
514 originPos
= mData
.mOriginPos
;
515 canvasElement
= mCanvasElement
;
518 MakeRefPtr
<SnapshotWorkerRunnable
>(mWorkerRef
->Private(), this);
519 workerRunnable
->Dispatch(mWorkerRef
->Private());
523 if (workerRunnable
) {
524 // We transferred to a DOM worker, so we need to do the readback on the
525 // owning thread and wait for the result.
526 surface
= workerRunnable
->Wait(
527 StaticPrefs::gfx_offscreencanvas_snapshot_timeout_ms());
528 } else if (canvasElement
) {
529 // If we have a context, it is owned by the main thread.
530 const auto* offscreenCanvas
= canvasElement
->GetOffscreenCanvas();
531 if (nsICanvasRenderingContextInternal
* context
=
532 offscreenCanvas
->GetContext()) {
534 context
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
538 return TransformSurface(surface
, hasAlpha
, isAlphaPremult
, originPos
);
541 already_AddRefed
<layers::Image
> OffscreenCanvasDisplayHelper::GetAsImage() {
542 MOZ_ASSERT(NS_IsMainThread());
544 RefPtr
<gfx::SourceSurface
> surface
= GetSurfaceSnapshot();
548 return MakeAndAddRef
<layers::SourceSurfaceImage
>(surface
);
551 UniquePtr
<uint8_t[]> OffscreenCanvasDisplayHelper::GetImageBuffer(
552 int32_t* aOutFormat
, gfx::IntSize
* aOutImageSize
) {
553 RefPtr
<gfx::SourceSurface
> surface
= GetSurfaceSnapshot();
558 RefPtr
<gfx::DataSourceSurface
> dataSurface
= surface
->GetDataSurface();
563 *aOutFormat
= imgIEncoder::INPUT_FORMAT_HOSTARGB
;
564 *aOutImageSize
= dataSurface
->GetSize();
566 UniquePtr
<uint8_t[]> imageBuffer
= gfx::SurfaceToPackedBGRA(dataSurface
);
571 bool resistFingerprinting
;
572 nsICookieJarSettings
* cookieJarSettings
= nullptr;
574 MutexAutoLock
lock(mMutex
);
575 if (mCanvasElement
) {
576 Document
* doc
= mCanvasElement
->OwnerDoc();
577 resistFingerprinting
=
578 doc
->ShouldResistFingerprinting(RFPTarget::CanvasRandomization
);
579 if (resistFingerprinting
) {
580 cookieJarSettings
= doc
->CookieJarSettings();
583 resistFingerprinting
= nsContentUtils::ShouldResistFingerprinting(
584 "Fallback", RFPTarget::CanvasRandomization
);
588 if (resistFingerprinting
) {
589 nsRFPService::RandomizePixels(
590 cookieJarSettings
, imageBuffer
.get(), dataSurface
->GetSize().width
,
591 dataSurface
->GetSize().height
,
592 dataSurface
->GetSize().width
* dataSurface
->GetSize().height
* 4,
593 gfx::SurfaceFormat::A8R8G8B8_UINT32
);
598 } // namespace mozilla::dom