1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "OffscreenCanvasDisplayHelper.h"
8 #include "mozilla/dom/Document.h"
9 #include "mozilla/dom/WorkerRef.h"
10 #include "mozilla/dom/WorkerRunnable.h"
11 #include "mozilla/gfx/2D.h"
12 #include "mozilla/gfx/CanvasManagerChild.h"
13 #include "mozilla/gfx/DataSurfaceHelpers.h"
14 #include "mozilla/gfx/Swizzle.h"
15 #include "mozilla/layers/ImageBridgeChild.h"
16 #include "mozilla/layers/PersistentBufferProvider.h"
17 #include "mozilla/layers/TextureClientSharedSurface.h"
18 #include "mozilla/layers/TextureWrapperImage.h"
19 #include "mozilla/StaticPrefs_gfx.h"
20 #include "mozilla/SVGObserverUtils.h"
21 #include "nsICanvasRenderingContextInternal.h"
22 #include "nsRFPService.h"
24 namespace mozilla::dom
{
26 OffscreenCanvasDisplayHelper::OffscreenCanvasDisplayHelper(
27 HTMLCanvasElement
* aCanvasElement
, uint32_t aWidth
, uint32_t aHeight
)
28 : mMutex("mozilla::dom::OffscreenCanvasDisplayHelper"),
29 mCanvasElement(aCanvasElement
),
30 mImageProducerID(layers::ImageContainer::AllocateProducerID()) {
31 mData
.mSize
.width
= aWidth
;
32 mData
.mSize
.height
= aHeight
;
35 OffscreenCanvasDisplayHelper::~OffscreenCanvasDisplayHelper() = default;
37 void OffscreenCanvasDisplayHelper::DestroyElement() {
38 MOZ_ASSERT(NS_IsMainThread());
40 MutexAutoLock
lock(mMutex
);
41 if (mImageContainer
) {
42 mImageContainer
->ClearAllImages();
43 mImageContainer
= nullptr;
45 mFrontBufferSurface
= nullptr;
46 mCanvasElement
= nullptr;
49 void OffscreenCanvasDisplayHelper::DestroyCanvas() {
50 if (auto* cm
= gfx::CanvasManagerChild::Get()) {
51 cm
->EndCanvasTransaction();
54 MutexAutoLock
lock(mMutex
);
55 if (mImageContainer
) {
56 mImageContainer
->ClearAllImages();
57 mImageContainer
= nullptr;
59 mFrontBufferSurface
= nullptr;
60 mOffscreenCanvas
= nullptr;
64 bool OffscreenCanvasDisplayHelper::CanElementCaptureStream() const {
65 MutexAutoLock
lock(mMutex
);
69 bool OffscreenCanvasDisplayHelper::UsingElementCaptureStream() const {
70 MutexAutoLock
lock(mMutex
);
72 if (NS_WARN_IF(!NS_IsMainThread())) {
73 MOZ_ASSERT_UNREACHABLE("Should not call off main-thread!");
74 return !!mCanvasElement
;
77 return mCanvasElement
&& mCanvasElement
->UsingCaptureStream();
80 CanvasContextType
OffscreenCanvasDisplayHelper::GetContextType() const {
81 MutexAutoLock
lock(mMutex
);
85 RefPtr
<layers::ImageContainer
> OffscreenCanvasDisplayHelper::GetImageContainer()
87 MutexAutoLock
lock(mMutex
);
88 return mImageContainer
;
91 void OffscreenCanvasDisplayHelper::UpdateContext(
92 OffscreenCanvas
* aOffscreenCanvas
, RefPtr
<ThreadSafeWorkerRef
>&& aWorkerRef
,
93 CanvasContextType aType
, const Maybe
<int32_t>& aChildId
) {
94 RefPtr
<layers::ImageContainer
> imageContainer
=
95 MakeRefPtr
<layers::ImageContainer
>(layers::ImageContainer::ASYNCHRONOUS
);
97 MutexAutoLock
lock(mMutex
);
99 mOffscreenCanvas
= aOffscreenCanvas
;
100 mWorkerRef
= std::move(aWorkerRef
);
102 mContextChildId
= aChildId
;
103 mImageContainer
= std::move(imageContainer
);
106 mContextManagerId
= Some(gfx::CanvasManagerChild::Get()->Id());
108 mContextManagerId
.reset();
111 MaybeQueueInvalidateElement();
114 void OffscreenCanvasDisplayHelper::FlushForDisplay() {
115 MOZ_ASSERT(NS_IsMainThread());
117 MutexAutoLock
lock(mMutex
);
119 // Without an OffscreenCanvas object bound to us, we either have not drawn
120 // using the canvas at all, or we have already destroyed it.
121 if (!mOffscreenCanvas
) {
125 // We assign/destroy the worker ref at the same time as the OffscreenCanvas
126 // ref, so we can only have the canvas without a worker ref if it exists on
129 // We queue to ensure that we have the same asynchronous update behaviour
130 // for a main thread and a worker based OffscreenCanvas.
131 mOffscreenCanvas
->QueueCommitToCompositor();
135 class FlushWorkerRunnable final
: public WorkerRunnable
{
137 FlushWorkerRunnable(WorkerPrivate
* aWorkerPrivate
,
138 OffscreenCanvasDisplayHelper
* aDisplayHelper
)
139 : WorkerRunnable(aWorkerPrivate
, "FlushWorkerRunnable"),
140 mDisplayHelper(aDisplayHelper
) {}
142 bool WorkerRun(JSContext
*, WorkerPrivate
*) override
{
143 // The OffscreenCanvas can only be freed on the worker thread, so we
144 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
145 // destructor. We just need to make sure we don't call into
146 // OffscreenCanvas while holding the lock since it calls back into the
147 // OffscreenCanvasDisplayHelper.
148 RefPtr
<OffscreenCanvas
> canvas
;
150 MutexAutoLock
lock(mDisplayHelper
->mMutex
);
151 canvas
= mDisplayHelper
->mOffscreenCanvas
;
155 canvas
->CommitFrameToCompositor();
161 RefPtr
<OffscreenCanvasDisplayHelper
> mDisplayHelper
;
164 // Otherwise we are calling from the main thread during painting to a canvas
165 // on a worker thread.
166 auto task
= MakeRefPtr
<FlushWorkerRunnable
>(mWorkerRef
->Private(), this);
170 bool OffscreenCanvasDisplayHelper::CommitFrameToCompositor(
171 nsICanvasRenderingContextInternal
* aContext
,
172 layers::TextureType aTextureType
,
173 const Maybe
<OffscreenCanvasDisplayData
>& aData
) {
174 auto endTransaction
= MakeScopeExit([&]() {
175 if (auto* cm
= gfx::CanvasManagerChild::Get()) {
176 cm
->EndCanvasTransaction();
180 MutexAutoLock
lock(mMutex
);
182 gfx::SurfaceFormat format
= gfx::SurfaceFormat::B8G8R8A8
;
183 layers::TextureFlags flags
= layers::TextureFlags::IMMUTABLE
;
185 if (!mCanvasElement
) {
186 // Our weak reference to the canvas element has been cleared, so we cannot
187 // present directly anymore.
193 MaybeQueueInvalidateElement();
196 if (mData
.mOwnerId
.isSome()) {
197 // No need to update the ImageContainer as the presentation itself is
198 // handled in the compositor process.
202 if (!mImageContainer
) {
206 if (mData
.mIsOpaque
) {
207 flags
|= layers::TextureFlags::IS_OPAQUE
;
208 format
= gfx::SurfaceFormat::B8G8R8X8
;
209 } else if (!mData
.mIsAlphaPremult
) {
210 flags
|= layers::TextureFlags::NON_PREMULTIPLIED
;
213 switch (mData
.mOriginPos
) {
214 case gl::OriginPos::BottomLeft
:
215 flags
|= layers::TextureFlags::ORIGIN_BOTTOM_LEFT
;
217 case gl::OriginPos::TopLeft
:
220 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
224 auto imageBridge
= layers::ImageBridgeChild::GetSingleton();
229 bool paintCallbacks
= mData
.mDoPaintCallbacks
;
230 bool hasRemoteTextureDesc
= false;
231 RefPtr
<layers::Image
> image
;
232 RefPtr
<layers::TextureClient
> texture
;
233 RefPtr
<gfx::SourceSurface
> surface
;
234 Maybe
<layers::SurfaceDescriptor
> desc
;
235 RefPtr
<layers::FwdTransactionTracker
> tracker
;
238 MutexAutoUnlock
unlock(mMutex
);
239 if (paintCallbacks
) {
240 aContext
->OnBeforePaintTransaction();
243 desc
= aContext
->PresentFrontBuffer(nullptr, aTextureType
);
245 hasRemoteTextureDesc
=
247 layers::SurfaceDescriptor::TSurfaceDescriptorRemoteTexture
;
248 if (hasRemoteTextureDesc
) {
249 tracker
= aContext
->UseCompositableForwarder(imageBridge
);
251 flags
|= layers::TextureFlags::WAIT_FOR_REMOTE_TEXTURE_OWNER
;
255 if (layers::PersistentBufferProvider
* provider
=
256 aContext
->GetBufferProvider()) {
257 texture
= provider
->GetTextureClient();
262 aContext
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
263 if (surface
&& surface
->GetType() == gfx::SurfaceType::WEBGL
) {
264 // Ensure we can map in the surface. If we get a SourceSurfaceWebgl
265 // surface, then it may not be backed by raw pixels yet. We need to
266 // map it on the owning thread rather than the ImageBridge thread.
267 gfx::DataSourceSurface::ScopedMap
map(
268 static_cast<gfx::DataSourceSurface
*>(surface
.get()),
269 gfx::DataSourceSurface::READ
);
270 if (!map
.IsMapped()) {
277 if (paintCallbacks
) {
278 aContext
->OnDidPaintTransaction();
282 // We save any current surface because we might need it in GetSnapshot. If we
283 // are on a worker thread and not WebGL, then this will be the only way we can
284 // access the pixel data on the main thread.
285 mFrontBufferSurface
= surface
;
287 // We do not use the ImageContainer plumbing with remote textures, so if we
288 // have that, we can just early return here.
289 if (hasRemoteTextureDesc
) {
290 const auto& textureDesc
= desc
->get_SurfaceDescriptorRemoteTexture();
291 imageBridge
->UpdateCompositable(mImageContainer
, textureDesc
.textureId(),
292 textureDesc
.ownerId(), mData
.mSize
, flags
,
298 auto surfaceImage
= MakeRefPtr
<layers::SourceSurfaceImage
>(surface
);
299 surfaceImage
->SetTextureFlags(flags
);
300 image
= surfaceImage
;
302 if (desc
&& !texture
) {
303 texture
= layers::SharedSurfaceTextureData::CreateTextureClient(
304 *desc
, format
, mData
.mSize
, flags
, imageBridge
);
307 image
= new layers::TextureWrapperImage(
308 texture
, gfx::IntRect(gfx::IntPoint(0, 0), texture
->GetSize()));
313 AutoTArray
<layers::ImageContainer::NonOwningImage
, 1> imageList
;
314 imageList
.AppendElement(layers::ImageContainer::NonOwningImage(
315 image
, TimeStamp(), mLastFrameID
++, mImageProducerID
));
316 mImageContainer
->SetCurrentImages(imageList
);
318 mImageContainer
->ClearAllImages();
324 void OffscreenCanvasDisplayHelper::MaybeQueueInvalidateElement() {
325 if (!mPendingInvalidate
) {
326 mPendingInvalidate
= true;
327 NS_DispatchToMainThread(NS_NewRunnableFunction(
328 "OffscreenCanvasDisplayHelper::InvalidateElement",
329 [self
= RefPtr
{this}] { self
->InvalidateElement(); }));
333 void OffscreenCanvasDisplayHelper::InvalidateElement() {
334 MOZ_ASSERT(NS_IsMainThread());
336 HTMLCanvasElement
* canvasElement
;
340 MutexAutoLock
lock(mMutex
);
341 MOZ_ASSERT(mPendingInvalidate
);
342 mPendingInvalidate
= false;
343 canvasElement
= mCanvasElement
;
348 SVGObserverUtils::InvalidateDirectRenderingObservers(canvasElement
);
349 canvasElement
->InvalidateCanvasPlaceholder(size
.width
, size
.height
);
350 canvasElement
->InvalidateCanvasContent(nullptr);
354 already_AddRefed
<gfx::SourceSurface
>
355 OffscreenCanvasDisplayHelper::TransformSurface(gfx::SourceSurface
* aSurface
,
357 bool aIsAlphaPremult
,
358 gl::OriginPos aOriginPos
) const {
363 if (aOriginPos
== gl::OriginPos::TopLeft
&& (!aHasAlpha
|| aIsAlphaPremult
)) {
364 // If we don't need to y-flip, and it is either opaque or premultiplied,
365 // we can just return the same surface.
366 return do_AddRef(aSurface
);
369 // Otherwise we need to copy and apply the necessary transformations.
370 RefPtr
<gfx::DataSourceSurface
> srcSurface
= aSurface
->GetDataSurface();
375 const auto size
= srcSurface
->GetSize();
376 const auto format
= srcSurface
->GetFormat();
378 RefPtr
<gfx::DataSourceSurface
> dstSurface
=
379 gfx::Factory::CreateDataSourceSurface(size
, format
, /* aZero */ false);
384 gfx::DataSourceSurface::ScopedMap
srcMap(srcSurface
,
385 gfx::DataSourceSurface::READ
);
386 gfx::DataSourceSurface::ScopedMap
dstMap(dstSurface
,
387 gfx::DataSourceSurface::WRITE
);
388 if (!srcMap
.IsMapped() || !dstMap
.IsMapped()) {
393 switch (aOriginPos
) {
394 case gl::OriginPos::BottomLeft
:
395 if (aHasAlpha
&& !aIsAlphaPremult
) {
396 success
= gfx::PremultiplyYFlipData(
397 srcMap
.GetData(), srcMap
.GetStride(), format
, dstMap
.GetData(),
398 dstMap
.GetStride(), format
, size
);
400 success
= gfx::SwizzleYFlipData(srcMap
.GetData(), srcMap
.GetStride(),
401 format
, dstMap
.GetData(),
402 dstMap
.GetStride(), format
, size
);
405 case gl::OriginPos::TopLeft
:
406 if (aHasAlpha
&& !aIsAlphaPremult
) {
407 success
= gfx::PremultiplyData(srcMap
.GetData(), srcMap
.GetStride(),
408 format
, dstMap
.GetData(),
409 dstMap
.GetStride(), format
, size
);
411 success
= gfx::SwizzleData(srcMap
.GetData(), srcMap
.GetStride(), format
,
412 dstMap
.GetData(), dstMap
.GetStride(), format
,
417 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
426 return dstSurface
.forget();
429 already_AddRefed
<gfx::SourceSurface
>
430 OffscreenCanvasDisplayHelper::GetSurfaceSnapshot() {
431 MOZ_ASSERT(NS_IsMainThread());
433 class SnapshotWorkerRunnable final
: public MainThreadWorkerRunnable
{
435 SnapshotWorkerRunnable(WorkerPrivate
* aWorkerPrivate
,
436 OffscreenCanvasDisplayHelper
* aDisplayHelper
)
437 : MainThreadWorkerRunnable(aWorkerPrivate
, "SnapshotWorkerRunnable"),
438 mMonitor("SnapshotWorkerRunnable::mMonitor"),
439 mDisplayHelper(aDisplayHelper
) {}
441 bool WorkerRun(JSContext
*, WorkerPrivate
*) override
{
442 // The OffscreenCanvas can only be freed on the worker thread, so we
443 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
444 // destructor. We just need to make sure we don't call into
445 // OffscreenCanvas while holding the lock since it calls back into the
446 // OffscreenCanvasDisplayHelper.
447 RefPtr
<OffscreenCanvas
> canvas
;
449 MutexAutoLock
lock(mDisplayHelper
->mMutex
);
450 canvas
= mDisplayHelper
->mOffscreenCanvas
;
453 // Now that we are on the correct thread, we can extract the snapshot. If
454 // it is a Skia surface, perform a copy to threading issues.
455 RefPtr
<gfx::SourceSurface
> surface
;
457 if (auto* context
= canvas
->GetContext()) {
459 context
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
460 if (surface
&& surface
->GetType() == gfx::SurfaceType::SKIA
) {
461 surface
= gfx::Factory::CopyDataSourceSurface(
462 static_cast<gfx::DataSourceSurface
*>(surface
.get()));
467 MonitorAutoLock
lock(mMonitor
);
468 mSurface
= std::move(surface
);
474 already_AddRefed
<gfx::SourceSurface
> Wait(int32_t aTimeoutMs
) {
475 MonitorAutoLock
lock(mMonitor
);
477 TimeDuration timeout
= TimeDuration::FromMilliseconds(aTimeoutMs
);
479 if (lock
.Wait(timeout
) == CVStatus::Timeout
) {
484 return mSurface
.forget();
489 RefPtr
<OffscreenCanvasDisplayHelper
> mDisplayHelper
;
490 RefPtr
<gfx::SourceSurface
> mSurface
MOZ_GUARDED_BY(mMonitor
);
491 bool mComplete
MOZ_GUARDED_BY(mMonitor
) = false;
496 gl::OriginPos originPos
;
497 HTMLCanvasElement
* canvasElement
;
498 RefPtr
<gfx::SourceSurface
> surface
;
499 RefPtr
<SnapshotWorkerRunnable
> workerRunnable
;
502 MutexAutoLock
lock(mMutex
);
503 #ifdef MOZ_WIDGET_ANDROID
504 // On Android, we cannot both display a GL context and read back the pixels.
505 if (mCanvasElement
) {
510 hasAlpha
= !mData
.mIsOpaque
;
511 isAlphaPremult
= mData
.mIsAlphaPremult
;
512 originPos
= mData
.mOriginPos
;
513 canvasElement
= mCanvasElement
;
516 MakeRefPtr
<SnapshotWorkerRunnable
>(mWorkerRef
->Private(), this);
517 workerRunnable
->Dispatch();
521 if (workerRunnable
) {
522 // We transferred to a DOM worker, so we need to do the readback on the
523 // owning thread and wait for the result.
524 surface
= workerRunnable
->Wait(
525 StaticPrefs::gfx_offscreencanvas_snapshot_timeout_ms());
526 } else if (canvasElement
) {
527 // If we have a context, it is owned by the main thread.
528 const auto* offscreenCanvas
= canvasElement
->GetOffscreenCanvas();
529 if (nsICanvasRenderingContextInternal
* context
=
530 offscreenCanvas
->GetContext()) {
532 context
->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
536 return TransformSurface(surface
, hasAlpha
, isAlphaPremult
, originPos
);
539 already_AddRefed
<layers::Image
> OffscreenCanvasDisplayHelper::GetAsImage() {
540 MOZ_ASSERT(NS_IsMainThread());
542 RefPtr
<gfx::SourceSurface
> surface
= GetSurfaceSnapshot();
546 return MakeAndAddRef
<layers::SourceSurfaceImage
>(surface
);
549 UniquePtr
<uint8_t[]> OffscreenCanvasDisplayHelper::GetImageBuffer(
550 int32_t* aOutFormat
, gfx::IntSize
* aOutImageSize
) {
551 RefPtr
<gfx::SourceSurface
> surface
= GetSurfaceSnapshot();
556 RefPtr
<gfx::DataSourceSurface
> dataSurface
= surface
->GetDataSurface();
561 *aOutFormat
= imgIEncoder::INPUT_FORMAT_HOSTARGB
;
562 *aOutImageSize
= dataSurface
->GetSize();
564 UniquePtr
<uint8_t[]> imageBuffer
= gfx::SurfaceToPackedBGRA(dataSurface
);
569 bool resistFingerprinting
;
570 nsICookieJarSettings
* cookieJarSettings
= nullptr;
572 MutexAutoLock
lock(mMutex
);
573 if (mCanvasElement
) {
574 Document
* doc
= mCanvasElement
->OwnerDoc();
575 resistFingerprinting
=
576 doc
->ShouldResistFingerprinting(RFPTarget::CanvasRandomization
);
577 if (resistFingerprinting
) {
578 cookieJarSettings
= doc
->CookieJarSettings();
581 resistFingerprinting
= nsContentUtils::ShouldResistFingerprinting(
582 "Fallback", RFPTarget::CanvasRandomization
);
586 if (resistFingerprinting
) {
587 nsRFPService::RandomizePixels(
588 cookieJarSettings
, imageBuffer
.get(), dataSurface
->GetSize().width
,
589 dataSurface
->GetSize().height
,
590 dataSurface
->GetSize().width
* dataSurface
->GetSize().height
* 4,
591 gfx::SurfaceFormat::A8R8G8B8_UINT32
);
596 } // namespace mozilla::dom