Bug 1892041 - Part 1: Update test262 features. r=spidermonkey-reviewers,dminor
[gecko.git] / dom / canvas / OffscreenCanvasDisplayHelper.cpp
blob2a2d37dbbbe809d760661a5a346b84a6c2089d9f
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "OffscreenCanvasDisplayHelper.h"
8 #include "mozilla/dom/Document.h"
9 #include "mozilla/dom/WorkerRef.h"
10 #include "mozilla/dom/WorkerRunnable.h"
11 #include "mozilla/gfx/2D.h"
12 #include "mozilla/gfx/CanvasManagerChild.h"
13 #include "mozilla/gfx/DataSurfaceHelpers.h"
14 #include "mozilla/gfx/Swizzle.h"
15 #include "mozilla/layers/ImageBridgeChild.h"
16 #include "mozilla/layers/PersistentBufferProvider.h"
17 #include "mozilla/layers/TextureClientSharedSurface.h"
18 #include "mozilla/layers/TextureWrapperImage.h"
19 #include "mozilla/StaticPrefs_gfx.h"
20 #include "mozilla/SVGObserverUtils.h"
21 #include "nsICanvasRenderingContextInternal.h"
22 #include "nsRFPService.h"
24 namespace mozilla::dom {
26 OffscreenCanvasDisplayHelper::OffscreenCanvasDisplayHelper(
27 HTMLCanvasElement* aCanvasElement, uint32_t aWidth, uint32_t aHeight)
28 : mMutex("mozilla::dom::OffscreenCanvasDisplayHelper"),
29 mCanvasElement(aCanvasElement),
30 mImageProducerID(layers::ImageContainer::AllocateProducerID()) {
31 mData.mSize.width = aWidth;
32 mData.mSize.height = aHeight;
35 OffscreenCanvasDisplayHelper::~OffscreenCanvasDisplayHelper() = default;
37 void OffscreenCanvasDisplayHelper::DestroyElement() {
38 MOZ_ASSERT(NS_IsMainThread());
40 MutexAutoLock lock(mMutex);
41 if (mImageContainer) {
42 mImageContainer->ClearAllImages();
43 mImageContainer = nullptr;
45 mFrontBufferSurface = nullptr;
46 mCanvasElement = nullptr;
49 void OffscreenCanvasDisplayHelper::DestroyCanvas() {
50 if (auto* cm = gfx::CanvasManagerChild::Get()) {
51 cm->EndCanvasTransaction();
54 MutexAutoLock lock(mMutex);
55 if (mImageContainer) {
56 mImageContainer->ClearAllImages();
57 mImageContainer = nullptr;
59 mFrontBufferSurface = nullptr;
60 mOffscreenCanvas = nullptr;
61 mWorkerRef = nullptr;
64 bool OffscreenCanvasDisplayHelper::CanElementCaptureStream() const {
65 MutexAutoLock lock(mMutex);
66 return !!mWorkerRef;
69 bool OffscreenCanvasDisplayHelper::UsingElementCaptureStream() const {
70 MutexAutoLock lock(mMutex);
72 if (NS_WARN_IF(!NS_IsMainThread())) {
73 MOZ_ASSERT_UNREACHABLE("Should not call off main-thread!");
74 return !!mCanvasElement;
77 return mCanvasElement && mCanvasElement->UsingCaptureStream();
80 CanvasContextType OffscreenCanvasDisplayHelper::GetContextType() const {
81 MutexAutoLock lock(mMutex);
82 return mType;
85 RefPtr<layers::ImageContainer> OffscreenCanvasDisplayHelper::GetImageContainer()
86 const {
87 MutexAutoLock lock(mMutex);
88 return mImageContainer;
91 void OffscreenCanvasDisplayHelper::UpdateContext(
92 OffscreenCanvas* aOffscreenCanvas, RefPtr<ThreadSafeWorkerRef>&& aWorkerRef,
93 CanvasContextType aType, const Maybe<int32_t>& aChildId) {
94 RefPtr<layers::ImageContainer> imageContainer =
95 MakeRefPtr<layers::ImageContainer>(layers::ImageContainer::ASYNCHRONOUS);
97 MutexAutoLock lock(mMutex);
99 mOffscreenCanvas = aOffscreenCanvas;
100 mWorkerRef = std::move(aWorkerRef);
101 mType = aType;
102 mContextChildId = aChildId;
103 mImageContainer = std::move(imageContainer);
105 if (aChildId) {
106 mContextManagerId = Some(gfx::CanvasManagerChild::Get()->Id());
107 } else {
108 mContextManagerId.reset();
111 MaybeQueueInvalidateElement();
114 void OffscreenCanvasDisplayHelper::FlushForDisplay() {
115 MOZ_ASSERT(NS_IsMainThread());
117 MutexAutoLock lock(mMutex);
119 // Without an OffscreenCanvas object bound to us, we either have not drawn
120 // using the canvas at all, or we have already destroyed it.
121 if (!mOffscreenCanvas) {
122 return;
125 // We assign/destroy the worker ref at the same time as the OffscreenCanvas
126 // ref, so we can only have the canvas without a worker ref if it exists on
127 // the main thread.
128 if (!mWorkerRef) {
129 // We queue to ensure that we have the same asynchronous update behaviour
130 // for a main thread and a worker based OffscreenCanvas.
131 mOffscreenCanvas->QueueCommitToCompositor();
132 return;
135 class FlushWorkerRunnable final : public WorkerRunnable {
136 public:
137 FlushWorkerRunnable(WorkerPrivate* aWorkerPrivate,
138 OffscreenCanvasDisplayHelper* aDisplayHelper)
139 : WorkerRunnable(aWorkerPrivate, "FlushWorkerRunnable"),
140 mDisplayHelper(aDisplayHelper) {}
142 bool WorkerRun(JSContext*, WorkerPrivate*) override {
143 // The OffscreenCanvas can only be freed on the worker thread, so we
144 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
145 // destructor. We just need to make sure we don't call into
146 // OffscreenCanvas while holding the lock since it calls back into the
147 // OffscreenCanvasDisplayHelper.
148 RefPtr<OffscreenCanvas> canvas;
150 MutexAutoLock lock(mDisplayHelper->mMutex);
151 canvas = mDisplayHelper->mOffscreenCanvas;
154 if (canvas) {
155 canvas->CommitFrameToCompositor();
157 return true;
160 private:
161 RefPtr<OffscreenCanvasDisplayHelper> mDisplayHelper;
164 // Otherwise we are calling from the main thread during painting to a canvas
165 // on a worker thread.
166 auto task = MakeRefPtr<FlushWorkerRunnable>(mWorkerRef->Private(), this);
167 task->Dispatch();
170 bool OffscreenCanvasDisplayHelper::CommitFrameToCompositor(
171 nsICanvasRenderingContextInternal* aContext,
172 layers::TextureType aTextureType,
173 const Maybe<OffscreenCanvasDisplayData>& aData) {
174 auto endTransaction = MakeScopeExit([&]() {
175 if (auto* cm = gfx::CanvasManagerChild::Get()) {
176 cm->EndCanvasTransaction();
180 MutexAutoLock lock(mMutex);
182 gfx::SurfaceFormat format = gfx::SurfaceFormat::B8G8R8A8;
183 layers::TextureFlags flags = layers::TextureFlags::IMMUTABLE;
185 if (!mCanvasElement) {
186 // Our weak reference to the canvas element has been cleared, so we cannot
187 // present directly anymore.
188 return false;
191 if (aData) {
192 mData = aData.ref();
193 MaybeQueueInvalidateElement();
196 if (mData.mOwnerId.isSome()) {
197 // No need to update the ImageContainer as the presentation itself is
198 // handled in the compositor process.
199 return true;
202 if (!mImageContainer) {
203 return false;
206 if (mData.mIsOpaque) {
207 flags |= layers::TextureFlags::IS_OPAQUE;
208 format = gfx::SurfaceFormat::B8G8R8X8;
209 } else if (!mData.mIsAlphaPremult) {
210 flags |= layers::TextureFlags::NON_PREMULTIPLIED;
213 switch (mData.mOriginPos) {
214 case gl::OriginPos::BottomLeft:
215 flags |= layers::TextureFlags::ORIGIN_BOTTOM_LEFT;
216 break;
217 case gl::OriginPos::TopLeft:
218 break;
219 default:
220 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
221 break;
224 auto imageBridge = layers::ImageBridgeChild::GetSingleton();
225 if (!imageBridge) {
226 return false;
229 bool paintCallbacks = mData.mDoPaintCallbacks;
230 bool hasRemoteTextureDesc = false;
231 RefPtr<layers::Image> image;
232 RefPtr<layers::TextureClient> texture;
233 RefPtr<gfx::SourceSurface> surface;
234 Maybe<layers::SurfaceDescriptor> desc;
235 RefPtr<layers::FwdTransactionTracker> tracker;
238 MutexAutoUnlock unlock(mMutex);
239 if (paintCallbacks) {
240 aContext->OnBeforePaintTransaction();
243 desc = aContext->PresentFrontBuffer(nullptr, aTextureType);
244 if (desc) {
245 hasRemoteTextureDesc =
246 desc->type() ==
247 layers::SurfaceDescriptor::TSurfaceDescriptorRemoteTexture;
248 if (hasRemoteTextureDesc) {
249 tracker = aContext->UseCompositableForwarder(imageBridge);
250 if (tracker) {
251 flags |= layers::TextureFlags::WAIT_FOR_REMOTE_TEXTURE_OWNER;
254 } else {
255 if (layers::PersistentBufferProvider* provider =
256 aContext->GetBufferProvider()) {
257 texture = provider->GetTextureClient();
260 if (!texture) {
261 surface =
262 aContext->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
263 if (surface && surface->GetType() == gfx::SurfaceType::WEBGL) {
264 // Ensure we can map in the surface. If we get a SourceSurfaceWebgl
265 // surface, then it may not be backed by raw pixels yet. We need to
266 // map it on the owning thread rather than the ImageBridge thread.
267 gfx::DataSourceSurface::ScopedMap map(
268 static_cast<gfx::DataSourceSurface*>(surface.get()),
269 gfx::DataSourceSurface::READ);
270 if (!map.IsMapped()) {
271 surface = nullptr;
277 if (paintCallbacks) {
278 aContext->OnDidPaintTransaction();
282 // We save any current surface because we might need it in GetSnapshot. If we
283 // are on a worker thread and not WebGL, then this will be the only way we can
284 // access the pixel data on the main thread.
285 mFrontBufferSurface = surface;
287 // We do not use the ImageContainer plumbing with remote textures, so if we
288 // have that, we can just early return here.
289 if (hasRemoteTextureDesc) {
290 const auto& textureDesc = desc->get_SurfaceDescriptorRemoteTexture();
291 imageBridge->UpdateCompositable(mImageContainer, textureDesc.textureId(),
292 textureDesc.ownerId(), mData.mSize, flags,
293 tracker);
294 return true;
297 if (surface) {
298 auto surfaceImage = MakeRefPtr<layers::SourceSurfaceImage>(surface);
299 surfaceImage->SetTextureFlags(flags);
300 image = surfaceImage;
301 } else {
302 if (desc && !texture) {
303 texture = layers::SharedSurfaceTextureData::CreateTextureClient(
304 *desc, format, mData.mSize, flags, imageBridge);
306 if (texture) {
307 image = new layers::TextureWrapperImage(
308 texture, gfx::IntRect(gfx::IntPoint(0, 0), texture->GetSize()));
312 if (image) {
313 AutoTArray<layers::ImageContainer::NonOwningImage, 1> imageList;
314 imageList.AppendElement(layers::ImageContainer::NonOwningImage(
315 image, TimeStamp(), mLastFrameID++, mImageProducerID));
316 mImageContainer->SetCurrentImages(imageList);
317 } else {
318 mImageContainer->ClearAllImages();
321 return true;
324 void OffscreenCanvasDisplayHelper::MaybeQueueInvalidateElement() {
325 if (!mPendingInvalidate) {
326 mPendingInvalidate = true;
327 NS_DispatchToMainThread(NS_NewRunnableFunction(
328 "OffscreenCanvasDisplayHelper::InvalidateElement",
329 [self = RefPtr{this}] { self->InvalidateElement(); }));
333 void OffscreenCanvasDisplayHelper::InvalidateElement() {
334 MOZ_ASSERT(NS_IsMainThread());
336 HTMLCanvasElement* canvasElement;
337 gfx::IntSize size;
340 MutexAutoLock lock(mMutex);
341 MOZ_ASSERT(mPendingInvalidate);
342 mPendingInvalidate = false;
343 canvasElement = mCanvasElement;
344 size = mData.mSize;
347 if (canvasElement) {
348 SVGObserverUtils::InvalidateDirectRenderingObservers(canvasElement);
349 canvasElement->InvalidateCanvasPlaceholder(size.width, size.height);
350 canvasElement->InvalidateCanvasContent(nullptr);
354 already_AddRefed<gfx::SourceSurface>
355 OffscreenCanvasDisplayHelper::TransformSurface(gfx::SourceSurface* aSurface,
356 bool aHasAlpha,
357 bool aIsAlphaPremult,
358 gl::OriginPos aOriginPos) const {
359 if (!aSurface) {
360 return nullptr;
363 if (aOriginPos == gl::OriginPos::TopLeft && (!aHasAlpha || aIsAlphaPremult)) {
364 // If we don't need to y-flip, and it is either opaque or premultiplied,
365 // we can just return the same surface.
366 return do_AddRef(aSurface);
369 // Otherwise we need to copy and apply the necessary transformations.
370 RefPtr<gfx::DataSourceSurface> srcSurface = aSurface->GetDataSurface();
371 if (!srcSurface) {
372 return nullptr;
375 const auto size = srcSurface->GetSize();
376 const auto format = srcSurface->GetFormat();
378 RefPtr<gfx::DataSourceSurface> dstSurface =
379 gfx::Factory::CreateDataSourceSurface(size, format, /* aZero */ false);
380 if (!dstSurface) {
381 return nullptr;
384 gfx::DataSourceSurface::ScopedMap srcMap(srcSurface,
385 gfx::DataSourceSurface::READ);
386 gfx::DataSourceSurface::ScopedMap dstMap(dstSurface,
387 gfx::DataSourceSurface::WRITE);
388 if (!srcMap.IsMapped() || !dstMap.IsMapped()) {
389 return nullptr;
392 bool success;
393 switch (aOriginPos) {
394 case gl::OriginPos::BottomLeft:
395 if (aHasAlpha && !aIsAlphaPremult) {
396 success = gfx::PremultiplyYFlipData(
397 srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(),
398 dstMap.GetStride(), format, size);
399 } else {
400 success = gfx::SwizzleYFlipData(srcMap.GetData(), srcMap.GetStride(),
401 format, dstMap.GetData(),
402 dstMap.GetStride(), format, size);
404 break;
405 case gl::OriginPos::TopLeft:
406 if (aHasAlpha && !aIsAlphaPremult) {
407 success = gfx::PremultiplyData(srcMap.GetData(), srcMap.GetStride(),
408 format, dstMap.GetData(),
409 dstMap.GetStride(), format, size);
410 } else {
411 success = gfx::SwizzleData(srcMap.GetData(), srcMap.GetStride(), format,
412 dstMap.GetData(), dstMap.GetStride(), format,
413 size);
415 break;
416 default:
417 MOZ_ASSERT_UNREACHABLE("Unhandled origin position!");
418 success = false;
419 break;
422 if (!success) {
423 return nullptr;
426 return dstSurface.forget();
429 already_AddRefed<gfx::SourceSurface>
430 OffscreenCanvasDisplayHelper::GetSurfaceSnapshot() {
431 MOZ_ASSERT(NS_IsMainThread());
433 class SnapshotWorkerRunnable final : public MainThreadWorkerRunnable {
434 public:
435 SnapshotWorkerRunnable(WorkerPrivate* aWorkerPrivate,
436 OffscreenCanvasDisplayHelper* aDisplayHelper)
437 : MainThreadWorkerRunnable(aWorkerPrivate, "SnapshotWorkerRunnable"),
438 mMonitor("SnapshotWorkerRunnable::mMonitor"),
439 mDisplayHelper(aDisplayHelper) {}
441 bool WorkerRun(JSContext*, WorkerPrivate*) override {
442 // The OffscreenCanvas can only be freed on the worker thread, so we
443 // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its
444 // destructor. We just need to make sure we don't call into
445 // OffscreenCanvas while holding the lock since it calls back into the
446 // OffscreenCanvasDisplayHelper.
447 RefPtr<OffscreenCanvas> canvas;
449 MutexAutoLock lock(mDisplayHelper->mMutex);
450 canvas = mDisplayHelper->mOffscreenCanvas;
453 // Now that we are on the correct thread, we can extract the snapshot. If
454 // it is a Skia surface, perform a copy to threading issues.
455 RefPtr<gfx::SourceSurface> surface;
456 if (canvas) {
457 if (auto* context = canvas->GetContext()) {
458 surface =
459 context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
460 if (surface && surface->GetType() == gfx::SurfaceType::SKIA) {
461 surface = gfx::Factory::CopyDataSourceSurface(
462 static_cast<gfx::DataSourceSurface*>(surface.get()));
467 MonitorAutoLock lock(mMonitor);
468 mSurface = std::move(surface);
469 mComplete = true;
470 lock.NotifyAll();
471 return true;
474 already_AddRefed<gfx::SourceSurface> Wait(int32_t aTimeoutMs) {
475 MonitorAutoLock lock(mMonitor);
477 TimeDuration timeout = TimeDuration::FromMilliseconds(aTimeoutMs);
478 while (!mComplete) {
479 if (lock.Wait(timeout) == CVStatus::Timeout) {
480 return nullptr;
484 return mSurface.forget();
487 private:
488 Monitor mMonitor;
489 RefPtr<OffscreenCanvasDisplayHelper> mDisplayHelper;
490 RefPtr<gfx::SourceSurface> mSurface MOZ_GUARDED_BY(mMonitor);
491 bool mComplete MOZ_GUARDED_BY(mMonitor) = false;
494 bool hasAlpha;
495 bool isAlphaPremult;
496 gl::OriginPos originPos;
497 HTMLCanvasElement* canvasElement;
498 RefPtr<gfx::SourceSurface> surface;
499 RefPtr<SnapshotWorkerRunnable> workerRunnable;
502 MutexAutoLock lock(mMutex);
503 #ifdef MOZ_WIDGET_ANDROID
504 // On Android, we cannot both display a GL context and read back the pixels.
505 if (mCanvasElement) {
506 return nullptr;
508 #endif
510 hasAlpha = !mData.mIsOpaque;
511 isAlphaPremult = mData.mIsAlphaPremult;
512 originPos = mData.mOriginPos;
513 canvasElement = mCanvasElement;
514 if (mWorkerRef) {
515 workerRunnable =
516 MakeRefPtr<SnapshotWorkerRunnable>(mWorkerRef->Private(), this);
517 workerRunnable->Dispatch();
521 if (workerRunnable) {
522 // We transferred to a DOM worker, so we need to do the readback on the
523 // owning thread and wait for the result.
524 surface = workerRunnable->Wait(
525 StaticPrefs::gfx_offscreencanvas_snapshot_timeout_ms());
526 } else if (canvasElement) {
527 // If we have a context, it is owned by the main thread.
528 const auto* offscreenCanvas = canvasElement->GetOffscreenCanvas();
529 if (nsICanvasRenderingContextInternal* context =
530 offscreenCanvas->GetContext()) {
531 surface =
532 context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false);
536 return TransformSurface(surface, hasAlpha, isAlphaPremult, originPos);
539 already_AddRefed<layers::Image> OffscreenCanvasDisplayHelper::GetAsImage() {
540 MOZ_ASSERT(NS_IsMainThread());
542 RefPtr<gfx::SourceSurface> surface = GetSurfaceSnapshot();
543 if (!surface) {
544 return nullptr;
546 return MakeAndAddRef<layers::SourceSurfaceImage>(surface);
549 UniquePtr<uint8_t[]> OffscreenCanvasDisplayHelper::GetImageBuffer(
550 int32_t* aOutFormat, gfx::IntSize* aOutImageSize) {
551 RefPtr<gfx::SourceSurface> surface = GetSurfaceSnapshot();
552 if (!surface) {
553 return nullptr;
556 RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
557 if (!dataSurface) {
558 return nullptr;
561 *aOutFormat = imgIEncoder::INPUT_FORMAT_HOSTARGB;
562 *aOutImageSize = dataSurface->GetSize();
564 UniquePtr<uint8_t[]> imageBuffer = gfx::SurfaceToPackedBGRA(dataSurface);
565 if (!imageBuffer) {
566 return nullptr;
569 bool resistFingerprinting;
570 nsICookieJarSettings* cookieJarSettings = nullptr;
572 MutexAutoLock lock(mMutex);
573 if (mCanvasElement) {
574 Document* doc = mCanvasElement->OwnerDoc();
575 resistFingerprinting =
576 doc->ShouldResistFingerprinting(RFPTarget::CanvasRandomization);
577 if (resistFingerprinting) {
578 cookieJarSettings = doc->CookieJarSettings();
580 } else {
581 resistFingerprinting = nsContentUtils::ShouldResistFingerprinting(
582 "Fallback", RFPTarget::CanvasRandomization);
586 if (resistFingerprinting) {
587 nsRFPService::RandomizePixels(
588 cookieJarSettings, imageBuffer.get(), dataSurface->GetSize().width,
589 dataSurface->GetSize().height,
590 dataSurface->GetSize().width * dataSurface->GetSize().height * 4,
591 gfx::SurfaceFormat::A8R8G8B8_UINT32);
593 return imageBuffer;
596 } // namespace mozilla::dom