image/Image.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "Image.h"
   7
   8 #include "imgRequest.h"
   9 #include "Layers.h"  // for LayerManager
  10 #include "nsIObserverService.h"
  11 #include "nsRefreshDriver.h"
  12 #include "nsContentUtils.h"
  13 #include "mozilla/gfx/Point.h"
  14 #include "mozilla/gfx/Rect.h"
  15 #include "mozilla/Services.h"
  16 #include "mozilla/SizeOfState.h"
  17 #include "mozilla/TimeStamp.h"
  18 #include "mozilla/Tuple.h"  // for Tie
  19 #include "mozilla/layers/SharedSurfacesChild.h"
  20 #include "SourceSurfaceBlobImage.h"
  21
  22 namespace mozilla {
  23 namespace image {
  24
  25 ///////////////////////////////////////////////////////////////////////////////
  26 // Memory Reporting
  27 ///////////////////////////////////////////////////////////////////////////////
  28
  29 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest,
  30                                        SizeOfState& aState, bool aIsUsed)
  31     : mProgress(UINT32_MAX),
  32       mType(UINT16_MAX),
  33       mIsUsed(aIsUsed),
  34       mHasError(false),
  35       mValidating(false) {
  36   MOZ_ASSERT(aRequest);
  37
  38   // We don't have the image object yet, but we can get some information.
  39   nsCOMPtr<nsIURI> imageURL;
  40   nsresult rv = aRequest->GetURI(getter_AddRefs(imageURL));
  41   if (NS_SUCCEEDED(rv) && imageURL) {
  42     imageURL->GetSpec(mURI);
  43   }
  44
  45   mType = imgIContainer::TYPE_REQUEST;
  46   mHasError = NS_FAILED(aRequest->GetImageErrorCode());
  47   mValidating = !!aRequest->GetValidator();
  48
  49   RefPtr<ProgressTracker> tracker = aRequest->GetProgressTracker();
  50   if (tracker) {
  51     mProgress = tracker->GetProgress();
  52   }
  53 }
  54
  55 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest, Image* aImage,
  56                                        SizeOfState& aState, bool aIsUsed)
  57     : mProgress(UINT32_MAX),
  58       mType(UINT16_MAX),
  59       mIsUsed(aIsUsed),
  60       mHasError(false),
  61       mValidating(false) {
  62   MOZ_ASSERT(aRequest);
  63   MOZ_ASSERT(aImage);
  64
  65   // Extract metadata about the image.
  66   nsCOMPtr<nsIURI> imageURL(aImage->GetURI());
  67   if (imageURL) {
  68     imageURL->GetSpec(mURI);
  69   }
  70
  71   int32_t width = 0;
  72   int32_t height = 0;
  73   aImage->GetWidth(&width);
  74   aImage->GetHeight(&height);
  75   mIntrinsicSize.SizeTo(width, height);
  76
  77   mType = aImage->GetType();
  78   mHasError = aImage->HasError();
  79   mValidating = !!aRequest->GetValidator();
  80
  81   RefPtr<ProgressTracker> tracker = aImage->GetProgressTracker();
  82   if (tracker) {
  83     mProgress = tracker->GetProgress();
  84   }
  85
  86   // Populate memory counters for source and decoded data.
  87   mValues.SetSource(aImage->SizeOfSourceWithComputedFallback(aState));
  88   aImage->CollectSizeOfSurfaces(mSurfaces, aState.mMallocSizeOf);
  89
  90   // Compute totals.
  91   for (const SurfaceMemoryCounter& surfaceCounter : mSurfaces) {
  92     mValues += surfaceCounter.Values();
  93   }
  94 }
  95
  96 ///////////////////////////////////////////////////////////////////////////////
  97 // Image Base Types
  98 ///////////////////////////////////////////////////////////////////////////////
  99
 100 bool ImageResource::GetSpecTruncatedTo1k(nsCString& aSpec) const {
 101   static const size_t sMaxTruncatedLength = 1024;
 102
 103   mURI->GetSpec(aSpec);
 104   if (sMaxTruncatedLength >= aSpec.Length()) {
 105     return true;
 106   }
 107
 108   aSpec.Truncate(sMaxTruncatedLength);
 109   return false;
 110 }
 111
 112 void ImageResource::SetCurrentImage(layers::ImageContainer* aContainer,
 113                                     gfx::SourceSurface* aSurface,
 114                                     const Maybe<gfx::IntRect>& aDirtyRect) {
 115   MOZ_ASSERT(NS_IsMainThread());
 116   MOZ_ASSERT(aContainer);
 117
 118   if (!aSurface) {
 119     // The OS threw out some or all of our buffer. We'll need to wait for the
 120     // redecode (which was automatically triggered by GetFrame) to complete.
 121     return;
 122   }
 123
 124   // |image| holds a reference to a SourceSurface which in turn holds a lock on
 125   // the current frame's data buffer, ensuring that it doesn't get freed as
 126   // long as the layer system keeps this ImageContainer alive.
 127   RefPtr<layers::Image> image = new layers::SourceSurfaceImage(aSurface);
 128
 129   // We can share the producer ID with other containers because it is only
 130   // used internally to validate the frames given to a particular container
 131   // so that another object cannot add its own. Similarly the frame ID is
 132   // only used internally to ensure it is always increasing, and skipping
 133   // IDs from an individual container's perspective is acceptable.
 134   AutoTArray<layers::ImageContainer::NonOwningImage, 1> imageList;
 135   imageList.AppendElement(layers::ImageContainer::NonOwningImage(
 136       image, TimeStamp(), mLastFrameID++, mImageProducerID));
 137
 138   if (aDirtyRect) {
 139     aContainer->SetCurrentImagesInTransaction(imageList);
 140   } else {
 141     aContainer->SetCurrentImages(imageList);
 142   }
 143
 144   // If we are animated, then we should request that the image container be
 145   // treated as such, to avoid display list rebuilding to update frames for
 146   // WebRender.
 147   if (mProgressTracker->GetProgress() & FLAG_IS_ANIMATED) {
 148     if (aDirtyRect) {
 149       layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
 150                                                    aDirtyRect.ref());
 151     } else {
 152       gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), aSurface->GetSize());
 153       layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
 154                                                    dirtyRect);
 155     }
 156   }
 157 }
 158
 159 ImgDrawResult ImageResource::GetImageContainerImpl(
 160     layers::LayerManager* aManager, const gfx::IntSize& aSize,
 161     const Maybe<SVGImageContext>& aSVGContext,
 162     const Maybe<ImageIntRegion>& aRegion, uint32_t aFlags,
 163     layers::ImageContainer** aOutContainer) {
 164   MOZ_ASSERT(NS_IsMainThread());
 165   MOZ_ASSERT(aManager);
 166   MOZ_ASSERT((aFlags &
 167               ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST | FLAG_RECORD_BLOB |
 168                 FLAG_ASYNC_NOTIFY | FLAG_HIGH_QUALITY_SCALING)) == FLAG_NONE,
 169              "Unsupported flag passed to GetImageContainer");
 170
 171   ImgDrawResult drawResult;
 172   gfx::IntSize size;
 173   Tie(drawResult, size) = GetImageContainerSize(aManager, aSize, aFlags);
 174   if (drawResult != ImgDrawResult::SUCCESS) {
 175     return drawResult;
 176   }
 177
 178   MOZ_ASSERT(!size.IsEmpty());
 179
 180   if (mAnimationConsumers == 0) {
 181     SendOnUnlockedDraw(aFlags);
 182   }
 183
 184   uint32_t flags = (aFlags & ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST)) |
 185                    FLAG_ASYNC_NOTIFY;
 186   RefPtr<layers::ImageContainer> container;
 187   ImageContainerEntry* entry = nullptr;
 188   int i = mImageContainers.Length() - 1;
 189   for (; i >= 0; --i) {
 190     entry = &mImageContainers[i];
 191     if (size == entry->mSize && flags == entry->mFlags &&
 192         aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
 193       // Lack of a container is handled below.
 194       container = RefPtr<layers::ImageContainer>(entry->mContainer);
 195       break;
 196     } else if (entry->mContainer.IsDead()) {
 197       // Stop tracking if our weak pointer to the image container was freed.
 198       mImageContainers.RemoveElementAt(i);
 199     }
 200   }
 201
 202   if (container) {
 203     switch (entry->mLastDrawResult) {
 204       case ImgDrawResult::SUCCESS:
 205       case ImgDrawResult::BAD_IMAGE:
 206       case ImgDrawResult::BAD_ARGS:
 207       case ImgDrawResult::NOT_SUPPORTED:
 208         container.forget(aOutContainer);
 209         return entry->mLastDrawResult;
 210       case ImgDrawResult::NOT_READY:
 211       case ImgDrawResult::INCOMPLETE:
 212       case ImgDrawResult::TEMPORARY_ERROR:
 213         // Temporary conditions where we need to rerequest the frame to recover.
 214         break;
 215       case ImgDrawResult::WRONG_SIZE:
 216         // Unused by GetFrameInternal
 217       default:
 218         MOZ_ASSERT_UNREACHABLE("Unhandled ImgDrawResult type!");
 219         container.forget(aOutContainer);
 220         return entry->mLastDrawResult;
 221     }
 222   }
 223
 224   AutoProfilerImagePaintMarker PROFILER_RAII(this);
 225 #ifdef DEBUG
 226   NotifyDrawingObservers();
 227 #endif
 228
 229   gfx::IntSize bestSize;
 230   RefPtr<gfx::SourceSurface> surface;
 231   Tie(drawResult, bestSize, surface) = GetFrameInternal(
 232       size, aSVGContext, aRegion, FRAME_CURRENT, aFlags | FLAG_ASYNC_NOTIFY);
 233
 234   // The requested size might be refused by the surface cache (i.e. due to
 235   // factor-of-2 mode). In that case we don't want to create an entry for this
 236   // specific size, but rather re-use the entry for the substituted size.
 237   if (bestSize != size) {
 238     MOZ_ASSERT(!bestSize.IsEmpty());
 239
 240     // We can only remove the entry if we no longer have a container, because if
 241     // there are strong references to it remaining, we need to still update it
 242     // in UpdateImageContainer.
 243     if (i >= 0 && !container) {
 244       mImageContainers.RemoveElementAt(i);
 245     }
 246
 247     // Forget about the stale container, if any. This lets the entry creation
 248     // logic do its job below, if it turns out there is no existing best entry
 249     // or the best entry doesn't have a container.
 250     container = nullptr;
 251
 252     // We need to do the entry search again for the new size. We skip pruning
 253     // because we did this above once already, but ImageContainer is threadsafe,
 254     // so there is a remote possibility it got freed.
 255     i = mImageContainers.Length() - 1;
 256     for (; i >= 0; --i) {
 257       entry = &mImageContainers[i];
 258       if (bestSize == entry->mSize && flags == entry->mFlags &&
 259           aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
 260         container = RefPtr<layers::ImageContainer>(entry->mContainer);
 261         if (container) {
 262           switch (entry->mLastDrawResult) {
 263             case ImgDrawResult::SUCCESS:
 264             case ImgDrawResult::BAD_IMAGE:
 265             case ImgDrawResult::BAD_ARGS:
 266             case ImgDrawResult::NOT_SUPPORTED:
 267               container.forget(aOutContainer);
 268               return entry->mLastDrawResult;
 269             case ImgDrawResult::NOT_READY:
 270             case ImgDrawResult::INCOMPLETE:
 271             case ImgDrawResult::TEMPORARY_ERROR:
 272               // Temporary conditions where we need to rerequest the frame to
 273               // recover. We have already done so!
 274               break;
 275             case ImgDrawResult::WRONG_SIZE:
 276               // Unused by GetFrameInternal
 277             default:
 278               MOZ_ASSERT_UNREACHABLE("Unhandled DrawResult type!");
 279               container.forget(aOutContainer);
 280               return entry->mLastDrawResult;
 281           }
 282         }
 283         break;
 284       }
 285     }
 286   }
 287
 288   if (!container) {
 289     // We need a new ImageContainer, so create one.
 290     container = layers::LayerManager::CreateImageContainer();
 291
 292     if (i >= 0) {
 293       entry->mContainer = container;
 294     } else {
 295       entry = mImageContainers.AppendElement(ImageContainerEntry(
 296           bestSize, aSVGContext, aRegion, container.get(), flags));
 297     }
 298   }
 299
 300   SetCurrentImage(container, surface, Nothing());
 301   entry->mLastDrawResult = drawResult;
 302   container.forget(aOutContainer);
 303   return drawResult;
 304 }
 305
 306 bool ImageResource::UpdateImageContainer(
 307     const Maybe<gfx::IntRect>& aDirtyRect) {
 308   MOZ_ASSERT(NS_IsMainThread());
 309
 310   for (int i = mImageContainers.Length() - 1; i >= 0; --i) {
 311     ImageContainerEntry& entry = mImageContainers[i];
 312     RefPtr<layers::ImageContainer> container(entry.mContainer);
 313     if (container) {
 314       // Blob recordings should just be marked as dirty. We will regenerate the
 315       // recording when the display list update comes around.
 316       if (entry.mFlags & FLAG_RECORD_BLOB) {
 317         AutoTArray<layers::ImageContainer::OwningImage, 1> images;
 318         container->GetCurrentImages(&images);
 319         if (images.IsEmpty()) {
 320           MOZ_ASSERT_UNREACHABLE("Empty container!");
 321           continue;
 322         }
 323
 324         RefPtr<gfx::SourceSurface> surface =
 325             images[0].mImage->GetAsSourceSurface();
 326         if (!surface || surface->GetType() != gfx::SurfaceType::BLOB_IMAGE) {
 327           MOZ_ASSERT_UNREACHABLE("No/wrong surface in container!");
 328           continue;
 329         }
 330
 331         static_cast<SourceSurfaceBlobImage*>(surface.get())->MarkDirty();
 332         continue;
 333       }
 334
 335       gfx::IntSize bestSize;
 336       RefPtr<gfx::SourceSurface> surface;
 337       Tie(entry.mLastDrawResult, bestSize, surface) =
 338           GetFrameInternal(entry.mSize, entry.mSVGContext, entry.mRegion,
 339                            FRAME_CURRENT, entry.mFlags);
 340
 341       // It is possible that this is a factor-of-2 substitution. Since we
 342       // managed to convert the weak reference into a strong reference, that
 343       // means that an imagelib user still is holding onto the container. thus
 344       // we cannot consolidate and must keep updating the duplicate container.
 345       if (aDirtyRect) {
 346         SetCurrentImage(container, surface, aDirtyRect);
 347       } else {
 348         gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), bestSize);
 349         SetCurrentImage(container, surface, Some(dirtyRect));
 350       }
 351     } else {
 352       // Stop tracking if our weak pointer to the image container was freed.
 353       mImageContainers.RemoveElementAt(i);
 354     }
 355   }
 356
 357   return !mImageContainers.IsEmpty();
 358 }
 359
 360 void ImageResource::ReleaseImageContainer() {
 361   MOZ_ASSERT(NS_IsMainThread());
 362   mImageContainers.Clear();
 363 }
 364
 365 // Constructor
 366 ImageResource::ImageResource(nsIURI* aURI)
 367     : mURI(aURI),
 368       mInnerWindowId(0),
 369       mAnimationConsumers(0),
 370       mAnimationMode(kNormalAnimMode),
 371       mInitialized(false),
 372       mAnimating(false),
 373       mError(false),
 374       mImageProducerID(layers::ImageContainer::AllocateProducerID()),
 375       mLastFrameID(0) {}
 376
 377 ImageResource::~ImageResource() {
 378   // Ask our ProgressTracker to drop its weak reference to us.
 379   mProgressTracker->ResetImage();
 380 }
 381
 382 void ImageResource::IncrementAnimationConsumers() {
 383   MOZ_ASSERT(NS_IsMainThread(),
 384              "Main thread only to encourage serialization "
 385              "with DecrementAnimationConsumers");
 386   mAnimationConsumers++;
 387 }
 388
 389 void ImageResource::DecrementAnimationConsumers() {
 390   MOZ_ASSERT(NS_IsMainThread(),
 391              "Main thread only to encourage serialization "
 392              "with IncrementAnimationConsumers");
 393   MOZ_ASSERT(mAnimationConsumers >= 1, "Invalid no. of animation consumers!");
 394   mAnimationConsumers--;
 395 }
 396
 397 nsresult ImageResource::GetAnimationModeInternal(uint16_t* aAnimationMode) {
 398   if (mError) {
 399     return NS_ERROR_FAILURE;
 400   }
 401
 402   NS_ENSURE_ARG_POINTER(aAnimationMode);
 403
 404   *aAnimationMode = mAnimationMode;
 405   return NS_OK;
 406 }
 407
 408 nsresult ImageResource::SetAnimationModeInternal(uint16_t aAnimationMode) {
 409   if (mError) {
 410     return NS_ERROR_FAILURE;
 411   }
 412
 413   NS_ASSERTION(aAnimationMode == kNormalAnimMode ||
 414                    aAnimationMode == kDontAnimMode ||
 415                    aAnimationMode == kLoopOnceAnimMode,
 416                "Wrong Animation Mode is being set!");
 417
 418   mAnimationMode = aAnimationMode;
 419
 420   return NS_OK;
 421 }
 422
 423 bool ImageResource::HadRecentRefresh(const TimeStamp& aTime) {
 424   // Our threshold for "recent" is 1/2 of the default refresh-driver interval.
 425   // This ensures that we allow for frame rates at least as fast as the
 426   // refresh driver's default rate.
 427   static TimeDuration recentThreshold =
 428       TimeDuration::FromMilliseconds(nsRefreshDriver::DefaultInterval() / 2.0);
 429
 430   if (!mLastRefreshTime.IsNull() &&
 431       aTime - mLastRefreshTime < recentThreshold) {
 432     return true;
 433   }
 434
 435   // else, we can proceed with a refresh.
 436   // But first, update our last refresh time:
 437   mLastRefreshTime = aTime;
 438   return false;
 439 }
 440
 441 void ImageResource::EvaluateAnimation() {
 442   if (!mAnimating && ShouldAnimate()) {
 443     nsresult rv = StartAnimation();
 444     mAnimating = NS_SUCCEEDED(rv);
 445   } else if (mAnimating && !ShouldAnimate()) {
 446     StopAnimation();
 447   }
 448 }
 449
 450 void ImageResource::SendOnUnlockedDraw(uint32_t aFlags) {
 451   if (!mProgressTracker) {
 452     return;
 453   }
 454
 455   if (!(aFlags & FLAG_ASYNC_NOTIFY)) {
 456     mProgressTracker->OnUnlockedDraw();
 457   } else {
 458     NotNull<RefPtr<ImageResource>> image = WrapNotNull(this);
 459     nsCOMPtr<nsIEventTarget> eventTarget = mProgressTracker->GetEventTarget();
 460     nsCOMPtr<nsIRunnable> ev = NS_NewRunnableFunction(
 461         "image::ImageResource::SendOnUnlockedDraw", [=]() -> void {
 462           RefPtr<ProgressTracker> tracker = image->GetProgressTracker();
 463           if (tracker) {
 464             tracker->OnUnlockedDraw();
 465           }
 466         });
 467     eventTarget->Dispatch(CreateMediumHighRunnable(ev.forget()),
 468                           NS_DISPATCH_NORMAL);
 469   }
 470 }
 471
 472 #ifdef DEBUG
 473 void ImageResource::NotifyDrawingObservers() {
 474   if (!mURI || !NS_IsMainThread()) {
 475     return;
 476   }
 477
 478   if (!mURI->SchemeIs("resource") && !mURI->SchemeIs("chrome")) {
 479     return;
 480   }
 481
 482   // Record the image drawing for startup performance testing.
 483   nsCOMPtr<nsIURI> uri = mURI;
 484   nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
 485       "image::ImageResource::NotifyDrawingObservers", [uri]() {
 486         nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
 487         NS_WARNING_ASSERTION(obs, "Can't get an observer service handle");
 488         if (obs) {
 489           nsAutoCString spec;
 490           uri->GetSpec(spec);
 491           obs->NotifyObservers(nullptr, "image-drawing",
 492                                NS_ConvertUTF8toUTF16(spec).get());
 493         }
 494       }));
 495 }
 496 #endif
 497
 498 }  // namespace image
 499 }  // namespace mozilla