image/Image.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "Image.h"
   7
   8 #include "imgRequest.h"
   9 #include "Layers.h"  // for LayerManager
  10 #include "nsIObserverService.h"
  11 #include "nsRefreshDriver.h"
  12 #include "nsContentUtils.h"
  13 #include "mozilla/gfx/Point.h"
  14 #include "mozilla/gfx/Rect.h"
  15 #include "mozilla/gfx/SourceSurfaceRawData.h"
  16 #include "mozilla/Services.h"
  17 #include "mozilla/SizeOfState.h"
  18 #include "mozilla/TimeStamp.h"
  19 #include "mozilla/Tuple.h"  // for Tie
  20 #include "mozilla/layers/SharedSurfacesChild.h"
  21 #include "SourceSurfaceBlobImage.h"
  22
  23 namespace mozilla {
  24 namespace image {
  25
  26 ///////////////////////////////////////////////////////////////////////////////
  27 // Memory Reporting
  28 ///////////////////////////////////////////////////////////////////////////////
  29
  30 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest,
  31                                        SizeOfState& aState, bool aIsUsed)
  32     : mProgress(UINT32_MAX),
  33       mType(UINT16_MAX),
  34       mIsUsed(aIsUsed),
  35       mHasError(false),
  36       mValidating(false) {
  37   MOZ_ASSERT(aRequest);
  38
  39   // We don't have the image object yet, but we can get some information.
  40   nsCOMPtr<nsIURI> imageURL;
  41   nsresult rv = aRequest->GetURI(getter_AddRefs(imageURL));
  42   if (NS_SUCCEEDED(rv) && imageURL) {
  43     imageURL->GetSpec(mURI);
  44   }
  45
  46   mType = imgIContainer::TYPE_REQUEST;
  47   mHasError = NS_FAILED(aRequest->GetImageErrorCode());
  48   mValidating = !!aRequest->GetValidator();
  49
  50   RefPtr<ProgressTracker> tracker = aRequest->GetProgressTracker();
  51   if (tracker) {
  52     mProgress = tracker->GetProgress();
  53   }
  54 }
  55
  56 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest, Image* aImage,
  57                                        SizeOfState& aState, bool aIsUsed)
  58     : mProgress(UINT32_MAX),
  59       mType(UINT16_MAX),
  60       mIsUsed(aIsUsed),
  61       mHasError(false),
  62       mValidating(false) {
  63   MOZ_ASSERT(aRequest);
  64   MOZ_ASSERT(aImage);
  65
  66   // Extract metadata about the image.
  67   nsCOMPtr<nsIURI> imageURL(aImage->GetURI());
  68   if (imageURL) {
  69     imageURL->GetSpec(mURI);
  70   }
  71
  72   int32_t width = 0;
  73   int32_t height = 0;
  74   aImage->GetWidth(&width);
  75   aImage->GetHeight(&height);
  76   mIntrinsicSize.SizeTo(width, height);
  77
  78   mType = aImage->GetType();
  79   mHasError = aImage->HasError();
  80   mValidating = !!aRequest->GetValidator();
  81
  82   RefPtr<ProgressTracker> tracker = aImage->GetProgressTracker();
  83   if (tracker) {
  84     mProgress = tracker->GetProgress();
  85   }
  86
  87   // Populate memory counters for source and decoded data.
  88   mValues.SetSource(aImage->SizeOfSourceWithComputedFallback(aState));
  89   aImage->CollectSizeOfSurfaces(mSurfaces, aState.mMallocSizeOf);
  90
  91   // Compute totals.
  92   for (const SurfaceMemoryCounter& surfaceCounter : mSurfaces) {
  93     mValues += surfaceCounter.Values();
  94   }
  95 }
  96
  97 ///////////////////////////////////////////////////////////////////////////////
  98 // Image Base Types
  99 ///////////////////////////////////////////////////////////////////////////////
 100
 101 bool ImageResource::GetSpecTruncatedTo1k(nsCString& aSpec) const {
 102   static const size_t sMaxTruncatedLength = 1024;
 103
 104   mURI->GetSpec(aSpec);
 105   if (sMaxTruncatedLength >= aSpec.Length()) {
 106     return true;
 107   }
 108
 109   aSpec.Truncate(sMaxTruncatedLength);
 110   return false;
 111 }
 112
 113 void ImageResource::SetCurrentImage(layers::ImageContainer* aContainer,
 114                                     gfx::SourceSurface* aSurface,
 115                                     const Maybe<gfx::IntRect>& aDirtyRect) {
 116   MOZ_ASSERT(NS_IsMainThread());
 117   MOZ_ASSERT(aContainer);
 118
 119   if (!aSurface) {
 120     // The OS threw out some or all of our buffer. We'll need to wait for the
 121     // redecode (which was automatically triggered by GetFrame) to complete.
 122     return;
 123   }
 124
 125   // |image| holds a reference to a SourceSurface which in turn holds a lock on
 126   // the current frame's data buffer, ensuring that it doesn't get freed as
 127   // long as the layer system keeps this ImageContainer alive.
 128   RefPtr<layers::Image> image = new layers::SourceSurfaceImage(aSurface);
 129
 130   // We can share the producer ID with other containers because it is only
 131   // used internally to validate the frames given to a particular container
 132   // so that another object cannot add its own. Similarly the frame ID is
 133   // only used internally to ensure it is always increasing, and skipping
 134   // IDs from an individual container's perspective is acceptable.
 135   AutoTArray<layers::ImageContainer::NonOwningImage, 1> imageList;
 136   imageList.AppendElement(layers::ImageContainer::NonOwningImage(
 137       image, TimeStamp(), mLastFrameID++, mImageProducerID));
 138
 139   if (aDirtyRect) {
 140     aContainer->SetCurrentImagesInTransaction(imageList);
 141   } else {
 142     aContainer->SetCurrentImages(imageList);
 143   }
 144
 145   // If we are animated, then we should request that the image container be
 146   // treated as such, to avoid display list rebuilding to update frames for
 147   // WebRender.
 148   if (mProgressTracker->GetProgress() & FLAG_IS_ANIMATED) {
 149     if (aDirtyRect) {
 150       layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
 151                                                    aDirtyRect.ref());
 152     } else {
 153       gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), aSurface->GetSize());
 154       layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
 155                                                    dirtyRect);
 156     }
 157   }
 158 }
 159
 160 ImgDrawResult ImageResource::GetImageContainerImpl(
 161     WindowRenderer* aRenderer, const gfx::IntSize& aSize,
 162     const Maybe<SVGImageContext>& aSVGContext,
 163     const Maybe<ImageIntRegion>& aRegion, uint32_t aFlags,
 164     layers::ImageContainer** aOutContainer) {
 165   MOZ_ASSERT(NS_IsMainThread());
 166   MOZ_ASSERT(aRenderer);
 167   MOZ_ASSERT((aFlags &
 168               ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST | FLAG_RECORD_BLOB |
 169                 FLAG_ASYNC_NOTIFY | FLAG_HIGH_QUALITY_SCALING)) == FLAG_NONE,
 170              "Unsupported flag passed to GetImageContainer");
 171
 172   ImgDrawResult drawResult;
 173   gfx::IntSize size;
 174   Tie(drawResult, size) = GetImageContainerSize(aRenderer, aSize, aFlags);
 175   if (drawResult != ImgDrawResult::SUCCESS) {
 176     return drawResult;
 177   }
 178
 179   MOZ_ASSERT(!size.IsEmpty());
 180
 181   if (mAnimationConsumers == 0) {
 182     SendOnUnlockedDraw(aFlags);
 183   }
 184
 185   uint32_t flags = (aFlags & ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST)) |
 186                    FLAG_ASYNC_NOTIFY;
 187   RefPtr<layers::ImageContainer> container;
 188   ImageContainerEntry* entry = nullptr;
 189   int i = mImageContainers.Length() - 1;
 190   for (; i >= 0; --i) {
 191     entry = &mImageContainers[i];
 192     if (size == entry->mSize && flags == entry->mFlags &&
 193         aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
 194       // Lack of a container is handled below.
 195       container = RefPtr<layers::ImageContainer>(entry->mContainer);
 196       break;
 197     } else if (entry->mContainer.IsDead()) {
 198       // Stop tracking if our weak pointer to the image container was freed.
 199       mImageContainers.RemoveElementAt(i);
 200     }
 201   }
 202
 203   if (container) {
 204     switch (entry->mLastDrawResult) {
 205       case ImgDrawResult::SUCCESS:
 206       case ImgDrawResult::BAD_IMAGE:
 207       case ImgDrawResult::BAD_ARGS:
 208       case ImgDrawResult::NOT_SUPPORTED:
 209         container.forget(aOutContainer);
 210         return entry->mLastDrawResult;
 211       case ImgDrawResult::NOT_READY:
 212       case ImgDrawResult::INCOMPLETE:
 213       case ImgDrawResult::TEMPORARY_ERROR:
 214         // Temporary conditions where we need to rerequest the frame to recover.
 215         break;
 216       case ImgDrawResult::WRONG_SIZE:
 217         // Unused by GetFrameInternal
 218       default:
 219         MOZ_ASSERT_UNREACHABLE("Unhandled ImgDrawResult type!");
 220         container.forget(aOutContainer);
 221         return entry->mLastDrawResult;
 222     }
 223   }
 224
 225   AutoProfilerImagePaintMarker PROFILER_RAII(this);
 226 #ifdef DEBUG
 227   NotifyDrawingObservers();
 228 #endif
 229
 230   gfx::IntSize bestSize;
 231   RefPtr<gfx::SourceSurface> surface;
 232   Tie(drawResult, bestSize, surface) = GetFrameInternal(
 233       size, aSVGContext, aRegion, FRAME_CURRENT, aFlags | FLAG_ASYNC_NOTIFY);
 234
 235   // The requested size might be refused by the surface cache (i.e. due to
 236   // factor-of-2 mode). In that case we don't want to create an entry for this
 237   // specific size, but rather re-use the entry for the substituted size.
 238   if (bestSize != size) {
 239     MOZ_ASSERT(!bestSize.IsEmpty());
 240
 241     // We can only remove the entry if we no longer have a container, because if
 242     // there are strong references to it remaining, we need to still update it
 243     // in UpdateImageContainer.
 244     if (i >= 0 && !container) {
 245       mImageContainers.RemoveElementAt(i);
 246     }
 247
 248     // Forget about the stale container, if any. This lets the entry creation
 249     // logic do its job below, if it turns out there is no existing best entry
 250     // or the best entry doesn't have a container.
 251     container = nullptr;
 252
 253     // We need to do the entry search again for the new size. We skip pruning
 254     // because we did this above once already, but ImageContainer is threadsafe,
 255     // so there is a remote possibility it got freed.
 256     i = mImageContainers.Length() - 1;
 257     for (; i >= 0; --i) {
 258       entry = &mImageContainers[i];
 259       if (bestSize == entry->mSize && flags == entry->mFlags &&
 260           aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
 261         container = RefPtr<layers::ImageContainer>(entry->mContainer);
 262         if (container) {
 263           switch (entry->mLastDrawResult) {
 264             case ImgDrawResult::SUCCESS:
 265             case ImgDrawResult::BAD_IMAGE:
 266             case ImgDrawResult::BAD_ARGS:
 267             case ImgDrawResult::NOT_SUPPORTED:
 268               container.forget(aOutContainer);
 269               return entry->mLastDrawResult;
 270             case ImgDrawResult::NOT_READY:
 271             case ImgDrawResult::INCOMPLETE:
 272             case ImgDrawResult::TEMPORARY_ERROR:
 273               // Temporary conditions where we need to rerequest the frame to
 274               // recover. We have already done so!
 275               break;
 276             case ImgDrawResult::WRONG_SIZE:
 277               // Unused by GetFrameInternal
 278             default:
 279               MOZ_ASSERT_UNREACHABLE("Unhandled DrawResult type!");
 280               container.forget(aOutContainer);
 281               return entry->mLastDrawResult;
 282           }
 283         }
 284         break;
 285       }
 286     }
 287   }
 288
 289   if (!container) {
 290     // We need a new ImageContainer, so create one.
 291     container = layers::LayerManager::CreateImageContainer();
 292
 293     if (i >= 0) {
 294       entry->mContainer = container;
 295     } else {
 296       entry = mImageContainers.AppendElement(ImageContainerEntry(
 297           bestSize, aSVGContext, aRegion, container.get(), flags));
 298     }
 299   }
 300
 301   SetCurrentImage(container, surface, Nothing());
 302   entry->mLastDrawResult = drawResult;
 303   container.forget(aOutContainer);
 304   return drawResult;
 305 }
 306
 307 bool ImageResource::UpdateImageContainer(
 308     const Maybe<gfx::IntRect>& aDirtyRect) {
 309   MOZ_ASSERT(NS_IsMainThread());
 310
 311   for (int i = mImageContainers.Length() - 1; i >= 0; --i) {
 312     ImageContainerEntry& entry = mImageContainers[i];
 313     RefPtr<layers::ImageContainer> container(entry.mContainer);
 314     if (container) {
 315       // Blob recordings should just be marked as dirty. We will regenerate the
 316       // recording when the display list update comes around.
 317       if (entry.mFlags & FLAG_RECORD_BLOB) {
 318         AutoTArray<layers::ImageContainer::OwningImage, 1> images;
 319         container->GetCurrentImages(&images);
 320         if (images.IsEmpty()) {
 321           MOZ_ASSERT_UNREACHABLE("Empty container!");
 322           continue;
 323         }
 324
 325         RefPtr<gfx::SourceSurface> surface =
 326             images[0].mImage->GetAsSourceSurface();
 327         if (!surface || surface->GetType() != gfx::SurfaceType::BLOB_IMAGE) {
 328           MOZ_ASSERT_UNREACHABLE("No/wrong surface in container!");
 329           continue;
 330         }
 331
 332         static_cast<SourceSurfaceBlobImage*>(surface.get())->MarkDirty();
 333         continue;
 334       }
 335
 336       gfx::IntSize bestSize;
 337       RefPtr<gfx::SourceSurface> surface;
 338       Tie(entry.mLastDrawResult, bestSize, surface) =
 339           GetFrameInternal(entry.mSize, entry.mSVGContext, entry.mRegion,
 340                            FRAME_CURRENT, entry.mFlags);
 341
 342       // It is possible that this is a factor-of-2 substitution. Since we
 343       // managed to convert the weak reference into a strong reference, that
 344       // means that an imagelib user still is holding onto the container. thus
 345       // we cannot consolidate and must keep updating the duplicate container.
 346       if (aDirtyRect) {
 347         SetCurrentImage(container, surface, aDirtyRect);
 348       } else {
 349         gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), bestSize);
 350         SetCurrentImage(container, surface, Some(dirtyRect));
 351       }
 352     } else {
 353       // Stop tracking if our weak pointer to the image container was freed.
 354       mImageContainers.RemoveElementAt(i);
 355     }
 356   }
 357
 358   return !mImageContainers.IsEmpty();
 359 }
 360
 361 void ImageResource::CollectSizeOfSurfaces(
 362     nsTArray<SurfaceMemoryCounter>& aCounters,
 363     MallocSizeOf aMallocSizeOf) const {
 364   MOZ_ASSERT(NS_IsMainThread());
 365
 366   for (const auto& entry : mImageContainers) {
 367     RefPtr<layers::ImageContainer> container(entry.mContainer);
 368     if (!container) {
 369       continue;
 370     }
 371
 372     AutoTArray<layers::ImageContainer::OwningImage, 1> images;
 373     container->GetCurrentImages(&images);
 374     if (images.IsEmpty()) {
 375       continue;
 376     }
 377
 378     RefPtr<gfx::SourceSurface> surface = images[0].mImage->GetAsSourceSurface();
 379     if (!surface) {
 380       MOZ_ASSERT_UNREACHABLE("No surface in container!");
 381       continue;
 382     }
 383
 384     // The surface might be wrapping another.
 385     bool isMappedSurface = surface->GetType() == gfx::SurfaceType::DATA_MAPPED;
 386     const gfx::SourceSurface* actualSurface =
 387         isMappedSurface
 388             ? static_cast<gfx::SourceSurfaceMappedData*>(surface.get())
 389                   ->GetScopedSurface()
 390             : surface.get();
 391
 392     // Check if the surface is already in the report. Ignore if so.
 393     bool found = false;
 394     for (const auto& counter : aCounters) {
 395       if (counter.Surface() == actualSurface) {
 396         found = true;
 397         break;
 398       }
 399     }
 400     if (found) {
 401       continue;
 402     }
 403
 404     // The surface isn't in the report, so it isn't stored in SurfaceCache. We
 405     // need to add our own entry here so that it will be included in the memory
 406     // report.
 407     gfx::SourceSurface::SizeOfInfo info;
 408     surface->SizeOfExcludingThis(aMallocSizeOf, info);
 409
 410     uint32_t heapBytes = aMallocSizeOf(actualSurface);
 411     if (isMappedSurface) {
 412       heapBytes += aMallocSizeOf(surface.get());
 413     }
 414
 415     SurfaceKey key = ContainerSurfaceKey(surface->GetSize(), entry.mSVGContext,
 416                                          ToSurfaceFlags(entry.mFlags));
 417     SurfaceMemoryCounter counter(key, actualSurface, /* aIsLocked */ false,
 418                                  /* aCannotSubstitute */ false,
 419                                  /* aIsFactor2 */ false, /* aFinished */ true,
 420                                  SurfaceMemoryCounterType::CONTAINER);
 421
 422     counter.Values().SetDecodedHeap(info.mHeapBytes + heapBytes);
 423     counter.Values().SetDecodedNonHeap(info.mNonHeapBytes);
 424     counter.Values().SetDecodedUnknown(info.mUnknownBytes);
 425     counter.Values().SetExternalHandles(info.mExternalHandles);
 426     counter.Values().SetExternalId(info.mExternalId);
 427     counter.Values().SetSurfaceTypes(info.mTypes);
 428
 429     aCounters.AppendElement(counter);
 430   }
 431 }
 432
 433 void ImageResource::ReleaseImageContainer() {
 434   MOZ_ASSERT(NS_IsMainThread());
 435   mImageContainers.Clear();
 436 }
 437
 438 // Constructor
 439 ImageResource::ImageResource(nsIURI* aURI)
 440     : mURI(aURI),
 441       mInnerWindowId(0),
 442       mAnimationConsumers(0),
 443       mAnimationMode(kNormalAnimMode),
 444       mInitialized(false),
 445       mAnimating(false),
 446       mError(false),
 447       mImageProducerID(layers::ImageContainer::AllocateProducerID()),
 448       mLastFrameID(0) {}
 449
 450 ImageResource::~ImageResource() {
 451   // Ask our ProgressTracker to drop its weak reference to us.
 452   mProgressTracker->ResetImage();
 453 }
 454
 455 void ImageResource::IncrementAnimationConsumers() {
 456   MOZ_ASSERT(NS_IsMainThread(),
 457              "Main thread only to encourage serialization "
 458              "with DecrementAnimationConsumers");
 459   mAnimationConsumers++;
 460 }
 461
 462 void ImageResource::DecrementAnimationConsumers() {
 463   MOZ_ASSERT(NS_IsMainThread(),
 464              "Main thread only to encourage serialization "
 465              "with IncrementAnimationConsumers");
 466   MOZ_ASSERT(mAnimationConsumers >= 1, "Invalid no. of animation consumers!");
 467   mAnimationConsumers--;
 468 }
 469
 470 nsresult ImageResource::GetAnimationModeInternal(uint16_t* aAnimationMode) {
 471   if (mError) {
 472     return NS_ERROR_FAILURE;
 473   }
 474
 475   NS_ENSURE_ARG_POINTER(aAnimationMode);
 476
 477   *aAnimationMode = mAnimationMode;
 478   return NS_OK;
 479 }
 480
 481 nsresult ImageResource::SetAnimationModeInternal(uint16_t aAnimationMode) {
 482   if (mError) {
 483     return NS_ERROR_FAILURE;
 484   }
 485
 486   NS_ASSERTION(aAnimationMode == kNormalAnimMode ||
 487                    aAnimationMode == kDontAnimMode ||
 488                    aAnimationMode == kLoopOnceAnimMode,
 489                "Wrong Animation Mode is being set!");
 490
 491   mAnimationMode = aAnimationMode;
 492
 493   return NS_OK;
 494 }
 495
 496 bool ImageResource::HadRecentRefresh(const TimeStamp& aTime) {
 497   // Our threshold for "recent" is 1/2 of the default refresh-driver interval.
 498   // This ensures that we allow for frame rates at least as fast as the
 499   // refresh driver's default rate.
 500   static TimeDuration recentThreshold =
 501       TimeDuration::FromMilliseconds(nsRefreshDriver::DefaultInterval() / 2.0);
 502
 503   if (!mLastRefreshTime.IsNull() &&
 504       aTime - mLastRefreshTime < recentThreshold) {
 505     return true;
 506   }
 507
 508   // else, we can proceed with a refresh.
 509   // But first, update our last refresh time:
 510   mLastRefreshTime = aTime;
 511   return false;
 512 }
 513
 514 void ImageResource::EvaluateAnimation() {
 515   if (!mAnimating && ShouldAnimate()) {
 516     nsresult rv = StartAnimation();
 517     mAnimating = NS_SUCCEEDED(rv);
 518   } else if (mAnimating && !ShouldAnimate()) {
 519     StopAnimation();
 520   }
 521 }
 522
 523 void ImageResource::SendOnUnlockedDraw(uint32_t aFlags) {
 524   if (!mProgressTracker) {
 525     return;
 526   }
 527
 528   if (!(aFlags & FLAG_ASYNC_NOTIFY)) {
 529     mProgressTracker->OnUnlockedDraw();
 530   } else {
 531     NotNull<RefPtr<ImageResource>> image = WrapNotNull(this);
 532     nsCOMPtr<nsIEventTarget> eventTarget = mProgressTracker->GetEventTarget();
 533     nsCOMPtr<nsIRunnable> ev = NS_NewRunnableFunction(
 534         "image::ImageResource::SendOnUnlockedDraw", [=]() -> void {
 535           RefPtr<ProgressTracker> tracker = image->GetProgressTracker();
 536           if (tracker) {
 537             tracker->OnUnlockedDraw();
 538           }
 539         });
 540     eventTarget->Dispatch(CreateMediumHighRunnable(ev.forget()),
 541                           NS_DISPATCH_NORMAL);
 542   }
 543 }
 544
 545 #ifdef DEBUG
 546 void ImageResource::NotifyDrawingObservers() {
 547   if (!mURI || !NS_IsMainThread()) {
 548     return;
 549   }
 550
 551   if (!mURI->SchemeIs("resource") && !mURI->SchemeIs("chrome")) {
 552     return;
 553   }
 554
 555   // Record the image drawing for startup performance testing.
 556   nsCOMPtr<nsIURI> uri = mURI;
 557   nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
 558       "image::ImageResource::NotifyDrawingObservers", [uri]() {
 559         nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
 560         NS_WARNING_ASSERTION(obs, "Can't get an observer service handle");
 561         if (obs) {
 562           nsAutoCString spec;
 563           uri->GetSpec(spec);
 564           obs->NotifyObservers(nullptr, "image-drawing",
 565                                NS_ConvertUTF8toUTF16(spec).get());
 566         }
 567       }));
 568 }
 569 #endif
 570
 571 }  // namespace image
 572 }  // namespace mozilla