image/Image.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef mozilla_image_Image_h
   7 #define mozilla_image_Image_h
   8
   9 #include "mozilla/Attributes.h"
  10 #include "mozilla/Maybe.h"
  11 #include "mozilla/MemoryReporting.h"
  12 #include "mozilla/ProfilerMarkers.h"
  13 #include "mozilla/SizeOfState.h"
  14 #include "mozilla/ThreadSafeWeakPtr.h"
  15 #include "mozilla/TimeStamp.h"
  16 #include "mozilla/Tuple.h"
  17 #include "gfx2DGlue.h"
  18 #include "imgIContainer.h"
  19 #include "ImageContainer.h"
  20 #include "ImageRegion.h"
  21 #include "LookupResult.h"
  22 #include "nsStringFwd.h"
  23 #include "ProgressTracker.h"
  24 #include "SurfaceCache.h"
  25
  26 class imgRequest;
  27 class nsIRequest;
  28 class nsIInputStream;
  29
  30 namespace mozilla {
  31 namespace image {
  32
  33 class Image;
  34
  35 ///////////////////////////////////////////////////////////////////////////////
  36 // Memory Reporting
  37 ///////////////////////////////////////////////////////////////////////////////
  38
  39 struct MemoryCounter {
  40   MemoryCounter()
  41       : mSource(0),
  42         mDecodedHeap(0),
  43         mDecodedNonHeap(0),
  44         mDecodedUnknown(0),
  45         mExternalHandles(0),
  46         mFrameIndex(0),
  47         mExternalId(0),
  48         mSurfaceTypes(0) {}
  49
  50   void SetSource(size_t aCount) { mSource = aCount; }
  51   size_t Source() const { return mSource; }
  52   void SetDecodedHeap(size_t aCount) { mDecodedHeap = aCount; }
  53   size_t DecodedHeap() const { return mDecodedHeap; }
  54   void SetDecodedNonHeap(size_t aCount) { mDecodedNonHeap = aCount; }
  55   size_t DecodedNonHeap() const { return mDecodedNonHeap; }
  56   void SetDecodedUnknown(size_t aCount) { mDecodedUnknown = aCount; }
  57   size_t DecodedUnknown() const { return mDecodedUnknown; }
  58   void SetExternalHandles(size_t aCount) { mExternalHandles = aCount; }
  59   size_t ExternalHandles() const { return mExternalHandles; }
  60   void SetFrameIndex(size_t aIndex) { mFrameIndex = aIndex; }
  61   size_t FrameIndex() const { return mFrameIndex; }
  62   void SetExternalId(uint64_t aId) { mExternalId = aId; }
  63   uint64_t ExternalId() const { return mExternalId; }
  64   void SetSurfaceTypes(uint32_t aTypes) { mSurfaceTypes = aTypes; }
  65   uint32_t SurfaceTypes() const { return mSurfaceTypes; }
  66
  67   MemoryCounter& operator+=(const MemoryCounter& aOther) {
  68     mSource += aOther.mSource;
  69     mDecodedHeap += aOther.mDecodedHeap;
  70     mDecodedNonHeap += aOther.mDecodedNonHeap;
  71     mDecodedUnknown += aOther.mDecodedUnknown;
  72     mExternalHandles += aOther.mExternalHandles;
  73     mSurfaceTypes |= aOther.mSurfaceTypes;
  74     return *this;
  75   }
  76
  77  private:
  78   size_t mSource;
  79   size_t mDecodedHeap;
  80   size_t mDecodedNonHeap;
  81   size_t mDecodedUnknown;
  82   size_t mExternalHandles;
  83   size_t mFrameIndex;
  84   uint64_t mExternalId;
  85   uint32_t mSurfaceTypes;
  86 };
  87
  88 enum class SurfaceMemoryCounterType { NORMAL, CONTAINER };
  89
  90 struct SurfaceMemoryCounter {
  91   SurfaceMemoryCounter(
  92       const SurfaceKey& aKey, const gfx::SourceSurface* aSurface,
  93       bool aIsLocked, bool aCannotSubstitute, bool aIsFactor2, bool aFinished,
  94       SurfaceMemoryCounterType aType = SurfaceMemoryCounterType::NORMAL)
  95       : mKey(aKey),
  96         mSurface(aSurface),
  97         mType(aType),
  98         mIsLocked(aIsLocked),
  99         mCannotSubstitute(aCannotSubstitute),
 100         mIsFactor2(aIsFactor2),
 101         mFinished(aFinished) {}
 102
 103   const SurfaceKey& Key() const { return mKey; }
 104   const gfx::SourceSurface* Surface() const { return mSurface; }
 105   MemoryCounter& Values() { return mValues; }
 106   const MemoryCounter& Values() const { return mValues; }
 107   SurfaceMemoryCounterType Type() const { return mType; }
 108   bool IsLocked() const { return mIsLocked; }
 109   bool CannotSubstitute() const { return mCannotSubstitute; }
 110   bool IsFactor2() const { return mIsFactor2; }
 111   bool IsFinished() const { return mFinished; }
 112
 113  private:
 114   const SurfaceKey mKey;
 115   const gfx::SourceSurface* MOZ_NON_OWNING_REF mSurface;
 116   MemoryCounter mValues;
 117   const SurfaceMemoryCounterType mType;
 118   const bool mIsLocked;
 119   const bool mCannotSubstitute;
 120   const bool mIsFactor2;
 121   const bool mFinished;
 122 };
 123
 124 struct ImageMemoryCounter {
 125   ImageMemoryCounter(imgRequest* aRequest, SizeOfState& aState, bool aIsUsed);
 126   ImageMemoryCounter(imgRequest* aRequest, Image* aImage, SizeOfState& aState,
 127                      bool aIsUsed);
 128
 129   nsCString& URI() { return mURI; }
 130   const nsCString& URI() const { return mURI; }
 131   const nsTArray<SurfaceMemoryCounter>& Surfaces() const { return mSurfaces; }
 132   const gfx::IntSize IntrinsicSize() const { return mIntrinsicSize; }
 133   const MemoryCounter& Values() const { return mValues; }
 134   uint32_t Progress() const { return mProgress; }
 135   uint16_t Type() const { return mType; }
 136   bool IsUsed() const { return mIsUsed; }
 137   bool HasError() const { return mHasError; }
 138   bool IsValidating() const { return mValidating; }
 139
 140   bool IsNotable() const {
 141     // Errors or requests without images are always notable.
 142     if (mHasError || mValidating || mProgress == UINT32_MAX ||
 143         mProgress & FLAG_HAS_ERROR || mType == imgIContainer::TYPE_REQUEST) {
 144       return true;
 145     }
 146
 147     // Sufficiently large images are notable.
 148     const size_t NotableThreshold = 16 * 1024;
 149     size_t total = mValues.Source() + mValues.DecodedHeap() +
 150                    mValues.DecodedNonHeap() + mValues.DecodedUnknown();
 151     if (total >= NotableThreshold) {
 152       return true;
 153     }
 154
 155     // Incomplete images are always notable as well; the odds of capturing
 156     // mid-decode should be fairly low.
 157     for (const auto& surface : mSurfaces) {
 158       if (!surface.IsFinished()) {
 159         return true;
 160       }
 161     }
 162
 163     return false;
 164   }
 165
 166  private:
 167   nsCString mURI;
 168   nsTArray<SurfaceMemoryCounter> mSurfaces;
 169   gfx::IntSize mIntrinsicSize;
 170   MemoryCounter mValues;
 171   uint32_t mProgress;
 172   uint16_t mType;
 173   const bool mIsUsed;
 174   bool mHasError;
 175   bool mValidating;
 176 };
 177
 178 ///////////////////////////////////////////////////////////////////////////////
 179 // Image Base Types
 180 ///////////////////////////////////////////////////////////////////////////////
 181
 182 class Image : public imgIContainer {
 183  public:
 184   /**
 185    * Flags for Image initialization.
 186    *
 187    * Meanings:
 188    *
 189    * INIT_FLAG_NONE: Lack of flags
 190    *
 191    * INIT_FLAG_DISCARDABLE: The container should be discardable
 192    *
 193    * INIT_FLAG_DECODE_IMMEDIATELY: The container should decode as soon as
 194    * possible, regardless of what our heuristics say.
 195    *
 196    * INIT_FLAG_TRANSIENT: The container is likely to exist for only a short time
 197    * before being destroyed. (For example, containers for
 198    * multipart/x-mixed-replace image parts fall into this category.) If this
 199    * flag is set, INIT_FLAG_DISCARDABLE and INIT_FLAG_DECODE_ONLY_ON_DRAW must
 200    * not be set.
 201    *
 202    * INIT_FLAG_SYNC_LOAD: The container is being loaded synchronously, so
 203    * it should avoid relying on async workers to get the container ready.
 204    */
 205   static const uint32_t INIT_FLAG_NONE = 0x0;
 206   static const uint32_t INIT_FLAG_DISCARDABLE = 0x1;
 207   static const uint32_t INIT_FLAG_DECODE_IMMEDIATELY = 0x2;
 208   static const uint32_t INIT_FLAG_TRANSIENT = 0x4;
 209   static const uint32_t INIT_FLAG_SYNC_LOAD = 0x8;
 210
 211   virtual already_AddRefed<ProgressTracker> GetProgressTracker() = 0;
 212   virtual void SetProgressTracker(ProgressTracker* aProgressTracker) {}
 213
 214   /**
 215    * The size, in bytes, occupied by the compressed source data of the image.
 216    * If MallocSizeOf does not work on this platform, uses a fallback approach to
 217    * ensure that something reasonable is always returned.
 218    */
 219   virtual size_t SizeOfSourceWithComputedFallback(
 220       SizeOfState& aState) const = 0;
 221
 222   /**
 223    * Collect an accounting of the memory occupied by the image's surfaces (which
 224    * together make up its decoded data). Each surface is recorded as a separate
 225    * SurfaceMemoryCounter, stored in @aCounters.
 226    */
 227   virtual void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
 228                                      MallocSizeOf aMallocSizeOf) const = 0;
 229
 230   virtual void IncrementAnimationConsumers() = 0;
 231   virtual void DecrementAnimationConsumers() = 0;
 232 #ifdef DEBUG
 233   virtual uint32_t GetAnimationConsumers() = 0;
 234 #endif
 235
 236   /**
 237    * Called from OnDataAvailable when the stream associated with the image has
 238    * received new image data. The arguments are the same as OnDataAvailable's,
 239    * but by separating this functionality into a different method we don't
 240    * interfere with subclasses which wish to implement nsIStreamListener.
 241    *
 242    * Images should not do anything that could send out notifications until they
 243    * have received their first OnImageDataAvailable notification; in
 244    * particular, this means that instantiating decoders should be deferred
 245    * until OnImageDataAvailable is called.
 246    */
 247   virtual nsresult OnImageDataAvailable(nsIRequest* aRequest,
 248                                         nsISupports* aContext,
 249                                         nsIInputStream* aInStr,
 250                                         uint64_t aSourceOffset,
 251                                         uint32_t aCount) = 0;
 252
 253   /**
 254    * Called from OnStopRequest when the image's underlying request completes.
 255    *
 256    * @param aRequest  The completed request.
 257    * @param aContext  Context from Necko's OnStopRequest.
 258    * @param aStatus   A success or failure code.
 259    * @param aLastPart Whether this is the final part of the underlying request.
 260    */
 261   virtual nsresult OnImageDataComplete(nsIRequest* aRequest,
 262                                        nsISupports* aContext, nsresult aStatus,
 263                                        bool aLastPart) = 0;
 264
 265   /**
 266    * Called when the SurfaceCache discards a surface belonging to this image.
 267    */
 268   virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) = 0;
 269
 270   virtual void SetInnerWindowID(uint64_t aInnerWindowId) = 0;
 271   virtual uint64_t InnerWindowID() const = 0;
 272
 273   virtual bool HasError() = 0;
 274   virtual void SetHasError() = 0;
 275
 276   virtual nsIURI* GetURI() const = 0;
 277
 278   NS_IMETHOD GetHotspotX(int32_t* aX) override {
 279     *aX = 0;
 280     return NS_OK;
 281   }
 282   NS_IMETHOD GetHotspotY(int32_t* aY) override {
 283     *aY = 0;
 284     return NS_OK;
 285   }
 286 };
 287
 288 class ImageResource : public Image {
 289  public:
 290   already_AddRefed<ProgressTracker> GetProgressTracker() override {
 291     RefPtr<ProgressTracker> progressTracker = mProgressTracker;
 292     MOZ_ASSERT(progressTracker);
 293     return progressTracker.forget();
 294   }
 295
 296   void SetProgressTracker(ProgressTracker* aProgressTracker) final {
 297     MOZ_ASSERT(aProgressTracker);
 298     MOZ_ASSERT(!mProgressTracker);
 299     mProgressTracker = aProgressTracker;
 300   }
 301
 302   virtual void IncrementAnimationConsumers() override;
 303   virtual void DecrementAnimationConsumers() override;
 304 #ifdef DEBUG
 305   virtual uint32_t GetAnimationConsumers() override {
 306     return mAnimationConsumers;
 307   }
 308 #endif
 309
 310   virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) override {}
 311
 312   virtual void SetInnerWindowID(uint64_t aInnerWindowId) override {
 313     mInnerWindowId = aInnerWindowId;
 314   }
 315   virtual uint64_t InnerWindowID() const override { return mInnerWindowId; }
 316
 317   virtual bool HasError() override { return mError; }
 318   virtual void SetHasError() override { mError = true; }
 319
 320   /*
 321    * Returns a non-AddRefed pointer to the URI associated with this image.
 322    * Illegal to use off-main-thread.
 323    */
 324   nsIURI* GetURI() const override { return mURI; }
 325
 326   /*
 327    * Should be called by its subclasses after they populate @aCounters so that
 328    * we can cross reference against any of our ImageContainers that contain
 329    * surfaces not in the cache.
 330    */
 331   void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
 332                              MallocSizeOf aMallocSizeOf) const override;
 333
 334  protected:
 335   explicit ImageResource(nsIURI* aURI);
 336   ~ImageResource();
 337
 338   layers::ContainerProducerID GetImageProducerId() const {
 339     return mImageProducerID;
 340   }
 341
 342   bool GetSpecTruncatedTo1k(nsCString& aSpec) const;
 343
 344   // Shared functionality for implementors of imgIContainer. Every
 345   // implementation of attribute animationMode should forward here.
 346   nsresult GetAnimationModeInternal(uint16_t* aAnimationMode);
 347   nsresult SetAnimationModeInternal(uint16_t aAnimationMode);
 348
 349   /**
 350    * Helper for RequestRefresh.
 351    *
 352    * If we've had a "recent" refresh (i.e. if this image is being used in
 353    * multiple documents & some other document *just* called RequestRefresh() on
 354    * this image with a timestamp close to aTime), this method returns true.
 355    *
 356    * Otherwise, this method updates mLastRefreshTime to aTime & returns false.
 357    */
 358   bool HadRecentRefresh(const TimeStamp& aTime);
 359
 360   /**
 361    * Decides whether animation should or should not be happening,
 362    * and makes sure the right thing is being done.
 363    */
 364   virtual void EvaluateAnimation();
 365
 366   /**
 367    * Extended by child classes, if they have additional
 368    * conditions for being able to animate.
 369    */
 370   virtual bool ShouldAnimate() {
 371     return mAnimationConsumers > 0 && mAnimationMode != kDontAnimMode;
 372   }
 373
 374   virtual nsresult StartAnimation() = 0;
 375   virtual nsresult StopAnimation() = 0;
 376
 377   void SendOnUnlockedDraw(uint32_t aFlags);
 378
 379 #ifdef DEBUG
 380   // Records the image drawing for startup performance testing.
 381   void NotifyDrawingObservers();
 382 #endif
 383
 384   // Member data shared by all implementations of this abstract class
 385   RefPtr<ProgressTracker> mProgressTracker;
 386   nsCOMPtr<nsIURI> mURI;
 387   TimeStamp mLastRefreshTime;
 388   uint64_t mInnerWindowId;
 389   uint32_t mAnimationConsumers;
 390   uint16_t mAnimationMode;  // Enum values in imgIContainer
 391   bool mInitialized : 1;    // Have we been initialized?
 392   bool mAnimating : 1;      // Are we currently animating?
 393   bool mError : 1;          // Error handling
 394
 395   /**
 396    * Attempt to find a matching cached surface in the SurfaceCache, and if not
 397    * available, request the production of such a surface (either synchronously
 398    * or asynchronously).
 399    *
 400    * If the draw result is BAD_IMAGE, BAD_ARGS or NOT_READY, the size will be
 401    * the same as aSize. If it is TEMPORARY_ERROR, INCOMPLETE, or SUCCESS, the
 402    * size is a hint as to what we expect the surface size to be, once the best
 403    * fitting size is available. It may or may not match the size of the surface
 404    * returned at this moment. This is useful for choosing how to store the final
 405    * result (e.g. if going into an ImageContainer, ideally we would share the
 406    * same container for many requested sizes, if they all end up with the same
 407    * best fit size in the end).
 408    *
 409    * A valid surface should only be returned for SUCCESS and INCOMPLETE.
 410    *
 411    * Any other draw result is invalid.
 412    */
 413   virtual Tuple<ImgDrawResult, gfx::IntSize, RefPtr<gfx::SourceSurface>>
 414   GetFrameInternal(const gfx::IntSize& aSize,
 415                    const Maybe<SVGImageContext>& aSVGContext,
 416                    const Maybe<ImageIntRegion>& aRegion, uint32_t aWhichFrame,
 417                    uint32_t aFlags) {
 418     return MakeTuple(ImgDrawResult::BAD_IMAGE, aSize,
 419                      RefPtr<gfx::SourceSurface>());
 420   }
 421
 422   /**
 423    * Calculate the estimated size to use for an image container with the given
 424    * parameters. It may not be the same as the given size, and it may not be
 425    * the same as the size of the surface in the image container, but it is the
 426    * best effort estimate.
 427    */
 428   virtual Tuple<ImgDrawResult, gfx::IntSize> GetImageContainerSize(
 429       WindowRenderer* aRenderer, const gfx::IntSize& aSize, uint32_t aFlags) {
 430     return MakeTuple(ImgDrawResult::NOT_SUPPORTED, gfx::IntSize(0, 0));
 431   }
 432
 433   ImgDrawResult GetImageContainerImpl(WindowRenderer* aRenderer,
 434                                       const gfx::IntSize& aSize,
 435                                       const Maybe<SVGImageContext>& aSVGContext,
 436                                       const Maybe<ImageIntRegion>& aRegion,
 437                                       uint32_t aFlags,
 438                                       layers::ImageContainer** aContainer);
 439
 440   /**
 441    * Re-requests the appropriate frames for each image container using
 442    * GetFrameInternal.
 443    * @returns True if any image containers were updated, else false.
 444    */
 445   bool UpdateImageContainer(const Maybe<gfx::IntRect>& aDirtyRect);
 446
 447   void ReleaseImageContainer();
 448
 449   class MOZ_RAII AutoProfilerImagePaintMarker {
 450    public:
 451     explicit AutoProfilerImagePaintMarker(ImageResource* self)
 452         : mStartTime(TimeStamp::Now()) {
 453       nsAutoCString spec;
 454       if (self->mURI && profiler_can_accept_markers()) {
 455         static const size_t sMaxTruncatedLength = 1024;
 456         self->mURI->GetSpec(mSpec);
 457         if (mSpec.Length() >= sMaxTruncatedLength) {
 458           mSpec.Truncate(sMaxTruncatedLength);
 459         }
 460       }
 461     }
 462
 463     ~AutoProfilerImagePaintMarker() {
 464       if (!mSpec.IsEmpty()) {
 465         PROFILER_MARKER_TEXT("Image Paint", GRAPHICS,
 466                              MarkerTiming::IntervalUntilNowFrom(mStartTime),
 467                              mSpec);
 468       }
 469     }
 470
 471    protected:
 472     TimeStamp mStartTime;
 473     nsAutoCString mSpec;
 474   };
 475
 476  private:
 477   void SetCurrentImage(layers::ImageContainer* aContainer,
 478                        gfx::SourceSurface* aSurface,
 479                        const Maybe<gfx::IntRect>& aDirtyRect);
 480
 481   struct ImageContainerEntry {
 482     ImageContainerEntry(const gfx::IntSize& aSize,
 483                         const Maybe<SVGImageContext>& aSVGContext,
 484                         const Maybe<ImageIntRegion>& aRegion,
 485                         layers::ImageContainer* aContainer, uint32_t aFlags)
 486         : mSize(aSize),
 487           mSVGContext(aSVGContext),
 488           mRegion(aRegion),
 489           mContainer(aContainer),
 490           mLastDrawResult(ImgDrawResult::NOT_READY),
 491           mFlags(aFlags) {}
 492
 493     gfx::IntSize mSize;
 494     Maybe<SVGImageContext> mSVGContext;
 495     Maybe<ImageIntRegion> mRegion;
 496     // A weak pointer to our ImageContainer, which stays alive only as long as
 497     // the layer system needs it.
 498     ThreadSafeWeakPtr<layers::ImageContainer> mContainer;
 499     // If mContainer is non-null, this contains the ImgDrawResult we obtained
 500     // the last time we updated it.
 501     ImgDrawResult mLastDrawResult;
 502     // Cached flags to use for decoding. FLAG_ASYNC_NOTIFY should always be set
 503     // but FLAG_HIGH_QUALITY_SCALING may vary.
 504     uint32_t mFlags;
 505   };
 506
 507   AutoTArray<ImageContainerEntry, 1> mImageContainers;
 508   layers::ImageContainer::ProducerID mImageProducerID;
 509   layers::ImageContainer::FrameID mLastFrameID;
 510 };
 511
 512 }  // namespace image
 513 }  // namespace mozilla
 514
 515 #endif  // mozilla_image_Image_h