image/RasterImage.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  *
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 /** @file
   8  * This file declares the RasterImage class, which
   9  * handles static and animated rasterized images.
  10  *
  11  * @author  Stuart Parmenter <pavlov@netscape.com>
  12  * @author  Chris Saari <saari@netscape.com>
  13  * @author  Arron Mogge <paper@animecity.nu>
  14  * @author  Andrew Smith <asmith15@learn.senecac.on.ca>
  15  */
  16
  17 #ifndef mozilla_image_RasterImage_h
  18 #define mozilla_image_RasterImage_h
  19
  20 #include "Image.h"
  21 #include "nsCOMPtr.h"
  22 #include "imgIContainer.h"
  23 #include "nsTArray.h"
  24 #include "LookupResult.h"
  25 #include "nsThreadUtils.h"
  26 #include "DecoderFactory.h"
  27 #include "FrameAnimator.h"
  28 #include "ImageMetadata.h"
  29 #include "ISurfaceProvider.h"
  30 #include "Orientation.h"
  31 #include "mozilla/Attributes.h"
  32 #include "mozilla/Maybe.h"
  33 #include "mozilla/MemoryReporting.h"
  34 #include "mozilla/NotNull.h"
  35 #include "mozilla/StaticPrefs_image.h"
  36 #include "mozilla/TimeStamp.h"
  37 #include "mozilla/WeakPtr.h"
  38 #include "mozilla/UniquePtr.h"
  39 #include "ImageContainer.h"
  40 #include "PlaybackType.h"
  41 #ifdef DEBUG
  42 #  include "imgIContainerDebug.h"
  43 #endif
  44
  45 class nsIInputStream;
  46 class nsIRequest;
  47
  48 #define NS_RASTERIMAGE_CID                           \
  49   { /* 376ff2c1-9bf6-418a-b143-3340c00112f7 */       \
  50     0x376ff2c1, 0x9bf6, 0x418a, {                    \
  51       0xb1, 0x43, 0x33, 0x40, 0xc0, 0x01, 0x12, 0xf7 \
  52     }                                                \
  53   }
  54
  55 /**
  56  * Handles static and animated image containers.
  57  *
  58  *
  59  * @par A Quick Walk Through
  60  * The decoder initializes this class and calls AppendFrame() to add a frame.
  61  * Once RasterImage detects more than one frame, it starts the animation
  62  * with StartAnimation(). Note that the invalidation events for RasterImage are
  63  * generated automatically using nsRefreshDriver.
  64  *
  65  * @par
  66  * StartAnimation() initializes the animation helper object and sets the time
  67  * the first frame was displayed to the current clock time.
  68  *
  69  * @par
  70  * When the refresh driver corresponding to the imgIContainer that this image is
  71  * a part of notifies the RasterImage that it's time to invalidate,
  72  * RequestRefresh() is called with a given TimeStamp to advance to. As long as
  73  * the timeout of the given frame (the frame's "delay") plus the time that frame
  74  * was first displayed is less than or equal to the TimeStamp given,
  75  * RequestRefresh() calls AdvanceFrame().
  76  *
  77  * @par
  78  * AdvanceFrame() is responsible for advancing a single frame of the animation.
  79  * It can return true, meaning that the frame advanced, or false, meaning that
  80  * the frame failed to advance (usually because the next frame hasn't been
  81  * decoded yet). It is also responsible for performing the final animation stop
  82  * procedure if the final frame of a non-looping animation is reached.
  83  *
  84  * @par
  85  * Each frame can have a different method of removing itself. These are
  86  * listed as imgIContainer::cDispose... constants.  Notify() calls
  87  * DoComposite() to handle any special frame destruction.
  88  *
  89  * @par
  90  * The basic path through DoComposite() is:
  91  * 1) Calculate Area that needs updating, which is at least the area of
  92  *    aNextFrame.
  93  * 2) Dispose of previous frame.
  94  * 3) Draw new image onto compositingFrame.
  95  * See comments in DoComposite() for more information and optimizations.
  96  *
  97  * @par
  98  * The rest of the RasterImage specific functions are used by DoComposite to
  99  * destroy the old frame and build the new one.
 100  *
 101  * @note
 102  * <li> "Mask", "Alpha", and "Alpha Level" are interchangeable phrases in
 103  * respects to RasterImage.
 104  *
 105  * @par
 106  * <li> GIFs never have more than a 1 bit alpha.
 107  * <li> APNGs may have a full alpha channel.
 108  *
 109  * @par
 110  * <li> Background color specified in GIF is ignored by web browsers.
 111  *
 112  * @par
 113  * <li> If Frame 3 wants to dispose by restoring previous, what it wants is to
 114  * restore the composition up to and including Frame 2, as well as Frame 2s
 115  * disposal.  So, in the middle of DoComposite when composing Frame 3, right
 116  * after destroying Frame 2's area, we copy compositingFrame to
 117  * prevCompositingFrame.  When DoComposite gets called to do Frame 4, we
 118  * copy prevCompositingFrame back, and then draw Frame 4 on top.
 119  *
 120  * @par
 121  * The mAnim structure has members only needed for animated images, so
 122  * it's not allocated until the second frame is added.
 123  */
 124
 125 namespace mozilla {
 126
 127 // Pixel values in an image considering orientation metadata, such as the size
 128 // of an image as seen by consumers of the image.
 129 //
 130 // Any public methods on RasterImage that use untyped units are interpreted as
 131 // oriented pixels.
 132 struct OrientedPixel {};
 133 template <>
 134 struct IsPixel<OrientedPixel> : std::true_type {};
 135 typedef gfx::IntSizeTyped<OrientedPixel> OrientedIntSize;
 136 typedef gfx::IntRectTyped<OrientedPixel> OrientedIntRect;
 137
 138 // Pixel values in an image ignoring orientation metadata, such as are stored
 139 // in surfaces and the raw pixel data in the image.
 140 struct UnorientedPixel {};
 141 template <>
 142 struct IsPixel<UnorientedPixel> : std::true_type {};
 143 typedef gfx::IntSizeTyped<UnorientedPixel> UnorientedIntSize;
 144 typedef gfx::IntRectTyped<UnorientedPixel> UnorientedIntRect;
 145
 146 namespace layers {
 147 class ImageContainer;
 148 class Image;
 149 class LayersManager;
 150 }  // namespace layers
 151
 152 namespace image {
 153
 154 class Decoder;
 155 struct DecoderFinalStatus;
 156 struct DecoderTelemetry;
 157 class ImageMetadata;
 158 class SourceBuffer;
 159
 160 class RasterImage final : public ImageResource,
 161                           public SupportsWeakPtr
 162 #ifdef DEBUG
 163     ,
 164                           public imgIContainerDebug
 165 #endif
 166 {
 167   // (no public constructor - use ImageFactory)
 168   virtual ~RasterImage();
 169
 170  public:
 171   NS_DECL_THREADSAFE_ISUPPORTS
 172   NS_DECL_IMGICONTAINER
 173 #ifdef DEBUG
 174   NS_DECL_IMGICONTAINERDEBUG
 175 #endif
 176
 177   nsresult GetNativeSizes(nsTArray<gfx::IntSize>& aNativeSizes) const override;
 178   size_t GetNativeSizesLength() const override;
 179   virtual nsresult StartAnimation() override;
 180   virtual nsresult StopAnimation() override;
 181
 182   // Methods inherited from Image
 183   virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) override;
 184
 185   virtual size_t SizeOfSourceWithComputedFallback(
 186       SizeOfState& aState) const override;
 187   virtual void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
 188                                      MallocSizeOf aMallocSizeOf) const override;
 189
 190   /* Triggers discarding. */
 191   void Discard();
 192
 193   //////////////////////////////////////////////////////////////////////////////
 194   // Decoder callbacks.
 195   //////////////////////////////////////////////////////////////////////////////
 196
 197   /**
 198    * Sends the provided progress notifications to ProgressTracker.
 199    *
 200    * Main-thread only.
 201    *
 202    * @param aProgress    The progress notifications to send.
 203    * @param aInvalidRect An invalidation rect to send.
 204    * @param aFrameCount  If Some(), an updated count of the number of frames of
 205    *                     animation the decoder has finished decoding so far.
 206    *                     This is a lower bound for the total number of
 207    *                     animation frames this image has.
 208    * @param aDecoderFlags The decoder flags used by the decoder that generated
 209    *                      these notifications, or DefaultDecoderFlags() if the
 210    *                      notifications don't come from a decoder.
 211    * @param aSurfaceFlags The surface flags used by the decoder that generated
 212    *                      these notifications, or DefaultSurfaceFlags() if the
 213    *                      notifications don't come from a decoder.
 214    */
 215   void NotifyProgress(
 216       Progress aProgress,
 217       const UnorientedIntRect& aInvalidRect = UnorientedIntRect(),
 218       const Maybe<uint32_t>& aFrameCount = Nothing(),
 219       DecoderFlags aDecoderFlags = DefaultDecoderFlags(),
 220       SurfaceFlags aSurfaceFlags = DefaultSurfaceFlags());
 221
 222   /**
 223    * Records decoding results, sends out any final notifications, updates the
 224    * state of this image, and records telemetry.
 225    *
 226    * Main-thread only.
 227    *
 228    * @param aStatus       Final status information about the decoder. (Whether
 229    * it encountered an error, etc.)
 230    * @param aMetadata     Metadata about this image that the decoder gathered.
 231    * @param aTelemetry    Telemetry data about the decoder.
 232    * @param aProgress     Any final progress notifications to send.
 233    * @param aInvalidRect  Any final invalidation rect to send.
 234    * @param aFrameCount   If Some(), a final updated count of the number of
 235    * frames of animation the decoder has finished decoding so far. This is a
 236    * lower bound for the total number of animation frames this image has.
 237    * @param aDecoderFlags The decoder flags used by the decoder.
 238    * @param aSurfaceFlags The surface flags used by the decoder.
 239    */
 240   void NotifyDecodeComplete(
 241       const DecoderFinalStatus& aStatus, const ImageMetadata& aMetadata,
 242       const DecoderTelemetry& aTelemetry, Progress aProgress,
 243       const UnorientedIntRect& aInvalidRect, const Maybe<uint32_t>& aFrameCount,
 244       DecoderFlags aDecoderFlags, SurfaceFlags aSurfaceFlags);
 245
 246   // Helper method for NotifyDecodeComplete.
 247   void ReportDecoderError();
 248
 249   //////////////////////////////////////////////////////////////////////////////
 250   // Network callbacks.
 251   //////////////////////////////////////////////////////////////////////////////
 252
 253   virtual nsresult OnImageDataAvailable(nsIRequest* aRequest,
 254                                         nsISupports* aContext,
 255                                         nsIInputStream* aInStr,
 256                                         uint64_t aSourceOffset,
 257                                         uint32_t aCount) override;
 258   virtual nsresult OnImageDataComplete(nsIRequest* aRequest,
 259                                        nsISupports* aContext, nsresult aStatus,
 260                                        bool aLastPart) override;
 261
 262   void NotifyForLoadEvent(Progress aProgress);
 263
 264   /**
 265    * A hint of the number of bytes of source data that the image contains. If
 266    * called early on, this can help reduce copying and reallocations by
 267    * appropriately preallocating the source data buffer.
 268    *
 269    * We take this approach rather than having the source data management code do
 270    * something more complicated (like chunklisting) because HTTP is by far the
 271    * dominant source of images, and the Content-Length header is quite reliable.
 272    * Thus, pre-allocation simplifies code and reduces the total number of
 273    * allocations.
 274    */
 275   nsresult SetSourceSizeHint(uint32_t aSizeHint);
 276
 277   nsCString GetURIString() {
 278     nsCString spec;
 279     if (GetURI()) {
 280       GetURI()->GetSpec(spec);
 281     }
 282     return spec;
 283   }
 284
 285  private:
 286   nsresult Init(const char* aMimeType, uint32_t aFlags);
 287
 288   /**
 289    * Tries to retrieve a surface for this image with size @aSize, surface flags
 290    * matching @aFlags, and a playback type of @aPlaybackType.
 291    *
 292    * If @aFlags specifies FLAG_SYNC_DECODE and we already have all the image
 293    * data, we'll attempt a sync decode if no matching surface is found. If
 294    * FLAG_SYNC_DECODE was not specified and no matching surface was found, we'll
 295    * kick off an async decode so that the surface is (hopefully) available next
 296    * time it's requested. aMarkUsed determines if we mark the surface used in
 297    * the surface cache or not.
 298    *
 299    * @return a drawable surface, which may be empty if the requested surface
 300    *         could not be found.
 301    */
 302   LookupResult LookupFrame(const UnorientedIntSize& aSize, uint32_t aFlags,
 303                            PlaybackType aPlaybackType, bool aMarkUsed);
 304
 305   /// Helper method for LookupFrame().
 306   LookupResult LookupFrameInternal(const UnorientedIntSize& aSize,
 307                                    uint32_t aFlags, PlaybackType aPlaybackType,
 308                                    bool aMarkUsed);
 309
 310   ImgDrawResult DrawInternal(DrawableSurface&& aFrameRef, gfxContext* aContext,
 311                              const UnorientedIntSize& aSize,
 312                              const ImageRegion& aRegion,
 313                              gfx::SamplingFilter aSamplingFilter,
 314                              uint32_t aFlags, float aOpacity);
 315
 316   Tuple<ImgDrawResult, gfx::IntSize, RefPtr<gfx::SourceSurface>>
 317   GetFrameInternal(const gfx::IntSize& aSize,
 318                    const Maybe<SVGImageContext>& aSVGContext,
 319                    uint32_t aWhichFrame, uint32_t aFlags) override;
 320
 321   Tuple<ImgDrawResult, gfx::IntSize> GetImageContainerSize(
 322       layers::LayerManager* aManager, const gfx::IntSize& aSize,
 323       uint32_t aFlags) override;
 324
 325   //////////////////////////////////////////////////////////////////////////////
 326   // Decoding.
 327   //////////////////////////////////////////////////////////////////////////////
 328
 329   /**
 330    * Creates and runs a decoder, either synchronously or asynchronously
 331    * according to @aFlags. Decodes at the provided target size @aSize, using
 332    * decode flags @aFlags. Performs a single-frame decode of this image unless
 333    * we know the image is animated *and* @aPlaybackType is
 334    * PlaybackType::eAnimated.
 335    *
 336    * It's an error to call Decode() before this image's intrinsic size is
 337    * available. A metadata decode must successfully complete first.
 338    *
 339    * aOutRanSync is set to true if the decode was run synchronously.
 340    * aOutFailed is set to true if failed to start a decode.
 341    */
 342   void Decode(const UnorientedIntSize& aSize, uint32_t aFlags,
 343               PlaybackType aPlaybackType, bool& aOutRanSync, bool& aOutFailed);
 344
 345   /**
 346    * Creates and runs a metadata decoder, either synchronously or
 347    * asynchronously according to @aFlags.
 348    */
 349   NS_IMETHOD DecodeMetadata(uint32_t aFlags);
 350
 351   /**
 352    * Sets the size, inherent orientation, animation metadata, and other
 353    * information about the image gathered during decoding.
 354    *
 355    * This function may be called multiple times, but will throw an error if
 356    * subsequent calls do not match the first.
 357    *
 358    * @param aMetadata The metadata to set on this image.
 359    * @param aFromMetadataDecode True if this metadata came from a metadata
 360    *                            decode; false if it came from a full decode.
 361    * @return |true| unless a catastrophic failure was discovered. If |false| is
 362    * returned, it indicates that the image is corrupt in a way that requires all
 363    * surfaces to be discarded to recover.
 364    */
 365   bool SetMetadata(const ImageMetadata& aMetadata, bool aFromMetadataDecode);
 366
 367   /**
 368    * In catastrophic circumstances like a GPU driver crash, the contents of our
 369    * frames may become invalid.  If the information we gathered during the
 370    * metadata decode proves to be wrong due to image corruption, the frames we
 371    * have may violate this class's invariants. Either way, we need to
 372    * immediately discard the invalid frames and redecode so that callers don't
 373    * perceive that we've entered an invalid state.
 374    *
 375    * RecoverFromInvalidFrames discards all existing frames and redecodes using
 376    * the provided @aSize and @aFlags.
 377    */
 378   void RecoverFromInvalidFrames(const UnorientedIntSize& aSize,
 379                                 uint32_t aFlags);
 380
 381   void OnSurfaceDiscardedInternal(bool aAnimatedFramesDiscarded);
 382
 383   /**
 384    * Computes a matrix that applies the rotation and reflection specified by
 385    * UsedOrientation(), or that matrix's inverse if aInvert is true.
 386    *
 387    * See OrientedImage::OrientationMatrix.
 388    */
 389   gfxMatrix OrientationMatrix(const UnorientedIntSize& aSize,
 390                               bool aInvert = false) const;
 391
 392   /**
 393    * The orientation value to honor for this image.
 394    *
 395    * If the image.honor-orientation-metadata pref is true, then this returns
 396    * the orientation from the image's metadata.  Otherwise, it returns an
 397    * Orientation that indicates no transformation is needed.
 398    */
 399   Orientation UsedOrientation() const {
 400     return mHandledOrientation ? mOrientation : Orientation();
 401   }
 402
 403   // Functions to convert between oriented and unoriented pixels.
 404   UnorientedIntSize ToUnoriented(OrientedIntSize aSize) const {
 405     return UsedOrientation().SwapsWidthAndHeight()
 406                ? UnorientedIntSize(aSize.height, aSize.width)
 407                : UnorientedIntSize(aSize.width, aSize.height);
 408   }
 409   OrientedIntSize ToOriented(UnorientedIntSize aSize) const {
 410     return UsedOrientation().SwapsWidthAndHeight()
 411                ? OrientedIntSize(aSize.height, aSize.width)
 412                : OrientedIntSize(aSize.width, aSize.height);
 413   }
 414   OrientedIntRect ToOriented(UnorientedIntRect aRect) const;
 415   UnorientedIntRect ToUnoriented(OrientedIntRect aRect) const;
 416
 417  private:  // data
 418   OrientedIntSize mSize;
 419   nsTArray<OrientedIntSize> mNativeSizes;
 420
 421   // The orientation required to correctly orient the image, from the image's
 422   // metadata.
 423   //
 424   // When the image.honor-orientation-metadata pref is enabled, the RasterImage
 425   // will handle and apply this orientation itself.
 426   //
 427   // When the pref is disabled, it is the responsibility of users of the
 428   // RasterImage to wrap it in an OrientedImage if desired.
 429   Orientation mOrientation;
 430
 431   /// If this has a value, we're waiting for SetSize() to send the load event.
 432   Maybe<Progress> mLoadProgress;
 433
 434   // Hotspot of this image, or (0, 0) if there is no hotspot data.
 435   //
 436   // We assume (and assert) that no image has both orientation metadata and a
 437   // hotspot, so we store this as an untyped point.
 438   gfx::IntPoint mHotspot;
 439
 440   /// If this image is animated, a FrameAnimator which manages its animation.
 441   UniquePtr<FrameAnimator> mFrameAnimator;
 442
 443   /// Animation timeline and other state for animation images.
 444   Maybe<AnimationState> mAnimationState;
 445
 446   // Image locking.
 447   uint32_t mLockCount;
 448
 449   // The type of decoder this image needs. Computed from the MIME type in
 450   // Init().
 451   DecoderType mDecoderType;
 452
 453   // How many times we've decoded this image.
 454   // This is currently only used for statistics
 455   int32_t mDecodeCount;
 456
 457 #ifdef DEBUG
 458   uint32_t mFramesNotified;
 459 #endif
 460
 461   // The source data for this image.
 462   NotNull<RefPtr<SourceBuffer>> mSourceBuffer;
 463
 464   // Boolean flags (clustered together to conserve space):
 465   bool mHasSize : 1;         // Has SetSize() been called?
 466   bool mTransient : 1;       // Is the image short-lived?
 467   bool mSyncLoad : 1;        // Are we loading synchronously?
 468   bool mDiscardable : 1;     // Is container discardable?
 469   bool mSomeSourceData : 1;  // Do we have some source data?
 470   bool mAllSourceData : 1;   // Do we have all the source data?
 471   bool mHasBeenDecoded : 1;  // Decoded at least once?
 472
 473   // Whether we're waiting to start animation. If we get a StartAnimation() call
 474   // but we don't yet have more than one frame, mPendingAnimation is set so that
 475   // we know to start animation later if/when we have more frames.
 476   bool mPendingAnimation : 1;
 477
 478   // Whether the animation can stop, due to running out
 479   // of frames, or no more owning request
 480   bool mAnimationFinished : 1;
 481
 482   // Whether, once we are done doing a metadata decode, we should immediately
 483   // kick off a full decode.
 484   bool mWantFullDecode : 1;
 485
 486   // Whether this RasterImage handled orientation of the image.
 487   //
 488   // This will be set based on the value of the image.honor-orientation-metadata
 489   // pref at the time the RasterImage is created.
 490   //
 491   // NOTE(heycam): Once the image.honor-orientation-metadata pref is removed,
 492   // this member (and the UsedOrientation() function) can also be removed.
 493   bool mHandledOrientation : 1;
 494
 495   TimeStamp mDrawStartTime;
 496
 497   //////////////////////////////////////////////////////////////////////////////
 498   // Scaling.
 499   //////////////////////////////////////////////////////////////////////////////
 500
 501   // Determines whether we can downscale during decode with the given
 502   // parameters.
 503   bool CanDownscaleDuringDecode(const UnorientedIntSize& aSize,
 504                                 uint32_t aFlags);
 505
 506   // Error handling.
 507   void DoError();
 508
 509   class HandleErrorWorker : public Runnable {
 510    public:
 511     /**
 512      * Called from decoder threads when DoError() is called, since errors can't
 513      * be handled safely off-main-thread. Dispatches an event which reinvokes
 514      * DoError on the main thread if there isn't one already pending.
 515      */
 516     static void DispatchIfNeeded(RasterImage* aImage);
 517
 518     NS_IMETHOD Run() override;
 519
 520    private:
 521     explicit HandleErrorWorker(RasterImage* aImage);
 522
 523     RefPtr<RasterImage> mImage;
 524   };
 525
 526   // Helpers
 527   bool CanDiscard();
 528
 529   bool IsOpaque();
 530
 531   LookupResult RequestDecodeForSizeInternal(const UnorientedIntSize& aSize,
 532                                             uint32_t aFlags,
 533                                             uint32_t aWhichFrame);
 534
 535  protected:
 536   explicit RasterImage(nsIURI* aURI = nullptr);
 537
 538   bool ShouldAnimate() override;
 539
 540   friend class ImageFactory;
 541 };
 542
 543 inline NS_IMETHODIMP RasterImage::GetAnimationMode(uint16_t* aAnimationMode) {
 544   return GetAnimationModeInternal(aAnimationMode);
 545 }
 546
 547 }  // namespace image
 548 }  // namespace mozilla
 549
 550 /**
 551  * Casting RasterImage to nsISupports is ambiguous. This method handles that.
 552  */
 553 inline nsISupports* ToSupports(mozilla::image::RasterImage* p) {
 554   return NS_ISUPPORTS_CAST(mozilla::image::ImageResource*, p);
 555 }
 556
 557 #endif /* mozilla_image_RasterImage_h */