Bug 1869043 assert that graph set access is main thread only r=padenot
[gecko.git] / image / Decoder.h
blobafb123688078485296ea5b89cab8fe2351d8b600
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef mozilla_image_Decoder_h
7 #define mozilla_image_Decoder_h
9 #include "FrameAnimator.h"
10 #include "RasterImage.h"
11 #include "mozilla/Maybe.h"
12 #include "mozilla/NotNull.h"
13 #include "mozilla/RefPtr.h"
14 #include "AnimationParams.h"
15 #include "DecoderFlags.h"
16 #include "ImageMetadata.h"
17 #include "Orientation.h"
18 #include "Resolution.h"
19 #include "SourceBuffer.h"
20 #include "StreamingLexer.h"
21 #include "SurfaceFlags.h"
22 #include "qcms.h"
24 enum class CMSMode : int32_t;
26 namespace mozilla {
28 namespace Telemetry {
29 enum HistogramID : uint32_t;
30 } // namespace Telemetry
32 namespace image {
34 class imgFrame;
36 struct DecoderFinalStatus final {
37 DecoderFinalStatus(bool aWasMetadataDecode, bool aFinished, bool aHadError,
38 bool aShouldReportError)
39 : mWasMetadataDecode(aWasMetadataDecode),
40 mFinished(aFinished),
41 mHadError(aHadError),
42 mShouldReportError(aShouldReportError) {}
44 /// True if this was a metadata decode.
45 const bool mWasMetadataDecode : 1;
47 /// True if this decoder finished, whether successfully or due to failure.
48 const bool mFinished : 1;
50 /// True if this decoder encountered an error.
51 const bool mHadError : 1;
53 /// True if this decoder encountered the kind of error that should be reported
54 /// to the console.
55 const bool mShouldReportError : 1;
58 struct DecoderTelemetry final {
59 DecoderTelemetry(const Maybe<Telemetry::HistogramID>& aSpeedHistogram,
60 size_t aBytesDecoded, uint32_t aChunkCount,
61 TimeDuration aDecodeTime)
62 : mSpeedHistogram(aSpeedHistogram),
63 mBytesDecoded(aBytesDecoded),
64 mChunkCount(aChunkCount),
65 mDecodeTime(aDecodeTime) {}
67 /// @return our decoder's speed, in KBps.
68 int32_t Speed() const {
69 return mBytesDecoded / (1024 * mDecodeTime.ToSeconds());
72 /// @return our decoder's decode time, in microseconds.
73 int32_t DecodeTimeMicros() { return mDecodeTime.ToMicroseconds(); }
75 /// The per-image-format telemetry ID for recording our decoder's speed, or
76 /// Nothing() if we don't record speed telemetry for this kind of decoder.
77 const Maybe<Telemetry::HistogramID> mSpeedHistogram;
79 /// The number of bytes of input our decoder processed.
80 const size_t mBytesDecoded;
82 /// The number of chunks our decoder's input was divided into.
83 const uint32_t mChunkCount;
85 /// The amount of time our decoder spent inside DoDecode().
86 const TimeDuration mDecodeTime;
89 /**
90 * Interface which owners of an animated Decoder object must implement in order
91 * to use recycling. It allows the decoder to get a handle to the recycled
92 * frames.
94 class IDecoderFrameRecycler {
95 public:
96 /**
97 * Request the next available recycled imgFrame from the recycler.
99 * @param aRecycleRect If a frame is returned, this must be set to the
100 * accumulated dirty rect between the frame being
101 * recycled, and the frame being generated.
103 * @returns The recycled frame, if any is available.
105 virtual RawAccessFrameRef RecycleFrame(gfx::IntRect& aRecycleRect) = 0;
108 class Decoder {
109 public:
110 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Decoder)
112 explicit Decoder(RasterImage* aImage);
115 * Initialize an image decoder. Decoders may not be re-initialized.
117 * @return NS_OK if the decoder could be initialized successfully.
119 nsresult Init();
122 * Decodes, reading all data currently available in the SourceBuffer.
124 * If more data is needed and @aOnResume is non-null, Decode() will schedule
125 * @aOnResume to be called when more data is available.
127 * @return a LexerResult which may indicate:
128 * - the image has been successfully decoded (TerminalState::SUCCESS), or
129 * - the image has failed to decode (TerminalState::FAILURE), or
130 * - the decoder is yielding until it gets more data
131 * (Yield::NEED_MORE_DATA), or
132 * - the decoder is yielding to allow the caller to access intermediate
133 * output (Yield::OUTPUT_AVAILABLE).
135 LexerResult Decode(IResumable* aOnResume = nullptr);
138 * Terminate this decoder in a failure state, just as if the decoder
139 * implementation had returned TerminalState::FAILURE from DoDecode().
141 * XXX(seth): This method should be removed ASAP; it exists only because
142 * RasterImage::FinalizeDecoder() requires an actual Decoder object as an
143 * argument, so we can't simply tell RasterImage a decode failed except via an
144 * intervening decoder. We'll fix this in bug 1291071.
146 LexerResult TerminateFailure();
149 * Given a maximum number of bytes we're willing to decode, @aByteLimit,
150 * returns true if we should attempt to run this decoder synchronously.
152 bool ShouldSyncDecode(size_t aByteLimit);
155 * Gets the invalidation region accumulated by the decoder so far, and clears
156 * the decoder's invalidation region. This means that each call to
157 * TakeInvalidRect() returns only the invalidation region accumulated since
158 * the last call to TakeInvalidRect().
160 OrientedIntRect TakeInvalidRect() {
161 OrientedIntRect invalidRect = mInvalidRect;
162 mInvalidRect.SetEmpty();
163 return invalidRect;
167 * Gets the progress changes accumulated by the decoder so far, and clears
168 * them. This means that each call to TakeProgress() returns only the changes
169 * accumulated since the last call to TakeProgress().
171 Progress TakeProgress() {
172 Progress progress = mProgress;
173 mProgress = NoProgress;
174 return progress;
178 * Returns true if there's any progress to report.
180 bool HasProgress() const {
181 return mProgress != NoProgress || !mInvalidRect.IsEmpty() ||
182 mFinishedNewFrame;
186 * State.
190 * If we're doing a metadata decode, we only decode the image's headers, which
191 * is enough to determine the image's intrinsic size. A metadata decode is
192 * enabled by calling SetMetadataDecode() *before* calling Init().
194 void SetMetadataDecode(bool aMetadataDecode) {
195 MOZ_ASSERT(!mInitialized, "Shouldn't be initialized yet");
196 mMetadataDecode = aMetadataDecode;
198 bool IsMetadataDecode() const { return mMetadataDecode; }
201 * Sets the output size of this decoder. If this is smaller than the intrinsic
202 * size of the image, we'll downscale it while decoding. For memory usage
203 * reasons, upscaling is forbidden and will trigger assertions in debug
204 * builds.
206 * Not calling SetOutputSize() means that we should just decode at the
207 * intrinsic size, whatever it is.
209 * If SetOutputSize() was called, ExplicitOutputSize() can be used to
210 * determine the value that was passed to it.
212 * This must be called before Init() is called.
214 void SetOutputSize(const OrientedIntSize& aSize);
217 * @return the output size of this decoder. If this is smaller than the
218 * intrinsic size, then the image will be downscaled during the decoding
219 * process.
221 * Illegal to call if HasSize() returns false.
223 OrientedIntSize OutputSize() const {
224 MOZ_ASSERT(HasSize());
225 return *mOutputSize;
229 * @return either the size passed to SetOutputSize() or Nothing(), indicating
230 * that SetOutputSize() was not explicitly called.
232 Maybe<OrientedIntSize> ExplicitOutputSize() const;
235 * Sets the expected image size of this decoder. Decoding will fail if this
236 * does not match.
238 void SetExpectedSize(const OrientedIntSize& aSize) {
239 mExpectedSize.emplace(aSize);
243 * Is the image size what was expected, if specified?
245 bool IsExpectedSize() const {
246 return mExpectedSize.isNothing() || *mExpectedSize == Size();
250 * Set an iterator to the SourceBuffer which will feed data to this decoder.
251 * This must always be called before calling Init(). (And only before Init().)
253 * XXX(seth): We should eliminate this method and pass a SourceBufferIterator
254 * to the various decoder constructors instead.
256 void SetIterator(SourceBufferIterator&& aIterator) {
257 MOZ_ASSERT(!mInitialized, "Shouldn't be initialized yet");
258 mIterator.emplace(std::move(aIterator));
261 SourceBuffer* GetSourceBuffer() const { return mIterator->Owner(); }
264 * Should this decoder send partial invalidations?
266 bool ShouldSendPartialInvalidations() const {
267 return !(mDecoderFlags & DecoderFlags::IS_REDECODE);
271 * Should we stop decoding after the first frame?
273 bool IsFirstFrameDecode() const {
274 return bool(mDecoderFlags & DecoderFlags::FIRST_FRAME_ONLY);
278 * @return the number of complete animation frames which have been decoded so
279 * far, if it has changed since the last call to TakeCompleteFrameCount();
280 * otherwise, returns Nothing().
282 Maybe<uint32_t> TakeCompleteFrameCount();
284 // The number of frames we have, including anything in-progress. Thus, this
285 // is only 0 if we haven't begun any frames.
286 uint32_t GetFrameCount() { return mFrameCount; }
288 // Did we discover that the image we're decoding is animated?
289 bool HasAnimation() const { return mImageMetadata.HasAnimation(); }
291 // Error tracking
292 bool HasError() const { return mError; }
293 bool ShouldReportError() const { return mShouldReportError; }
295 // Finalize frames
296 void SetFinalizeFrames(bool aFinalize) { mFinalizeFrames = aFinalize; }
297 bool GetFinalizeFrames() const { return mFinalizeFrames; }
299 /// Did we finish decoding enough that calling Decode() again would be
300 /// useless?
301 bool GetDecodeDone() const {
302 return mReachedTerminalState || mDecodeDone ||
303 (mMetadataDecode && HasSize()) || HasError();
306 /// Are we in the middle of a frame right now? Used for assertions only.
307 bool InFrame() const { return mInFrame; }
309 /// Is the image valid if embedded inside an ICO.
310 virtual bool IsValidICOResource() const { return false; }
312 /// Type of decoder.
313 virtual DecoderType GetType() const { return DecoderType::UNKNOWN; }
315 enum DecodeStyle {
316 PROGRESSIVE, // produce intermediate frames representing the partial
317 // state of the image
318 SEQUENTIAL // decode to final image immediately
322 * Get or set the DecoderFlags that influence the behavior of this decoder.
324 void SetDecoderFlags(DecoderFlags aDecoderFlags) {
325 MOZ_ASSERT(!mInitialized);
326 mDecoderFlags = aDecoderFlags;
328 DecoderFlags GetDecoderFlags() const { return mDecoderFlags; }
331 * Get or set the SurfaceFlags that select the kind of output this decoder
332 * will produce.
334 void SetSurfaceFlags(SurfaceFlags aSurfaceFlags);
335 SurfaceFlags GetSurfaceFlags() const { return mSurfaceFlags; }
337 /// @return true if we know the intrinsic size of the image we're decoding.
338 bool HasSize() const { return mImageMetadata.HasSize(); }
341 * @return the intrinsic size of the image we're decoding.
343 * Illegal to call if HasSize() returns false.
345 OrientedIntSize Size() const {
346 MOZ_ASSERT(HasSize());
347 return mImageMetadata.GetSize();
351 * @return an IntRect which covers the entire area of this image at its
352 * intrinsic size, appropriate for use as a frame rect when the image itself
353 * does not specify one.
355 * Illegal to call if HasSize() returns false.
357 OrientedIntRect FullFrame() const {
358 return OrientedIntRect(OrientedIntPoint(), Size());
362 * @return an IntRect which covers the entire area of this image at its size
363 * after scaling - that is, at its output size.
365 * XXX(seth): This is only used for decoders which are using the old
366 * Downscaler code instead of SurfacePipe, since the old AllocateFrame() and
367 * Downscaler APIs required that the frame rect be specified in output space.
368 * We should remove this once all decoders use SurfacePipe.
370 * Illegal to call if HasSize() returns false.
372 OrientedIntRect FullOutputFrame() const {
373 return OrientedIntRect(OrientedIntPoint(), OutputSize());
377 * @return the orientation of the image.
379 * Illegal to call if HasSize() returns false.
381 Orientation GetOrientation() const {
382 MOZ_ASSERT(HasSize());
383 return mImageMetadata.GetOrientation();
386 /// @return final status information about this decoder. Should be called
387 /// after we decide we're not going to run the decoder anymore.
388 DecoderFinalStatus FinalStatus() const;
390 /// @return the metadata we collected about this image while decoding.
391 const ImageMetadata& GetImageMetadata() { return mImageMetadata; }
393 /// @return performance telemetry we collected while decoding.
394 DecoderTelemetry Telemetry() const;
397 * @return a weak pointer to the image associated with this decoder. Illegal
398 * to call if this decoder is not associated with an image.
400 NotNull<RasterImage*> GetImage() const { return WrapNotNull(mImage.get()); }
403 * @return a possibly-null weak pointer to the image associated with this
404 * decoder. May be called even if this decoder is not associated with an
405 * image.
407 RasterImage* GetImageMaybeNull() const { return mImage.get(); }
409 RawAccessFrameRef GetCurrentFrameRef() {
410 return mCurrentFrame ? mCurrentFrame->RawAccessRef() : RawAccessFrameRef();
414 * For use during decoding only. Allows the BlendAnimationFilter to get the
415 * current frame we are producing for its animation parameters.
417 imgFrame* GetCurrentFrame() { return mCurrentFrame.get(); }
420 * For use during decoding only. Allows the BlendAnimationFilter to get the
421 * frame it should be pulling the previous frame data from.
423 const RawAccessFrameRef& GetRestoreFrameRef() const { return mRestoreFrame; }
425 const gfx::IntRect& GetRestoreDirtyRect() const { return mRestoreDirtyRect; }
427 const gfx::IntRect& GetRecycleRect() const { return mRecycleRect; }
429 const gfx::IntRect& GetFirstFrameRefreshArea() const {
430 return mFirstFrameRefreshArea;
433 bool HasFrameToTake() const { return mHasFrameToTake; }
434 void ClearHasFrameToTake() {
435 MOZ_ASSERT(mHasFrameToTake);
436 mHasFrameToTake = false;
439 IDecoderFrameRecycler* GetFrameRecycler() const { return mFrameRecycler; }
440 void SetFrameRecycler(IDecoderFrameRecycler* aFrameRecycler) {
441 mFrameRecycler = aFrameRecycler;
444 protected:
445 friend class AutoRecordDecoderTelemetry;
446 friend class DecoderTestHelper;
447 friend class nsBMPDecoder;
448 friend class nsICODecoder;
449 friend class ReorientSurfaceSink;
450 friend class SurfaceSink;
452 virtual ~Decoder();
455 * Internal hooks. Decoder implementations may override these and
456 * only these methods.
458 * BeforeFinishInternal() can be used to detect if decoding is in an
459 * incomplete state, e.g. due to file truncation, in which case it should
460 * return a failing nsresult.
462 virtual nsresult InitInternal();
463 virtual LexerResult DoDecode(SourceBufferIterator& aIterator,
464 IResumable* aOnResume) = 0;
465 virtual nsresult BeforeFinishInternal();
466 virtual nsresult FinishInternal();
467 virtual nsresult FinishWithErrorInternal();
469 qcms_profile* GetCMSOutputProfile() const;
470 qcms_transform* GetCMSsRGBTransform(gfx::SurfaceFormat aFormat) const;
473 * @return the per-image-format telemetry ID for recording this decoder's
474 * speed, or Nothing() if we don't record speed telemetry for this kind of
475 * decoder.
477 virtual Maybe<Telemetry::HistogramID> SpeedHistogram() const {
478 return Nothing();
482 * Progress notifications.
485 // Called by decoders when they determine the size of the image. Informs
486 // the image of its size and sends notifications.
487 void PostSize(int32_t aWidth, int32_t aHeight, Orientation = Orientation(),
488 Resolution = Resolution());
490 // Called by decoders if they determine that the image has transparency.
492 // This should be fired as early as possible to allow observers to do things
493 // that affect content, so it's necessarily pessimistic - if there's a
494 // possibility that the image has transparency, for example because its header
495 // specifies that it has an alpha channel, we fire PostHasTransparency
496 // immediately. PostFrameStop's aFrameOpacity argument, on the other hand, is
497 // only used internally to ImageLib. Because PostFrameStop isn't delivered
498 // until the entire frame has been decoded, decoders may take into account the
499 // actual contents of the frame and give a more accurate result.
500 void PostHasTransparency();
502 // Called by decoders if they determine that the image is animated.
504 // @param aTimeout The time for which the first frame should be shown before
505 // we advance to the next frame.
506 void PostIsAnimated(FrameTimeout aFirstFrameTimeout);
508 // Called by decoders when they end a frame. Informs the image, sends
509 // notifications, and does internal book-keeping.
510 // Specify whether this frame is opaque as an optimization.
511 // For animated images, specify the disposal, blend method and timeout for
512 // this frame.
513 void PostFrameStop(Opacity aFrameOpacity = Opacity::SOME_TRANSPARENCY);
516 * Called by the decoders when they have a region to invalidate. We may not
517 * actually pass these invalidations on right away.
519 * @param aRect The invalidation rect in the coordinate system of the unscaled
520 * image (that is, the image at its intrinsic size).
521 * @param aRectAtOutputSize If not Nothing(), the invalidation rect in the
522 * coordinate system of the scaled image (that is,
523 * the image at our output size). This must
524 * be supplied if we're downscaling during decode.
526 void PostInvalidation(
527 const OrientedIntRect& aRect,
528 const Maybe<OrientedIntRect>& aRectAtOutputSize = Nothing());
530 // Called by the decoders when they have successfully decoded the image. This
531 // may occur as the result of the decoder getting to the appropriate point in
532 // the stream, or by us calling FinishInternal().
534 // May not be called mid-frame.
536 // For animated images, specify the loop count. -1 means loop forever, 0
537 // means a single iteration, stopping on the last frame.
538 void PostDecodeDone(int32_t aLoopCount = 0);
541 * Allocates a new frame, making it our current frame if successful.
543 nsresult AllocateFrame(const gfx::IntSize& aOutputSize,
544 gfx::SurfaceFormat aFormat,
545 const Maybe<AnimationParams>& aAnimParams = Nothing());
547 private:
548 /// Report that an error was encountered while decoding.
549 void PostError();
552 * CompleteDecode() finishes up the decoding process after Decode() determines
553 * that we're finished. It records final progress and does all the cleanup
554 * that's possible off-main-thread.
556 void CompleteDecode();
558 /// @return the number of complete frames we have. Does not include the
559 /// current frame if it's unfinished.
560 uint32_t GetCompleteFrameCount() {
561 if (mFrameCount == 0) {
562 return 0;
565 return mInFrame ? mFrameCount - 1 : mFrameCount;
568 RawAccessFrameRef AllocateFrameInternal(
569 const gfx::IntSize& aOutputSize, gfx::SurfaceFormat aFormat,
570 const Maybe<AnimationParams>& aAnimParams,
571 RawAccessFrameRef&& aPreviousFrame);
573 protected:
574 /// Color management profile from the ICCP chunk in the image.
575 qcms_profile* mInProfile;
577 /// Color management transform to apply to image data.
578 qcms_transform* mTransform;
580 uint8_t* mImageData; // Pointer to image data in BGRA/X
581 uint32_t mImageDataLength;
583 CMSMode mCMSMode;
585 private:
586 RefPtr<RasterImage> mImage;
587 Maybe<SourceBufferIterator> mIterator;
588 IDecoderFrameRecycler* mFrameRecycler;
590 // The current frame the decoder is producing.
591 RawAccessFrameRef mCurrentFrame;
593 // The complete frame to combine with the current partial frame to produce
594 // a complete current frame.
595 RawAccessFrameRef mRestoreFrame;
597 ImageMetadata mImageMetadata;
599 OrientedIntRect
600 mInvalidRect; // Tracks new rows as the current frame is decoded.
601 gfx::IntRect mRestoreDirtyRect; // Tracks an invalidation region between the
602 // restore frame and the previous frame.
603 gfx::IntRect mRecycleRect; // Tracks an invalidation region between the
604 // recycled frame and the current frame.
605 Maybe<OrientedIntSize> mOutputSize; // The size of our output surface.
606 Maybe<OrientedIntSize> mExpectedSize; // The expected size of the image.
607 Progress mProgress;
609 uint32_t mFrameCount; // Number of frames, including anything in-progress
610 FrameTimeout mLoopLength; // Length of a single loop of this image.
611 gfx::IntRect
612 mFirstFrameRefreshArea; // The area of the image that needs to
613 // be invalidated when the animation loops.
615 // Telemetry data for this decoder.
616 TimeDuration mDecodeTime;
618 DecoderFlags mDecoderFlags;
619 SurfaceFlags mSurfaceFlags;
621 bool mInitialized : 1;
622 bool mMetadataDecode : 1;
623 bool mHaveExplicitOutputSize : 1;
624 bool mInFrame : 1;
625 bool mFinishedNewFrame : 1; // True if PostFrameStop() has been called since
626 // the last call to TakeCompleteFrameCount().
627 // Has a new frame that AnimationSurfaceProvider can take. Unfortunately this
628 // has to be separate from mFinishedNewFrame because the png decoder yields a
629 // new frame before calling PostFrameStop().
630 bool mHasFrameToTake : 1;
631 bool mReachedTerminalState : 1;
632 bool mDecodeDone : 1;
633 bool mError : 1;
634 bool mShouldReportError : 1;
635 bool mFinalizeFrames : 1;
638 } // namespace image
639 } // namespace mozilla
641 #endif // mozilla_image_Decoder_h