image/Decoder.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "Decoder.h"
   7
   8 #include "DecodePool.h"
   9 #include "GeckoProfiler.h"
  10 #include "IDecodingTask.h"
  11 #include "ISurfaceProvider.h"
  12 #include "mozilla/gfx/2D.h"
  13 #include "mozilla/gfx/Point.h"
  14 #include "mozilla/Telemetry.h"
  15 #include "nsComponentManagerUtils.h"
  16 #include "nsProxyRelease.h"
  17 #include "nsServiceManagerUtils.h"
  18
  19 using mozilla::gfx::IntPoint;
  20 using mozilla::gfx::IntRect;
  21 using mozilla::gfx::IntSize;
  22 using mozilla::gfx::SurfaceFormat;
  23
  24 namespace mozilla {
  25 namespace image {
  26
  27 class MOZ_STACK_CLASS AutoRecordDecoderTelemetry final {
  28  public:
  29   explicit AutoRecordDecoderTelemetry(Decoder* aDecoder) : mDecoder(aDecoder) {
  30     MOZ_ASSERT(mDecoder);
  31
  32     // Begin recording telemetry data.
  33     mStartTime = TimeStamp::Now();
  34   }
  35
  36   ~AutoRecordDecoderTelemetry() {
  37     // Finish telemetry.
  38     mDecoder->mDecodeTime += (TimeStamp::Now() - mStartTime);
  39   }
  40
  41  private:
  42   Decoder* mDecoder;
  43   TimeStamp mStartTime;
  44 };
  45
  46 Decoder::Decoder(RasterImage* aImage)
  47     : mImageData(nullptr),
  48       mImageDataLength(0),
  49       mColormap(nullptr),
  50       mColormapSize(0),
  51       mImage(aImage),
  52       mFrameRecycler(nullptr),
  53       mProgress(NoProgress),
  54       mFrameCount(0),
  55       mLoopLength(FrameTimeout::Zero()),
  56       mDecoderFlags(DefaultDecoderFlags()),
  57       mSurfaceFlags(DefaultSurfaceFlags()),
  58       mInitialized(false),
  59       mMetadataDecode(false),
  60       mHaveExplicitOutputSize(false),
  61       mInFrame(false),
  62       mFinishedNewFrame(false),
  63       mHasFrameToTake(false),
  64       mReachedTerminalState(false),
  65       mDecodeDone(false),
  66       mError(false),
  67       mShouldReportError(false),
  68       mFinalizeFrames(true) {}
  69
  70 Decoder::~Decoder() {
  71   MOZ_ASSERT(mProgress == NoProgress || !mImage,
  72              "Destroying Decoder without taking all its progress changes");
  73   MOZ_ASSERT(mInvalidRect.IsEmpty() || !mImage,
  74              "Destroying Decoder without taking all its invalidations");
  75   mInitialized = false;
  76
  77   if (mImage && !NS_IsMainThread()) {
  78     // Dispatch mImage to main thread to prevent it from being destructed by the
  79     // decode thread.
  80     NS_ReleaseOnMainThreadSystemGroup(mImage.forget());
  81   }
  82 }
  83
  84 /*
  85  * Common implementation of the decoder interface.
  86  */
  87
  88 nsresult Decoder::Init() {
  89   // No re-initializing
  90   MOZ_ASSERT(!mInitialized, "Can't re-initialize a decoder!");
  91
  92   // All decoders must have a SourceBufferIterator.
  93   MOZ_ASSERT(mIterator);
  94
  95   // Metadata decoders must not set an output size.
  96   MOZ_ASSERT_IF(mMetadataDecode, !mHaveExplicitOutputSize);
  97
  98   // All decoders must be anonymous except for metadata decoders.
  99   // XXX(seth): Soon that exception will be removed.
 100   MOZ_ASSERT_IF(mImage, IsMetadataDecode());
 101
 102   // Implementation-specific initialization.
 103   nsresult rv = InitInternal();
 104
 105   mInitialized = true;
 106
 107   return rv;
 108 }
 109
 110 LexerResult Decoder::Decode(IResumable* aOnResume /* = nullptr */) {
 111   MOZ_ASSERT(mInitialized, "Should be initialized here");
 112   MOZ_ASSERT(mIterator, "Should have a SourceBufferIterator");
 113
 114   // If we're already done, don't attempt to keep decoding.
 115   if (GetDecodeDone()) {
 116     return LexerResult(HasError() ? TerminalState::FAILURE
 117                                   : TerminalState::SUCCESS);
 118   }
 119
 120   LexerResult lexerResult(TerminalState::FAILURE);
 121   {
 122     AUTO_PROFILER_LABEL_CATEGORY_PAIR(GRAPHICS_ImageDecoding);
 123     AutoRecordDecoderTelemetry telemetry(this);
 124
 125     lexerResult = DoDecode(*mIterator, aOnResume);
 126   };
 127
 128   if (lexerResult.is<Yield>()) {
 129     // We either need more data to continue (in which case either @aOnResume or
 130     // the caller will reschedule us to run again later), or the decoder is
 131     // yielding to allow the caller access to some intermediate output.
 132     return lexerResult;
 133   }
 134
 135   // We reached a terminal state; we're now done decoding.
 136   MOZ_ASSERT(lexerResult.is<TerminalState>());
 137   mReachedTerminalState = true;
 138
 139   // If decoding failed, record that fact.
 140   if (lexerResult.as<TerminalState>() == TerminalState::FAILURE) {
 141     PostError();
 142   }
 143
 144   // Perform final cleanup.
 145   CompleteDecode();
 146
 147   return LexerResult(HasError() ? TerminalState::FAILURE
 148                                 : TerminalState::SUCCESS);
 149 }
 150
 151 LexerResult Decoder::TerminateFailure() {
 152   PostError();
 153
 154   // Perform final cleanup if need be.
 155   if (!mReachedTerminalState) {
 156     mReachedTerminalState = true;
 157     CompleteDecode();
 158   }
 159
 160   return LexerResult(TerminalState::FAILURE);
 161 }
 162
 163 bool Decoder::ShouldSyncDecode(size_t aByteLimit) {
 164   MOZ_ASSERT(aByteLimit > 0);
 165   MOZ_ASSERT(mIterator, "Should have a SourceBufferIterator");
 166
 167   return mIterator->RemainingBytesIsNoMoreThan(aByteLimit);
 168 }
 169
 170 void Decoder::CompleteDecode() {
 171   // Implementation-specific finalization.
 172   nsresult rv = BeforeFinishInternal();
 173   if (NS_FAILED(rv)) {
 174     PostError();
 175   }
 176
 177   rv = HasError() ? FinishWithErrorInternal() : FinishInternal();
 178   if (NS_FAILED(rv)) {
 179     PostError();
 180   }
 181
 182   if (IsMetadataDecode()) {
 183     // If this was a metadata decode and we never got a size, the decode failed.
 184     if (!HasSize()) {
 185       PostError();
 186     }
 187     return;
 188   }
 189
 190   // If the implementation left us mid-frame, finish that up. Note that it may
 191   // have left us transparent.
 192   if (mInFrame) {
 193     PostHasTransparency();
 194     PostFrameStop();
 195   }
 196
 197   // If PostDecodeDone() has not been called, we may need to send teardown
 198   // notifications if it is unrecoverable.
 199   if (!mDecodeDone) {
 200     // We should always report an error to the console in this case.
 201     mShouldReportError = true;
 202
 203     if (GetCompleteFrameCount() > 0) {
 204       // We're usable if we have at least one complete frame, so do exactly
 205       // what we should have when the decoder completed.
 206       PostHasTransparency();
 207       PostDecodeDone();
 208     } else {
 209       // We're not usable. Record some final progress indicating the error.
 210       mProgress |= FLAG_DECODE_COMPLETE | FLAG_HAS_ERROR;
 211     }
 212   }
 213
 214   if (mDecodeDone) {
 215     MOZ_ASSERT(HasError() || mCurrentFrame, "Should have an error or a frame");
 216
 217     // If this image wasn't animated and isn't a transient image, mark its frame
 218     // as optimizable. We don't support optimizing animated images and
 219     // optimizing transient images isn't worth it.
 220     if (!HasAnimation() &&
 221         !(mDecoderFlags & DecoderFlags::IMAGE_IS_TRANSIENT) && mCurrentFrame) {
 222       mCurrentFrame->SetOptimizable();
 223     }
 224   }
 225 }
 226
 227 void Decoder::SetOutputSize(const gfx::IntSize& aSize) {
 228   mOutputSize = Some(aSize);
 229   mHaveExplicitOutputSize = true;
 230 }
 231
 232 Maybe<gfx::IntSize> Decoder::ExplicitOutputSize() const {
 233   MOZ_ASSERT_IF(mHaveExplicitOutputSize, mOutputSize);
 234   return mHaveExplicitOutputSize ? mOutputSize : Nothing();
 235 }
 236
 237 Maybe<uint32_t> Decoder::TakeCompleteFrameCount() {
 238   const bool finishedNewFrame = mFinishedNewFrame;
 239   mFinishedNewFrame = false;
 240   return finishedNewFrame ? Some(GetCompleteFrameCount()) : Nothing();
 241 }
 242
 243 DecoderFinalStatus Decoder::FinalStatus() const {
 244   return DecoderFinalStatus(IsMetadataDecode(), GetDecodeDone(), HasError(),
 245                             ShouldReportError());
 246 }
 247
 248 DecoderTelemetry Decoder::Telemetry() const {
 249   MOZ_ASSERT(mIterator);
 250   return DecoderTelemetry(SpeedHistogram(),
 251                           mIterator ? mIterator->ByteCount() : 0,
 252                           mIterator ? mIterator->ChunkCount() : 0, mDecodeTime);
 253 }
 254
 255 nsresult Decoder::AllocateFrame(const gfx::IntSize& aOutputSize,
 256                                 const gfx::IntRect& aFrameRect,
 257                                 gfx::SurfaceFormat aFormat,
 258                                 uint8_t aPaletteDepth,
 259                                 const Maybe<AnimationParams>& aAnimParams) {
 260   mCurrentFrame =
 261       AllocateFrameInternal(aOutputSize, aFrameRect, aFormat, aPaletteDepth,
 262                             aAnimParams, std::move(mCurrentFrame));
 263
 264   if (mCurrentFrame) {
 265     mHasFrameToTake = true;
 266
 267     // Gather the raw pointers the decoders will use.
 268     mCurrentFrame->GetImageData(&mImageData, &mImageDataLength);
 269     mCurrentFrame->GetPaletteData(&mColormap, &mColormapSize);
 270
 271     // We should now be on |aFrameNum|. (Note that we're comparing the frame
 272     // number, which is zero-based, with the frame count, which is one-based.)
 273     MOZ_ASSERT_IF(aAnimParams, aAnimParams->mFrameNum + 1 == mFrameCount);
 274
 275     // If we're past the first frame, PostIsAnimated() should've been called.
 276     MOZ_ASSERT_IF(mFrameCount > 1, HasAnimation());
 277
 278     // Update our state to reflect the new frame.
 279     MOZ_ASSERT(!mInFrame, "Starting new frame but not done with old one!");
 280     mInFrame = true;
 281   }
 282
 283   return mCurrentFrame ? NS_OK : NS_ERROR_FAILURE;
 284 }
 285
 286 RawAccessFrameRef Decoder::AllocateFrameInternal(
 287     const gfx::IntSize& aOutputSize, const gfx::IntRect& aFrameRect,
 288     SurfaceFormat aFormat, uint8_t aPaletteDepth,
 289     const Maybe<AnimationParams>& aAnimParams,
 290     RawAccessFrameRef&& aPreviousFrame) {
 291   if (HasError()) {
 292     return RawAccessFrameRef();
 293   }
 294
 295   uint32_t frameNum = aAnimParams ? aAnimParams->mFrameNum : 0;
 296   if (frameNum != mFrameCount) {
 297     MOZ_ASSERT_UNREACHABLE("Allocating frames out of order");
 298     return RawAccessFrameRef();
 299   }
 300
 301   if (aOutputSize.width <= 0 || aOutputSize.height <= 0 ||
 302       aFrameRect.Width() <= 0 || aFrameRect.Height() <= 0) {
 303     NS_WARNING("Trying to add frame with zero or negative size");
 304     return RawAccessFrameRef();
 305   }
 306
 307   if (frameNum == 1) {
 308     MOZ_ASSERT(aPreviousFrame, "Must provide a previous frame when animated");
 309     aPreviousFrame->SetRawAccessOnly();
 310   }
 311
 312   if (frameNum > 0) {
 313     if (ShouldBlendAnimation()) {
 314       if (aPreviousFrame->GetDisposalMethod() !=
 315           DisposalMethod::RESTORE_PREVIOUS) {
 316         // If the new restore frame is the direct previous frame, then we know
 317         // the dirty rect is composed only of the current frame's blend rect and
 318         // the restore frame's clear rect (if applicable) which are handled in
 319         // filters.
 320         mRestoreFrame = std::move(aPreviousFrame);
 321         mRestoreDirtyRect.SetBox(0, 0, 0, 0);
 322       } else {
 323         // We only need the previous frame's dirty rect, because while there may
 324         // have been several frames between us and mRestoreFrame, the only areas
 325         // that changed are the restore frame's clear rect, the current frame
 326         // blending rect, and the previous frame's blending rect. All else is
 327         // forgotten due to us restoring the same frame again.
 328         mRestoreDirtyRect = aPreviousFrame->GetBoundedBlendRect();
 329       }
 330     }
 331   }
 332
 333   RawAccessFrameRef ref;
 334
 335   // If we have a frame recycler, it must be for an animated image producing
 336   // full frames. If the higher layers are discarding frames because of the
 337   // memory footprint, then the recycler will allow us to reuse the buffers.
 338   // Each frame should be the same size and have mostly the same properties.
 339   if (mFrameRecycler) {
 340     MOZ_ASSERT(ShouldBlendAnimation());
 341     MOZ_ASSERT(aPaletteDepth == 0);
 342     MOZ_ASSERT(aAnimParams);
 343     MOZ_ASSERT(aFrameRect.IsEqualEdges(IntRect(IntPoint(0, 0), aOutputSize)));
 344
 345     ref = mFrameRecycler->RecycleFrame(mRecycleRect);
 346     if (ref) {
 347       // If the recycled frame is actually the current restore frame, we cannot
 348       // use it. If the next restore frame is the new frame we are creating, in
 349       // theory we could reuse it, but we would need to store the restore frame
 350       // animation parameters elsewhere. For now we just drop it.
 351       bool blocked = ref.get() == mRestoreFrame.get();
 352       if (!blocked) {
 353         blocked = NS_FAILED(ref->InitForDecoderRecycle(aAnimParams.ref()));
 354       }
 355
 356       if (blocked) {
 357         ref.reset();
 358       }
 359     }
 360   }
 361
 362   // Either the recycler had nothing to give us, or we don't have a recycler.
 363   // Produce a new frame to store the data.
 364   if (!ref) {
 365     // There is no underlying data to reuse, so reset the recycle rect to be
 366     // the full frame, to ensure the restore frame is fully copied.
 367     mRecycleRect = IntRect(IntPoint(0, 0), aOutputSize);
 368
 369     bool nonPremult = bool(mSurfaceFlags & SurfaceFlags::NO_PREMULTIPLY_ALPHA);
 370     auto frame = MakeNotNull<RefPtr<imgFrame>>();
 371     if (NS_FAILED(frame->InitForDecoder(
 372             aOutputSize, aFrameRect, aFormat, aPaletteDepth, nonPremult,
 373             aAnimParams, ShouldBlendAnimation(), bool(mFrameRecycler)))) {
 374       NS_WARNING("imgFrame::Init should succeed");
 375       return RawAccessFrameRef();
 376     }
 377
 378     ref = frame->RawAccessRef();
 379     if (!ref) {
 380       frame->Abort();
 381       return RawAccessFrameRef();
 382     }
 383
 384     if (frameNum > 0) {
 385       frame->SetRawAccessOnly();
 386     }
 387   }
 388
 389   mFrameCount++;
 390
 391   return ref;
 392 }
 393
 394 /*
 395  * Hook stubs. Override these as necessary in decoder implementations.
 396  */
 397
 398 nsresult Decoder::InitInternal() { return NS_OK; }
 399 nsresult Decoder::BeforeFinishInternal() { return NS_OK; }
 400 nsresult Decoder::FinishInternal() { return NS_OK; }
 401
 402 nsresult Decoder::FinishWithErrorInternal() {
 403   MOZ_ASSERT(!mInFrame);
 404   return NS_OK;
 405 }
 406
 407 /*
 408  * Progress Notifications
 409  */
 410
 411 void Decoder::PostSize(int32_t aWidth, int32_t aHeight,
 412                        Orientation aOrientation /* = Orientation()*/) {
 413   // Validate.
 414   MOZ_ASSERT(aWidth >= 0, "Width can't be negative!");
 415   MOZ_ASSERT(aHeight >= 0, "Height can't be negative!");
 416
 417   // Set our intrinsic size.
 418   mImageMetadata.SetSize(aWidth, aHeight, aOrientation);
 419
 420   // Verify it is the expected size, if given. Note that this is only used by
 421   // the ICO decoder for embedded image types, so only its subdecoders are
 422   // required to handle failures in PostSize.
 423   if (!IsExpectedSize()) {
 424     PostError();
 425     return;
 426   }
 427
 428   // Set our output size if it's not already set.
 429   if (!mOutputSize) {
 430     mOutputSize = Some(IntSize(aWidth, aHeight));
 431   }
 432
 433   MOZ_ASSERT(mOutputSize->width <= aWidth && mOutputSize->height <= aHeight,
 434              "Output size will result in upscaling");
 435
 436   // Create a downscaler if we need to downscale. This is used by legacy
 437   // decoders that haven't been converted to use SurfacePipe yet.
 438   // XXX(seth): Obviously, we'll remove this once all decoders use SurfacePipe.
 439   if (mOutputSize->width < aWidth || mOutputSize->height < aHeight) {
 440     mDownscaler.emplace(*mOutputSize);
 441   }
 442
 443   // Record this notification.
 444   mProgress |= FLAG_SIZE_AVAILABLE;
 445 }
 446
 447 void Decoder::PostHasTransparency() { mProgress |= FLAG_HAS_TRANSPARENCY; }
 448
 449 void Decoder::PostIsAnimated(FrameTimeout aFirstFrameTimeout) {
 450   mProgress |= FLAG_IS_ANIMATED;
 451   mImageMetadata.SetHasAnimation();
 452   mImageMetadata.SetFirstFrameTimeout(aFirstFrameTimeout);
 453 }
 454
 455 void Decoder::PostFrameStop(Opacity aFrameOpacity) {
 456   // We should be mid-frame
 457   MOZ_ASSERT(!IsMetadataDecode(), "Stopping frame during metadata decode");
 458   MOZ_ASSERT(mInFrame, "Stopping frame when we didn't start one");
 459   MOZ_ASSERT(mCurrentFrame, "Stopping frame when we don't have one");
 460
 461   // Update our state.
 462   mInFrame = false;
 463   mFinishedNewFrame = true;
 464
 465   mCurrentFrame->Finish(aFrameOpacity, mFinalizeFrames);
 466
 467   mProgress |= FLAG_FRAME_COMPLETE;
 468
 469   mLoopLength += mCurrentFrame->GetTimeout();
 470
 471   if (mFrameCount == 1) {
 472     // If we're not sending partial invalidations, then we send an invalidation
 473     // here when the first frame is complete.
 474     if (!ShouldSendPartialInvalidations()) {
 475       mInvalidRect.UnionRect(mInvalidRect, IntRect(IntPoint(), Size()));
 476     }
 477
 478     // If we dispose of the first frame by clearing it, then the first frame's
 479     // refresh area is all of itself. RESTORE_PREVIOUS is invalid (assumed to
 480     // be DISPOSE_CLEAR).
 481     switch (mCurrentFrame->GetDisposalMethod()) {
 482       default:
 483         MOZ_FALLTHROUGH_ASSERT("Unexpected DisposalMethod");
 484       case DisposalMethod::CLEAR:
 485       case DisposalMethod::CLEAR_ALL:
 486       case DisposalMethod::RESTORE_PREVIOUS:
 487         mFirstFrameRefreshArea = IntRect(IntPoint(), Size());
 488         break;
 489       case DisposalMethod::KEEP:
 490       case DisposalMethod::NOT_SPECIFIED:
 491         break;
 492     }
 493   } else {
 494     // Some GIFs are huge but only have a small area that they animate. We only
 495     // need to refresh that small area when frame 0 comes around again.
 496     mFirstFrameRefreshArea.UnionRect(mFirstFrameRefreshArea,
 497                                      mCurrentFrame->GetBoundedBlendRect());
 498   }
 499 }
 500
 501 void Decoder::PostInvalidation(const gfx::IntRect& aRect,
 502                                const Maybe<gfx::IntRect>& aRectAtOutputSize
 503                                /* = Nothing() */) {
 504   // We should be mid-frame
 505   MOZ_ASSERT(mInFrame, "Can't invalidate when not mid-frame!");
 506   MOZ_ASSERT(mCurrentFrame, "Can't invalidate when not mid-frame!");
 507
 508   // Record this invalidation, unless we're not sending partial invalidations
 509   // or we're past the first frame.
 510   if (ShouldSendPartialInvalidations() && mFrameCount == 1) {
 511     mInvalidRect.UnionRect(mInvalidRect, aRect);
 512     mCurrentFrame->ImageUpdated(aRectAtOutputSize.valueOr(aRect));
 513   }
 514 }
 515
 516 void Decoder::PostDecodeDone(int32_t aLoopCount /* = 0 */) {
 517   MOZ_ASSERT(!IsMetadataDecode(), "Done with decoding in metadata decode");
 518   MOZ_ASSERT(!mInFrame, "Can't be done decoding if we're mid-frame!");
 519   MOZ_ASSERT(!mDecodeDone, "Decode already done!");
 520   mDecodeDone = true;
 521
 522   mImageMetadata.SetLoopCount(aLoopCount);
 523
 524   // Some metadata that we track should take into account every frame in the
 525   // image. If this is a first-frame-only decode, our accumulated loop length
 526   // and first frame refresh area only includes the first frame, so it's not
 527   // correct and we don't record it.
 528   if (!IsFirstFrameDecode()) {
 529     mImageMetadata.SetLoopLength(mLoopLength);
 530     mImageMetadata.SetFirstFrameRefreshArea(mFirstFrameRefreshArea);
 531   }
 532
 533   mProgress |= FLAG_DECODE_COMPLETE;
 534 }
 535
 536 void Decoder::PostError() {
 537   mError = true;
 538
 539   if (mInFrame) {
 540     MOZ_ASSERT(mCurrentFrame);
 541     MOZ_ASSERT(mFrameCount > 0);
 542     mCurrentFrame->Abort();
 543     mInFrame = false;
 544     --mFrameCount;
 545     mHasFrameToTake = false;
 546   }
 547 }
 548
 549 }  // namespace image
 550 }  // namespace mozilla