image/decoders/nsAVIFDecoder.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  *
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef mozilla_image_decoders_nsAVIFDecoder_h
   8 #define mozilla_image_decoders_nsAVIFDecoder_h
   9
  10 #include "Decoder.h"
  11 #include "mozilla/gfx/Types.h"
  12 #include "MP4Metadata.h"
  13 #include "mp4parse.h"
  14 #include "SampleIterator.h"
  15 #include "SurfacePipe.h"
  16
  17 #include <aom/aom_decoder.h>
  18 #include "dav1d/dav1d.h"
  19
  20 #include "mozilla/Telemetry.h"
  21
  22 namespace mozilla {
  23 namespace image {
  24 class RasterImage;
  25 class AVIFDecoderStream;
  26 class AVIFParser;
  27 class AVIFDecoderInterface;
  28
  29 class nsAVIFDecoder final : public Decoder {
  30  public:
  31   virtual ~nsAVIFDecoder();
  32
  33   DecoderType GetType() const override { return DecoderType::AVIF; }
  34
  35  protected:
  36   LexerResult DoDecode(SourceBufferIterator& aIterator,
  37                        IResumable* aOnResume) override;
  38   Maybe<Telemetry::HistogramID> SpeedHistogram() const override;
  39
  40  private:
  41   friend class DecoderFactory;
  42   friend class AVIFDecoderInterface;
  43   friend class AVIFParser;
  44
  45   // Decoders should only be instantiated via DecoderFactory.
  46   explicit nsAVIFDecoder(RasterImage* aImage);
  47
  48   static intptr_t ReadSource(uint8_t* aDestBuf, uintptr_t aDestBufSize,
  49                              void* aUserData);
  50
  51   typedef int Dav1dResult;
  52   enum class NonAOMCodecError { NoFrame, SizeOverflow };
  53   typedef Variant<aom_codec_err_t, NonAOMCodecError> AOMResult;
  54   enum class NonDecoderResult {
  55     NeedMoreData,
  56     OutputAvailable,
  57     Complete,
  58     SizeOverflow,
  59     OutOfMemory,
  60     PipeInitError,
  61     WriteBufferError,
  62     AlphaYSizeMismatch,
  63     AlphaYColorDepthMismatch,
  64     MetadataImageSizeMismatch,
  65     RenderSizeMismatch,
  66     FrameSizeChanged,
  67     InvalidCICP,
  68     NoSamples,
  69   };
  70   using DecodeResult =
  71       Variant<Mp4parseStatus, NonDecoderResult, Dav1dResult, AOMResult>;
  72   Mp4parseStatus CreateParser();
  73   DecodeResult CreateDecoder();
  74   DecodeResult DoDecodeInternal(SourceBufferIterator& aIterator,
  75                                 IResumable* aOnResume);
  76
  77   static bool IsDecodeSuccess(const DecodeResult& aResult);
  78
  79   void RecordDecodeResultTelemetry(const DecodeResult& aResult);
  80
  81   Vector<uint8_t> mBufferedData;
  82   RefPtr<AVIFDecoderStream> mBufferStream;
  83
  84   /// Pointer to the next place to read from mBufferedData
  85   const uint8_t* mReadCursor = nullptr;
  86
  87   UniquePtr<AVIFParser> mParser = nullptr;
  88   UniquePtr<AVIFDecoderInterface> mDecoder = nullptr;
  89
  90   bool mIsAnimated = false;
  91   bool mHasAlpha = false;
  92 };
  93
  94 class AVIFDecoderStream : public ByteStream {
  95  public:
  96   explicit AVIFDecoderStream(Vector<uint8_t>* aBuffer) { mBuffer = aBuffer; }
  97
  98   virtual bool ReadAt(int64_t offset, void* data, size_t size,
  99                       size_t* bytes_read) override;
 100   virtual bool CachedReadAt(int64_t offset, void* data, size_t size,
 101                             size_t* bytes_read) override {
 102     return ReadAt(offset, data, size, bytes_read);
 103   };
 104   virtual bool Length(int64_t* size) override;
 105   virtual const uint8_t* GetContiguousAccess(int64_t aOffset,
 106                                              size_t aSize) override;
 107
 108  private:
 109   Vector<uint8_t>* mBuffer;
 110 };
 111
 112 struct AVIFImage {
 113   uint32_t mFrameNum = 0;
 114   FrameTimeout mDuration = FrameTimeout::Zero();
 115   RefPtr<MediaRawData> mColorImage = nullptr;
 116   RefPtr<MediaRawData> mAlphaImage = nullptr;
 117 };
 118
 119 class AVIFParser {
 120  public:
 121   static Mp4parseStatus Create(const Mp4parseIo* aIo, ByteStream* aBuffer,
 122                                UniquePtr<AVIFParser>& aParserOut,
 123                                bool aAllowSequences, bool aAnimateAVIFMajor);
 124
 125   ~AVIFParser();
 126
 127   const Mp4parseAvifInfo& GetInfo() const { return mInfo; }
 128
 129   nsAVIFDecoder::DecodeResult GetImage(AVIFImage& aImage);
 130
 131   bool IsAnimated() const;
 132
 133  private:
 134   explicit AVIFParser(const Mp4parseIo* aIo);
 135
 136   Mp4parseStatus Init(ByteStream* aBuffer, bool aAllowSequences,
 137                       bool aAnimateAVIFMajor);
 138
 139   struct FreeAvifParser {
 140     void operator()(Mp4parseAvifParser* aPtr) { mp4parse_avif_free(aPtr); }
 141   };
 142
 143   const Mp4parseIo* mIo;
 144   UniquePtr<Mp4parseAvifParser, FreeAvifParser> mParser = nullptr;
 145   Mp4parseAvifInfo mInfo = {};
 146
 147   UniquePtr<SampleIterator> mColorSampleIter = nullptr;
 148   UniquePtr<SampleIterator> mAlphaSampleIter = nullptr;
 149   uint32_t mFrameNum = 0;
 150 };
 151
 152 struct Dav1dPictureUnref {
 153   void operator()(Dav1dPicture* aPtr) {
 154     dav1d_picture_unref(aPtr);
 155     delete aPtr;
 156   }
 157 };
 158
 159 using OwnedDav1dPicture = UniquePtr<Dav1dPicture, Dav1dPictureUnref>;
 160
 161 class OwnedAOMImage {
 162  public:
 163   ~OwnedAOMImage();
 164
 165   static OwnedAOMImage* CopyFrom(aom_image_t* aImage, bool aIsAlpha);
 166
 167   aom_image_t* GetImage() { return mImage.isSome() ? mImage.ptr() : nullptr; }
 168
 169  private:
 170   OwnedAOMImage();
 171
 172   bool CloneFrom(aom_image_t* aImage, bool aIsAlpha);
 173
 174   // The mImage's planes are referenced to mBuffer
 175   Maybe<aom_image_t> mImage;
 176   UniquePtr<uint8_t[]> mBuffer;
 177 };
 178
 179 struct AVIFDecodedData : layers::PlanarYCbCrData {
 180  public:
 181   Maybe<OrientedIntSize> mRenderSize = Nothing();
 182   gfx::CICP::ColourPrimaries mColourPrimaries = gfx::CICP::CP_UNSPECIFIED;
 183   gfx::CICP::TransferCharacteristics mTransferCharacteristics =
 184       gfx::CICP::TC_UNSPECIFIED;
 185   gfx::CICP::MatrixCoefficients mMatrixCoefficients = gfx::CICP::MC_UNSPECIFIED;
 186
 187   OwnedDav1dPicture mColorDav1d;
 188   OwnedDav1dPicture mAlphaDav1d;
 189   UniquePtr<OwnedAOMImage> mColorAOM;
 190   UniquePtr<OwnedAOMImage> mAlphaAOM;
 191
 192   // CICP values (either from the BMFF container or the AV1 sequence header) are
 193   // used to create the colorspace transform. CICP::MatrixCoefficients is only
 194   // stored for the sake of telemetry, since the relevant information for YUV ->
 195   // RGB conversion is stored in mYUVColorSpace.
 196   //
 197   // There are three potential sources of color information for an AVIF:
 198   // 1. ICC profile via a ColourInformationBox (colr) defined in [ISOBMFF]
 199   //    § 12.1.5 "Colour information" and [MIAF] § 7.3.6.4 "Colour information
 200   //    property"
 201   // 2. NCLX (AKA CICP see [ITU-T H.273]) values in the same
 202   // ColourInformationBox
 203   //    which can have an ICC profile or NCLX values, not both).
 204   // 3. NCLX values in the AV1 bitstream
 205   //
 206   // The 'colr' box is optional, but there are always CICP values in the AV1
 207   // bitstream, so it is possible to have both. Per ISOBMFF § 12.1.5.1
 208   // > If colour information is supplied in both this box, and also in the
 209   // > video bitstream, this box takes precedence, and over-rides the
 210   // > information in the bitstream.
 211   //
 212   // If present, the ICC profile takes precedence over CICP values, but only
 213   // specifies the color space, not the matrix coefficients necessary to convert
 214   // YCbCr data (as most AVIF are encoded) to RGB. The matrix coefficients are
 215   // always derived from the CICP values for matrix_coefficients (and
 216   // potentially colour_primaries, but in that case only the CICP values for
 217   // colour_primaries will be used, not anything harvested from the ICC
 218   // profile).
 219   //
 220   // If there is no ICC profile, the color space transform will be based on the
 221   // CICP values either from the 'colr' box, or if absent/unspecified, the
 222   // decoded AV1 sequence header.
 223   //
 224   // For values that are 2 (meaning unspecified) after trying both, the
 225   // fallback values are:
 226   // - CP:  1 (BT.709/sRGB)
 227   // - TC: 13 (sRGB)
 228   // - MC:  6 (BT.601)
 229   // - Range: Full
 230   //
 231   // Additional details here:
 232   // <https://github.com/AOMediaCodec/libavif/wiki/CICP#unspecified>. Note
 233   // that this contradicts the current version of [MIAF] § 7.3.6.4 which
 234   // specifies MC=1 (BT.709). This is revised in [MIAF DAMD2] and confirmed by
 235   // <https://github.com/AOMediaCodec/av1-avif/issues/77#issuecomment-676526097>
 236   //
 237   // The precedence for applying the various values and defaults in the event
 238   // no valid values are found are managed by the following functions.
 239   //
 240   // References:
 241   // [ISOBMFF]: ISO/IEC 14496-12:2020 <https://www.iso.org/standard/74428.html>
 242   // [MIAF]: ISO/IEC 23000-22:2019 <https://www.iso.org/standard/74417.html>
 243   // [MIAF DAMD2]: ISO/IEC 23000-22:2019/FDAmd 2
 244   // <https://www.iso.org/standard/81634.html>
 245   // [ITU-T H.273]: Rec. ITU-T H.273 (12/2016)
 246   //     <https://www.itu.int/rec/T-REC-H.273-201612-I/en>
 247   void SetCicpValues(
 248       const Mp4parseNclxColourInformation* aNclx,
 249       const gfx::CICP::ColourPrimaries aAv1ColourPrimaries,
 250       const gfx::CICP::TransferCharacteristics aAv1TransferCharacteristics,
 251       const gfx::CICP::MatrixCoefficients aAv1MatrixCoefficients);
 252 };
 253
 254 // An interface to do decode and get the decoded data
 255 class AVIFDecoderInterface {
 256  public:
 257   using Dav1dResult = nsAVIFDecoder::Dav1dResult;
 258   using NonAOMCodecError = nsAVIFDecoder::NonAOMCodecError;
 259   using AOMResult = nsAVIFDecoder::AOMResult;
 260   using NonDecoderResult = nsAVIFDecoder::NonDecoderResult;
 261   using DecodeResult = nsAVIFDecoder::DecodeResult;
 262
 263   virtual ~AVIFDecoderInterface() = default;
 264
 265   // Set the mDecodedData if Decode() succeeds
 266   virtual DecodeResult Decode(bool aShouldSendTelemetry,
 267                               const Mp4parseAvifInfo& aAVIFInfo,
 268                               const AVIFImage& aSamples) = 0;
 269   // Must be called only once after Decode() succeeds
 270   UniquePtr<AVIFDecodedData> GetDecodedData() {
 271     MOZ_ASSERT(mDecodedData);
 272     return std::move(mDecodedData);
 273   }
 274
 275  protected:
 276   explicit AVIFDecoderInterface() = default;
 277
 278   inline static bool IsDecodeSuccess(const DecodeResult& aResult) {
 279     return nsAVIFDecoder::IsDecodeSuccess(aResult);
 280   }
 281
 282   // The mDecodedData is valid after Decode() succeeds
 283   UniquePtr<AVIFDecodedData> mDecodedData;
 284 };
 285
 286 }  // namespace image
 287 }  // namespace mozilla
 288
 289 #endif  // mozilla_image_decoders_nsAVIFDecoder_h