dom/media/platforms/agnostic/DAV1DDecoder.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "DAV1DDecoder.h"
   8
   9 #include "gfxUtils.h"
  10 #include "ImageContainer.h"
  11 #include "mozilla/StaticPrefs_media.h"
  12 #include "mozilla/TaskQueue.h"
  13 #include "mozilla/gfx/gfxVars.h"
  14 #include "nsThreadUtils.h"
  15 #include "VideoUtils.h"
  16
  17 #undef LOG
  18 #define LOG(arg, ...)                                                  \
  19   DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
  20             ##__VA_ARGS__)
  21
  22 namespace mozilla {
  23
  24 static int GetDecodingThreadCount(uint32_t aCodedHeight) {
  25   /**
  26    * Based on the result we print out from the dav1decoder [1], the
  27    * following information shows the number of tiles for AV1 videos served on
  28    * Youtube. Each Tile can be decoded in parallel, so we would like to make
  29    * sure we at least use enough threads to match the number of tiles.
  30    *
  31    * ----------------------------
  32    * | resolution row col total |
  33    * |    480p      2  1     2  |
  34    * |    720p      2  2     4  |
  35    * |   1080p      4  2     8  |
  36    * |   1440p      4  2     8  |
  37    * |   2160p      8  4    32  |
  38    * ----------------------------
  39    *
  40    * Besides the tile thread count, the frame thread count also needs to be
  41    * considered. As we didn't find anything about what the best number is for
  42    * the count of frame thread, just simply use 2 for parallel jobs, which
  43    * is similar with Chromium's implementation. They uses 3 frame threads for
  44    * 720p+ but less tile threads, so we will still use more total threads. In
  45    * addition, their data is measured on 2019, our data should be closer to the
  46    * current real world situation.
  47    * [1]
  48    * https://searchfox.org/mozilla-central/rev/2f5ed7b7244172d46f538051250b14fb4d8f1a5f/third_party/dav1d/src/decode.c#2940
  49    */
  50   int tileThreads = 2, frameThreads = 2;
  51   if (aCodedHeight >= 2160) {
  52     tileThreads = 32;
  53   } else if (aCodedHeight >= 1080) {
  54     tileThreads = 8;
  55   } else if (aCodedHeight >= 720) {
  56     tileThreads = 4;
  57   }
  58   return tileThreads * frameThreads;
  59 }
  60
  61 DAV1DDecoder::DAV1DDecoder(const CreateDecoderParams& aParams)
  62     : mInfo(aParams.VideoConfig()),
  63       mTaskQueue(TaskQueue::Create(
  64           GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
  65           "Dav1dDecoder")),
  66       mImageContainer(aParams.mImageContainer),
  67       mImageAllocator(aParams.mKnowsCompositor) {}
  68
  69 RefPtr<MediaDataDecoder::InitPromise> DAV1DDecoder::Init() {
  70   Dav1dSettings settings;
  71   dav1d_default_settings(&settings);
  72   size_t decoder_threads = 2;
  73   if (mInfo.mDisplay.width >= 2048) {
  74     decoder_threads = 8;
  75   } else if (mInfo.mDisplay.width >= 1024) {
  76     decoder_threads = 4;
  77   }
  78   if (StaticPrefs::media_av1_new_thread_count_strategy()) {
  79     decoder_threads = GetDecodingThreadCount(mInfo.mImage.Height());
  80   }
  81   // Still need to consider the amount of physical cores in order to achieve
  82   // best performance.
  83   settings.n_threads =
  84       static_cast<int>(std::min(decoder_threads, GetNumberOfProcessors()));
  85   if (int32_t count = StaticPrefs::media_av1_force_thread_count(); count > 0) {
  86     settings.n_threads = count;
  87   }
  88
  89   int res = dav1d_open(&mContext, &settings);
  90   if (res < 0) {
  91     return DAV1DDecoder::InitPromise::CreateAndReject(
  92         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
  93                     RESULT_DETAIL("Couldn't get dAV1d decoder interface.")),
  94         __func__);
  95   }
  96   return DAV1DDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack,
  97                                                      __func__);
  98 }
  99
 100 RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Decode(
 101     MediaRawData* aSample) {
 102   return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
 103                                     &DAV1DDecoder::InvokeDecode, aSample);
 104 }
 105
 106 void ReleaseDataBuffer_s(const uint8_t* buf, void* user_data) {
 107   MOZ_ASSERT(user_data);
 108   MOZ_ASSERT(buf);
 109   DAV1DDecoder* d = static_cast<DAV1DDecoder*>(user_data);
 110   d->ReleaseDataBuffer(buf);
 111 }
 112
 113 void DAV1DDecoder::ReleaseDataBuffer(const uint8_t* buf) {
 114   // The release callback may be called on a different thread defined by the
 115   // third party dav1d execution. In that case post a task into TaskQueue to
 116   // ensure that mDecodingBuffers is only ever accessed on the TaskQueue.
 117   RefPtr<DAV1DDecoder> self = this;
 118   auto releaseBuffer = [self, buf] {
 119     MOZ_ASSERT(self->mTaskQueue->IsCurrentThreadIn());
 120     DebugOnly<bool> found = self->mDecodingBuffers.Remove(buf);
 121     MOZ_ASSERT(found);
 122   };
 123
 124   if (mTaskQueue->IsCurrentThreadIn()) {
 125     releaseBuffer();
 126   } else {
 127     nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
 128         "DAV1DDecoder::ReleaseDataBuffer", std::move(releaseBuffer)));
 129     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
 130     Unused << rv;
 131   }
 132 }
 133
 134 RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::InvokeDecode(
 135     MediaRawData* aSample) {
 136   MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 137   MOZ_ASSERT(aSample);
 138
 139   // Add the buffer to the hashtable in order to increase
 140   // the ref counter and keep it alive. When dav1d does not
 141   // need it any more will call it's release callback. Remove
 142   // the buffer, in there, to reduce the ref counter and eventually
 143   // free it. We need a hashtable and not an array because the
 144   // release callback are not coming in the same order that the
 145   // buffers have been added in the decoder (threading ordering
 146   // inside decoder)
 147   mDecodingBuffers.InsertOrUpdate(aSample->Data(), RefPtr{aSample});
 148   Dav1dData data;
 149   int res = dav1d_data_wrap(&data, aSample->Data(), aSample->Size(),
 150                             ReleaseDataBuffer_s, this);
 151   data.m.timestamp = aSample->mTimecode.ToMicroseconds();
 152   data.m.duration = aSample->mDuration.ToMicroseconds();
 153   data.m.offset = aSample->mOffset;
 154
 155   if (res < 0) {
 156     LOG("Create decoder data error.");
 157     return DecodePromise::CreateAndReject(
 158         MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
 159   }
 160   DecodedData results;
 161   do {
 162     res = dav1d_send_data(mContext, &data);
 163     if (res < 0 && res != -EAGAIN) {
 164       LOG("Decode error: %d", res);
 165       return DecodePromise::CreateAndReject(
 166           MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__), __func__);
 167     }
 168     // Alway consume the whole buffer on success.
 169     // At this point only -EAGAIN error is expected.
 170     MOZ_ASSERT((res == 0 && !data.sz) ||
 171                (res == -EAGAIN && data.sz == aSample->Size()));
 172
 173     MediaResult rs(NS_OK);
 174     res = GetPicture(results, rs);
 175     if (res < 0) {
 176       if (res == -EAGAIN) {
 177         // No frames ready to return. This is not an
 178         // error, in some circumstances, we need to
 179         // feed it with a certain amount of frames
 180         // before we get a picture.
 181         continue;
 182       }
 183       return DecodePromise::CreateAndReject(rs, __func__);
 184     }
 185   } while (data.sz > 0);
 186
 187   return DecodePromise::CreateAndResolve(std::move(results), __func__);
 188 }
 189
 190 int DAV1DDecoder::GetPicture(DecodedData& aData, MediaResult& aResult) {
 191   class Dav1dPictureWrapper {
 192    public:
 193     Dav1dPicture* operator&() { return &p; }
 194     const Dav1dPicture& operator*() const { return p; }
 195     ~Dav1dPictureWrapper() { dav1d_picture_unref(&p); }
 196
 197    private:
 198     Dav1dPicture p = Dav1dPicture();
 199   };
 200   Dav1dPictureWrapper picture;
 201
 202   int res = dav1d_get_picture(mContext, &picture);
 203   if (res < 0) {
 204     LOG("Decode error: %d", res);
 205     aResult = MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__);
 206     return res;
 207   }
 208
 209   if ((*picture).p.layout == DAV1D_PIXEL_LAYOUT_I400) {
 210     return 0;
 211   }
 212
 213 #ifdef ANDROID
 214   if (!gfxVars::UseWebRender() && (*picture).p.bpc != 8) {
 215     aResult = MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, __func__);
 216     return -1;
 217   }
 218 #endif
 219
 220   RefPtr<VideoData> v = ConstructImage(*picture);
 221   if (!v) {
 222     LOG("Image allocation error: %ux%u"
 223         " display %ux%u picture %ux%u",
 224         (*picture).p.w, (*picture).p.h, mInfo.mDisplay.width,
 225         mInfo.mDisplay.height, mInfo.mImage.width, mInfo.mImage.height);
 226     aResult = MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
 227     return -1;
 228   }
 229   aData.AppendElement(std::move(v));
 230   return 0;
 231 }
 232
 233 /* static */
 234 Maybe<gfx::YUVColorSpace> DAV1DDecoder::GetColorSpace(
 235     const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
 236   // When returning Nothing(), the caller chooses the appropriate default.
 237   if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
 238     return Nothing();
 239   }
 240
 241   return gfxUtils::CicpToColorSpace(
 242       static_cast<gfx::CICP::MatrixCoefficients>(aPicture.seq_hdr->mtrx),
 243       static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
 244 }
 245
 246 /* static */
 247 Maybe<gfx::ColorSpace2> DAV1DDecoder::GetColorPrimaries(
 248     const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
 249   // When returning Nothing(), the caller chooses the appropriate default.
 250   if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
 251     return Nothing();
 252   }
 253
 254   return gfxUtils::CicpToColorPrimaries(
 255       static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
 256 }
 257
 258 already_AddRefed<VideoData> DAV1DDecoder::ConstructImage(
 259     const Dav1dPicture& aPicture) {
 260   VideoData::YCbCrBuffer b;
 261   if (aPicture.p.bpc == 10) {
 262     b.mColorDepth = gfx::ColorDepth::COLOR_10;
 263   } else if (aPicture.p.bpc == 12) {
 264     b.mColorDepth = gfx::ColorDepth::COLOR_12;
 265   } else {
 266     b.mColorDepth = gfx::ColorDepth::COLOR_8;
 267   }
 268
 269   b.mYUVColorSpace =
 270       DAV1DDecoder::GetColorSpace(aPicture, sPDMLog)
 271           .valueOr(DefaultColorSpace({aPicture.p.w, aPicture.p.h}));
 272   b.mColorPrimaries = DAV1DDecoder::GetColorPrimaries(aPicture, sPDMLog)
 273                           .valueOr(gfx::ColorSpace2::BT709);
 274   b.mColorRange = aPicture.seq_hdr->color_range ? gfx::ColorRange::FULL
 275                                                 : gfx::ColorRange::LIMITED;
 276
 277   b.mPlanes[0].mData = static_cast<uint8_t*>(aPicture.data[0]);
 278   b.mPlanes[0].mStride = aPicture.stride[0];
 279   b.mPlanes[0].mHeight = aPicture.p.h;
 280   b.mPlanes[0].mWidth = aPicture.p.w;
 281   b.mPlanes[0].mSkip = 0;
 282
 283   b.mPlanes[1].mData = static_cast<uint8_t*>(aPicture.data[1]);
 284   b.mPlanes[1].mStride = aPicture.stride[1];
 285   b.mPlanes[1].mSkip = 0;
 286
 287   b.mPlanes[2].mData = static_cast<uint8_t*>(aPicture.data[2]);
 288   b.mPlanes[2].mStride = aPicture.stride[1];
 289   b.mPlanes[2].mSkip = 0;
 290
 291   // https://code.videolan.org/videolan/dav1d/blob/master/tools/output/yuv.c#L67
 292   const int ss_ver = aPicture.p.layout == DAV1D_PIXEL_LAYOUT_I420;
 293   const int ss_hor = aPicture.p.layout != DAV1D_PIXEL_LAYOUT_I444;
 294
 295   b.mPlanes[1].mHeight = (aPicture.p.h + ss_ver) >> ss_ver;
 296   b.mPlanes[1].mWidth = (aPicture.p.w + ss_hor) >> ss_hor;
 297
 298   b.mPlanes[2].mHeight = (aPicture.p.h + ss_ver) >> ss_ver;
 299   b.mPlanes[2].mWidth = (aPicture.p.w + ss_hor) >> ss_hor;
 300
 301   if (ss_ver) {
 302     b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
 303   } else if (ss_hor) {
 304     b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH;
 305   }
 306
 307   // Timestamp, duration and offset used here are wrong.
 308   // We need to take those values from the decoder. Latest
 309   // dav1d version allows for that.
 310   media::TimeUnit timecode =
 311       media::TimeUnit::FromMicroseconds(aPicture.m.timestamp);
 312   media::TimeUnit duration =
 313       media::TimeUnit::FromMicroseconds(aPicture.m.duration);
 314   int64_t offset = aPicture.m.offset;
 315   bool keyframe = aPicture.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY;
 316
 317   return VideoData::CreateAndCopyData(
 318       mInfo, mImageContainer, offset, timecode, duration, b, keyframe, timecode,
 319       mInfo.ScaledImageRect(aPicture.p.w, aPicture.p.h), mImageAllocator);
 320 }
 321
 322 RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Drain() {
 323   RefPtr<DAV1DDecoder> self = this;
 324   return InvokeAsync(mTaskQueue, __func__, [self, this] {
 325     int res = 0;
 326     DecodedData results;
 327     do {
 328       MediaResult rs(NS_OK);
 329       res = GetPicture(results, rs);
 330       if (res < 0 && res != -EAGAIN) {
 331         return DecodePromise::CreateAndReject(rs, __func__);
 332       }
 333     } while (res != -EAGAIN);
 334     return DecodePromise::CreateAndResolve(std::move(results), __func__);
 335   });
 336 }
 337
 338 RefPtr<MediaDataDecoder::FlushPromise> DAV1DDecoder::Flush() {
 339   RefPtr<DAV1DDecoder> self = this;
 340   return InvokeAsync(mTaskQueue, __func__, [self]() {
 341     dav1d_flush(self->mContext);
 342     return FlushPromise::CreateAndResolve(true, __func__);
 343   });
 344 }
 345
 346 RefPtr<ShutdownPromise> DAV1DDecoder::Shutdown() {
 347   RefPtr<DAV1DDecoder> self = this;
 348   return InvokeAsync(mTaskQueue, __func__, [self]() {
 349     dav1d_close(&self->mContext);
 350     return self->mTaskQueue->BeginShutdown();
 351   });
 352 }
 353
 354 }  // namespace mozilla
 355 #undef LOG