dom/media/VideoFrameConverter.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef VideoFrameConverter_h
   7 #define VideoFrameConverter_h
   8
   9 #include "ImageContainer.h"
  10 #include "ImageToI420.h"
  11 #include "Pacer.h"
  12 #include "PerformanceRecorder.h"
  13 #include "VideoSegment.h"
  14 #include "VideoUtils.h"
  15 #include "nsISupportsImpl.h"
  16 #include "nsThreadUtils.h"
  17 #include "jsapi/RTCStatsReport.h"
  18 #include "mozilla/TaskQueue.h"
  19 #include "mozilla/dom/ImageBitmapBinding.h"
  20 #include "mozilla/dom/ImageUtils.h"
  21 #include "api/video/video_frame.h"
  22 #include "common_video/include/video_frame_buffer_pool.h"
  23 #include "common_video/include/video_frame_buffer.h"
  24
  25 // The number of frame buffers VideoFrameConverter may create before returning
  26 // errors.
  27 // Sometimes these are released synchronously but they can be forwarded all the
  28 // way to the encoder for asynchronous encoding. With a pool size of 5,
  29 // we allow 1 buffer for the current conversion, and 4 buffers to be queued at
  30 // the encoder.
  31 #define CONVERTER_BUFFER_POOL_SIZE 5
  32
  33 namespace mozilla {
  34
  35 static mozilla::LazyLogModule gVideoFrameConverterLog("VideoFrameConverter");
  36
  37 // An async video frame format converter.
  38 //
  39 // Input is typically a MediaTrackListener driven by MediaTrackGraph.
  40 //
  41 // Output is passed through to VideoFrameConvertedEvent() whenever a frame is
  42 // converted.
  43 class VideoFrameConverter {
  44  public:
  45   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameConverter)
  46
  47  protected:
  48   explicit VideoFrameConverter(
  49       const dom::RTCStatsTimestampMaker& aTimestampMaker)
  50       : mTimestampMaker(aTimestampMaker),
  51         mTaskQueue(TaskQueue::Create(
  52             GetMediaThreadPool(MediaThreadType::WEBRTC_WORKER),
  53             "VideoFrameConverter")),
  54         mPacer(MakeAndAddRef<Pacer<FrameToProcess>>(
  55             mTaskQueue, TimeDuration::FromSeconds(1))),
  56         mBufferPool(false, CONVERTER_BUFFER_POOL_SIZE) {
  57     MOZ_COUNT_CTOR(VideoFrameConverter);
  58   }
  59
  60   void RegisterListener() {
  61     mPacingListener = mPacer->PacedItemEvent().Connect(
  62         mTaskQueue,
  63         [self = RefPtr(this)](FrameToProcess aFrame, TimeStamp aTime) {
  64           self->QueueForProcessing(std::move(aFrame.mImage), aTime,
  65                                    aFrame.mSize, aFrame.mForceBlack);
  66         });
  67   }
  68
  69  public:
  70   static already_AddRefed<VideoFrameConverter> Create(
  71       const dom::RTCStatsTimestampMaker& aTimestampMaker) {
  72     RefPtr<VideoFrameConverter> converter =
  73         new VideoFrameConverter(aTimestampMaker);
  74     converter->RegisterListener();
  75     return converter.forget();
  76   }
  77
  78   void QueueVideoChunk(const VideoChunk& aChunk, bool aForceBlack) {
  79     gfx::IntSize size = aChunk.mFrame.GetIntrinsicSize();
  80     if (size.width == 0 || size.height == 0) {
  81       return;
  82     }
  83
  84     TimeStamp t = aChunk.mTimeStamp;
  85     MOZ_ASSERT(!t.IsNull());
  86
  87     mPacer->Enqueue(
  88         FrameToProcess(aChunk.mFrame.GetImage(), t, size, aForceBlack), t);
  89   }
  90
  91   /**
  92    * An active VideoFrameConverter actively converts queued video frames.
  93    * While inactive, we keep track of the frame most recently queued for
  94    * processing, so it can be immediately sent out once activated.
  95    */
  96   void SetActive(bool aActive) {
  97     MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NS_NewRunnableFunction(
  98         __func__, [self = RefPtr<VideoFrameConverter>(this), this, aActive,
  99                    time = TimeStamp::Now()] {
 100           if (mActive == aActive) {
 101             return;
 102           }
 103           MOZ_LOG(gVideoFrameConverterLog, LogLevel::Debug,
 104                   ("VideoFrameConverter %p is now %s", this,
 105                    aActive ? "active" : "inactive"));
 106           mActive = aActive;
 107           if (aActive && mLastFrameQueuedForProcessing.Serial() != -2) {
 108             // After activating, we re-process the last image that was queued
 109             // for processing so it can be immediately sent.
 110             mLastFrameQueuedForProcessing.mTime = time;
 111
 112             MOZ_ALWAYS_SUCCEEDS(
 113                 mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>(
 114                     "VideoFrameConverter::ProcessVideoFrame", this,
 115                     &VideoFrameConverter::ProcessVideoFrame,
 116                     mLastFrameQueuedForProcessing)));
 117           }
 118         })));
 119   }
 120
 121   void SetTrackEnabled(bool aTrackEnabled) {
 122     MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NS_NewRunnableFunction(
 123         __func__, [self = RefPtr<VideoFrameConverter>(this), this,
 124                    aTrackEnabled, time = TimeStamp::Now()] {
 125           if (mTrackEnabled == aTrackEnabled) {
 126             return;
 127           }
 128           MOZ_LOG(gVideoFrameConverterLog, LogLevel::Debug,
 129                   ("VideoFrameConverter %p Track is now %s", this,
 130                    aTrackEnabled ? "enabled" : "disabled"));
 131           mTrackEnabled = aTrackEnabled;
 132           if (!aTrackEnabled) {
 133             // After disabling we immediately send a frame as black, so it can
 134             // be seen quickly, even if no frames are flowing. If no frame has
 135             // been queued for processing yet, we use the FrameToProcess default
 136             // size (640x480).
 137             mLastFrameQueuedForProcessing.mTime = time;
 138             mLastFrameQueuedForProcessing.mForceBlack = true;
 139             mLastFrameQueuedForProcessing.mImage = nullptr;
 140
 141             MOZ_ALWAYS_SUCCEEDS(
 142                 mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>(
 143                     "VideoFrameConverter::ProcessVideoFrame", this,
 144                     &VideoFrameConverter::ProcessVideoFrame,
 145                     mLastFrameQueuedForProcessing)));
 146           }
 147         })));
 148   }
 149
 150   void SetTrackingId(TrackingId aTrackingId) {
 151     MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NS_NewRunnableFunction(
 152         __func__, [self = RefPtr<VideoFrameConverter>(this), this,
 153                    id = std::move(aTrackingId)]() mutable {
 154           mTrackingId = Some(std::move(id));
 155         })));
 156   }
 157
 158   void Shutdown() {
 159     mPacer->Shutdown()->Then(mTaskQueue, __func__,
 160                              [self = RefPtr<VideoFrameConverter>(this), this] {
 161                                mPacingListener.DisconnectIfExists();
 162                                mBufferPool.Release();
 163                                mLastFrameQueuedForProcessing = FrameToProcess();
 164                                mLastFrameConverted = Nothing();
 165                              });
 166   }
 167
 168   MediaEventSourceExc<webrtc::VideoFrame>& VideoFrameConvertedEvent() {
 169     return mVideoFrameConvertedEvent;
 170   }
 171
 172  protected:
 173   struct FrameToProcess {
 174     FrameToProcess() = default;
 175
 176     FrameToProcess(RefPtr<layers::Image> aImage, TimeStamp aTime,
 177                    gfx::IntSize aSize, bool aForceBlack)
 178         : mImage(std::move(aImage)),
 179           mTime(aTime),
 180           mSize(aSize),
 181           mForceBlack(aForceBlack) {}
 182
 183     RefPtr<layers::Image> mImage;
 184     TimeStamp mTime = TimeStamp::Now();
 185     gfx::IntSize mSize = gfx::IntSize(640, 480);
 186     bool mForceBlack = false;
 187
 188     int32_t Serial() const {
 189       if (mForceBlack) {
 190         // Set the last-img check to indicate black.
 191         // -1 is not a guaranteed invalid serial. See bug 1262134.
 192         return -1;
 193       }
 194       if (!mImage) {
 195         // Set the last-img check to indicate reset.
 196         // -2 is not a guaranteed invalid serial. See bug 1262134.
 197         return -2;
 198       }
 199       return mImage->GetSerial();
 200     }
 201   };
 202
 203   struct FrameConverted {
 204     FrameConverted(webrtc::VideoFrame aFrame, int32_t aSerial)
 205         : mFrame(std::move(aFrame)), mSerial(aSerial) {}
 206
 207     webrtc::VideoFrame mFrame;
 208     int32_t mSerial;
 209   };
 210
 211   MOZ_COUNTED_DTOR_VIRTUAL(VideoFrameConverter)
 212
 213   void VideoFrameConverted(webrtc::VideoFrame aVideoFrame, int32_t aSerial) {
 214     MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 215
 216     MOZ_LOG(
 217         gVideoFrameConverterLog, LogLevel::Verbose,
 218         ("VideoFrameConverter %p: Converted a frame. Diff from last: %.3fms",
 219          this,
 220          static_cast<double>(aVideoFrame.timestamp_us() -
 221                              (mLastFrameConverted
 222                                   ? mLastFrameConverted->mFrame.timestamp_us()
 223                                   : aVideoFrame.timestamp_us())) /
 224              1000));
 225
 226     // Check that time doesn't go backwards
 227     MOZ_ASSERT_IF(mLastFrameConverted,
 228                   aVideoFrame.timestamp_us() >
 229                       mLastFrameConverted->mFrame.timestamp_us());
 230
 231     mLastFrameConverted = Some(FrameConverted(aVideoFrame, aSerial));
 232
 233     mVideoFrameConvertedEvent.Notify(std::move(aVideoFrame));
 234   }
 235
 236   void QueueForProcessing(RefPtr<layers::Image> aImage, TimeStamp aTime,
 237                           gfx::IntSize aSize, bool aForceBlack) {
 238     MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 239
 240     FrameToProcess frame{std::move(aImage), aTime, aSize,
 241                          aForceBlack || !mTrackEnabled};
 242
 243     if (frame.mTime <= mLastFrameQueuedForProcessing.mTime) {
 244       MOZ_LOG(
 245           gVideoFrameConverterLog, LogLevel::Debug,
 246           ("VideoFrameConverter %p: Dropping a frame because time did not "
 247            "progress (%.3fs)",
 248            this,
 249            (mLastFrameQueuedForProcessing.mTime - frame.mTime).ToSeconds()));
 250       return;
 251     }
 252
 253     if (frame.Serial() == mLastFrameQueuedForProcessing.Serial()) {
 254       // This is the same frame as the last one. We limit the same-frame rate to
 255       // 1 second, and rewrite the time so the frame-gap is in whole seconds.
 256       //
 257       // The pacer only starts duplicating frames every second if there is no
 258       // flow of frames into it. There are other reasons the same frame could
 259       // repeat here, and at a shorter interval than one second. For instance
 260       // after the sender is disabled (SetTrackEnabled) but there is still a
 261       // flow of frames into the pacer. All disabled frames have the same
 262       // serial.
 263       if (int32_t diffSec = static_cast<int32_t>(
 264               (frame.mTime - mLastFrameQueuedForProcessing.mTime).ToSeconds());
 265           diffSec != 0) {
 266         MOZ_LOG(
 267             gVideoFrameConverterLog, LogLevel::Verbose,
 268             ("VideoFrameConverter %p: Rewrote time interval for a duplicate "
 269              "frame from %.3fs to %.3fs",
 270              this,
 271              (frame.mTime - mLastFrameQueuedForProcessing.mTime).ToSeconds(),
 272              static_cast<float>(diffSec)));
 273         frame.mTime = mLastFrameQueuedForProcessing.mTime +
 274                       TimeDuration::FromSeconds(diffSec);
 275       } else {
 276         MOZ_LOG(
 277             gVideoFrameConverterLog, LogLevel::Verbose,
 278             ("VideoFrameConverter %p: Dropping a duplicate frame because a "
 279              "second hasn't passed (%.3fs)",
 280              this,
 281              (frame.mTime - mLastFrameQueuedForProcessing.mTime).ToSeconds()));
 282         return;
 283       }
 284     }
 285
 286     mLastFrameQueuedForProcessing = std::move(frame);
 287
 288     if (!mActive) {
 289       MOZ_LOG(
 290           gVideoFrameConverterLog, LogLevel::Debug,
 291           ("VideoFrameConverter %p: Ignoring a frame because we're inactive",
 292            this));
 293       return;
 294     }
 295
 296     MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>(
 297         "VideoFrameConverter::ProcessVideoFrame", this,
 298         &VideoFrameConverter::ProcessVideoFrame,
 299         mLastFrameQueuedForProcessing)));
 300   }
 301
 302   void ProcessVideoFrame(const FrameToProcess& aFrame) {
 303     MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
 304
 305     if (aFrame.mTime < mLastFrameQueuedForProcessing.mTime) {
 306       MOZ_LOG(
 307           gVideoFrameConverterLog, LogLevel::Debug,
 308           ("VideoFrameConverter %p: Dropping a frame that is %.3f seconds "
 309            "behind latest",
 310            this,
 311            (mLastFrameQueuedForProcessing.mTime - aFrame.mTime).ToSeconds()));
 312       return;
 313     }
 314
 315     const webrtc::Timestamp time =
 316         dom::RTCStatsTimestamp::FromMozTime(mTimestampMaker, aFrame.mTime)
 317             .ToRealtime();
 318
 319     if (mLastFrameConverted &&
 320         aFrame.Serial() == mLastFrameConverted->mSerial) {
 321       // This is the same input frame as last time. Avoid a conversion.
 322       webrtc::VideoFrame frame = mLastFrameConverted->mFrame;
 323       frame.set_timestamp_us(time.us());
 324       VideoFrameConverted(std::move(frame), mLastFrameConverted->mSerial);
 325       return;
 326     }
 327
 328     if (aFrame.mForceBlack) {
 329       // Send a black image.
 330       rtc::scoped_refptr<webrtc::I420Buffer> buffer =
 331           mBufferPool.CreateI420Buffer(aFrame.mSize.width, aFrame.mSize.height);
 332       if (!buffer) {
 333         MOZ_DIAGNOSTIC_ASSERT(false,
 334                               "Buffers not leaving scope except for "
 335                               "reconfig, should never leak");
 336         MOZ_LOG(gVideoFrameConverterLog, LogLevel::Warning,
 337                 ("VideoFrameConverter %p: Creating a buffer for a black video "
 338                  "frame failed",
 339                  this));
 340         return;
 341       }
 342
 343       MOZ_LOG(gVideoFrameConverterLog, LogLevel::Verbose,
 344               ("VideoFrameConverter %p: Sending a black video frame", this));
 345       webrtc::I420Buffer::SetBlack(buffer.get());
 346
 347       VideoFrameConverted(webrtc::VideoFrame::Builder()
 348                               .set_video_frame_buffer(buffer)
 349                               .set_timestamp_us(time.us())
 350                               .build(),
 351                           aFrame.Serial());
 352       return;
 353     }
 354
 355     if (!aFrame.mImage) {
 356       // Don't send anything for null images.
 357       return;
 358     }
 359
 360     MOZ_ASSERT(aFrame.mImage->GetSize() == aFrame.mSize);
 361
 362     RefPtr<layers::PlanarYCbCrImage> image =
 363         aFrame.mImage->AsPlanarYCbCrImage();
 364     if (image) {
 365       dom::ImageUtils utils(image);
 366       Maybe<dom::ImageBitmapFormat> format = utils.GetFormat();
 367       if (format.isSome() &&
 368           format.value() == dom::ImageBitmapFormat::YUV420P &&
 369           image->GetData()) {
 370         const layers::PlanarYCbCrData* data = image->GetData();
 371         rtc::scoped_refptr<webrtc::I420BufferInterface> video_frame_buffer =
 372             webrtc::WrapI420Buffer(
 373                 aFrame.mImage->GetSize().width, aFrame.mImage->GetSize().height,
 374                 data->mYChannel, data->mYStride, data->mCbChannel,
 375                 data->mCbCrStride, data->mCrChannel, data->mCbCrStride,
 376                 [image] { /* keep reference alive*/ });
 377
 378         MOZ_LOG(gVideoFrameConverterLog, LogLevel::Verbose,
 379                 ("VideoFrameConverter %p: Sending an I420 video frame", this));
 380         VideoFrameConverted(webrtc::VideoFrame::Builder()
 381                                 .set_video_frame_buffer(video_frame_buffer)
 382                                 .set_timestamp_us(time.us())
 383                                 .build(),
 384                             aFrame.Serial());
 385         return;
 386       }
 387     }
 388
 389     rtc::scoped_refptr<webrtc::I420Buffer> buffer =
 390         mBufferPool.CreateI420Buffer(aFrame.mSize.width, aFrame.mSize.height);
 391     if (!buffer) {
 392 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
 393       ++mFramesDropped;
 394 #endif
 395       MOZ_DIAGNOSTIC_ASSERT(mFramesDropped <= 100, "Buffers must be leaking");
 396       MOZ_LOG(gVideoFrameConverterLog, LogLevel::Warning,
 397               ("VideoFrameConverter %p: Creating a buffer failed", this));
 398       return;
 399     }
 400
 401 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
 402     mFramesDropped = 0;
 403 #endif
 404     PerformanceRecorder<CopyVideoStage> rec(
 405         "VideoFrameConverter::ConvertToI420"_ns, *mTrackingId, buffer->width(),
 406         buffer->height());
 407     nsresult rv =
 408         ConvertToI420(aFrame.mImage, buffer->MutableDataY(), buffer->StrideY(),
 409                       buffer->MutableDataU(), buffer->StrideU(),
 410                       buffer->MutableDataV(), buffer->StrideV());
 411
 412     if (NS_FAILED(rv)) {
 413       MOZ_LOG(gVideoFrameConverterLog, LogLevel::Warning,
 414               ("VideoFrameConverter %p: Image conversion failed", this));
 415       return;
 416     }
 417     rec.Record();
 418
 419     VideoFrameConverted(webrtc::VideoFrame::Builder()
 420                             .set_video_frame_buffer(buffer)
 421                             .set_timestamp_us(time.us())
 422                             .build(),
 423                         aFrame.Serial());
 424   }
 425
 426  public:
 427   const dom::RTCStatsTimestampMaker mTimestampMaker;
 428
 429   const RefPtr<TaskQueue> mTaskQueue;
 430
 431  protected:
 432   // Used to pace future frames close to their rendering-time. Thread-safe.
 433   const RefPtr<Pacer<FrameToProcess>> mPacer;
 434
 435   MediaEventProducerExc<webrtc::VideoFrame> mVideoFrameConvertedEvent;
 436
 437   // Accessed only from mTaskQueue.
 438   MediaEventListener mPacingListener;
 439   webrtc::VideoFrameBufferPool mBufferPool;
 440   FrameToProcess mLastFrameQueuedForProcessing;
 441   Maybe<FrameConverted> mLastFrameConverted;
 442   bool mActive = false;
 443   bool mTrackEnabled = true;
 444   Maybe<TrackingId> mTrackingId;
 445 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
 446   size_t mFramesDropped = 0;
 447 #endif
 448 };
 449
 450 }  // namespace mozilla
 451
 452 #endif  // VideoFrameConverter_h