dom/media/VideoFrameContainer.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "VideoFrameContainer.h"
   8
   9 #ifdef MOZ_WIDGET_ANDROID
  10 #  include "GLImages.h"  // for SurfaceTextureImage
  11 #endif
  12 #include "MediaDecoderOwner.h"
  13 #include "mozilla/Telemetry.h"
  14 #include "mozilla/AbstractThread.h"
  15
  16 using namespace mozilla::layers;
  17
  18 namespace mozilla {
  19 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
  20
  21 namespace {
  22 template <Telemetry::HistogramID ID>
  23 class AutoTimer {
  24   // Set a threshold to reduce performance overhead
  25   // for we're measuring hot spots.
  26   static const uint32_t sThresholdMS = 1000;
  27
  28  public:
  29   ~AutoTimer() {
  30     auto end = TimeStamp::Now();
  31     auto diff = uint32_t((end - mStart).ToMilliseconds());
  32     if (diff > sThresholdMS) {
  33       Telemetry::Accumulate(ID, diff);
  34     }
  35   }
  36
  37  private:
  38   const TimeStamp mStart = TimeStamp::Now();
  39 };
  40 }  // namespace
  41
  42 VideoFrameContainer::VideoFrameContainer(
  43     MediaDecoderOwner* aOwner, already_AddRefed<ImageContainer> aContainer)
  44     : mOwner(aOwner),
  45       mImageContainer(aContainer),
  46       mMutex("nsVideoFrameContainer"),
  47       mFrameID(0),
  48       mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE),
  49       mFrameIDForPendingPrincipalHandle(0),
  50       mMainThread(aOwner->AbstractMainThread()) {
  51   NS_ASSERTION(aOwner, "aOwner must not be null");
  52   NS_ASSERTION(mImageContainer, "aContainer must not be null");
  53 }
  54
  55 VideoFrameContainer::~VideoFrameContainer() = default;
  56
  57 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle() {
  58   MutexAutoLock lock(mMutex);
  59   return GetLastPrincipalHandleLocked();
  60 }
  61
  62 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandleLocked() {
  63   return mLastPrincipalHandle;
  64 }
  65
  66 void VideoFrameContainer::UpdatePrincipalHandleForFrameID(
  67     const PrincipalHandle& aPrincipalHandle,
  68     const ImageContainer::FrameID& aFrameID) {
  69   MutexAutoLock lock(mMutex);
  70   UpdatePrincipalHandleForFrameIDLocked(aPrincipalHandle, aFrameID);
  71 }
  72
  73 void VideoFrameContainer::UpdatePrincipalHandleForFrameIDLocked(
  74     const PrincipalHandle& aPrincipalHandle,
  75     const ImageContainer::FrameID& aFrameID) {
  76   if (mPendingPrincipalHandle == aPrincipalHandle) {
  77     return;
  78   }
  79   mPendingPrincipalHandle = aPrincipalHandle;
  80   mFrameIDForPendingPrincipalHandle = aFrameID;
  81 }
  82
  83 #ifdef MOZ_WIDGET_ANDROID
  84 static void NotifySetCurrent(Image* aImage) {
  85   if (aImage == nullptr) {
  86     return;
  87   }
  88
  89   SurfaceTextureImage* image = aImage->AsSurfaceTextureImage();
  90   if (image == nullptr) {
  91     return;
  92   }
  93
  94   image->OnSetCurrent();
  95 }
  96 #endif
  97
  98 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
  99                                           Image* aImage,
 100                                           const TimeStamp& aTargetTime) {
 101 #ifdef MOZ_WIDGET_ANDROID
 102   NotifySetCurrent(aImage);
 103 #endif
 104   if (aImage) {
 105     MutexAutoLock lock(mMutex);
 106     AutoTArray<ImageContainer::NonOwningImage, 1> imageList;
 107     imageList.AppendElement(
 108         ImageContainer::NonOwningImage(aImage, aTargetTime, ++mFrameID));
 109     SetCurrentFramesLocked(aIntrinsicSize, imageList);
 110   } else {
 111     ClearCurrentFrame(aIntrinsicSize);
 112   }
 113 }
 114
 115 void VideoFrameContainer::SetCurrentFrames(
 116     const gfx::IntSize& aIntrinsicSize,
 117     const nsTArray<ImageContainer::NonOwningImage>& aImages) {
 118 #ifdef MOZ_WIDGET_ANDROID
 119   // When there are multiple frames, only the last one is effective
 120   // (see bug 1299068 comment 4). Here I just count on VideoSink and VideoOutput
 121   // to send one frame at a time and warn if not.
 122   Unused << NS_WARN_IF(aImages.Length() > 1);
 123   for (auto& image : aImages) {
 124     NotifySetCurrent(image.mImage);
 125   }
 126 #endif
 127   MutexAutoLock lock(mMutex);
 128   SetCurrentFramesLocked(aIntrinsicSize, aImages);
 129 }
 130
 131 void VideoFrameContainer::SetCurrentFramesLocked(
 132     const gfx::IntSize& aIntrinsicSize,
 133     const nsTArray<ImageContainer::NonOwningImage>& aImages) {
 134   mMutex.AssertCurrentThreadOwns();
 135
 136   if (auto size = Some(aIntrinsicSize); size != mIntrinsicSize) {
 137     mIntrinsicSize = size;
 138     mMainThread->Dispatch(NS_NewRunnableFunction(
 139         "IntrinsicSizeChanged", [this, self = RefPtr(this), size]() {
 140           mMainThreadState.mNewIntrinsicSize = size;
 141         }));
 142   }
 143
 144   gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize();
 145
 146   // When using the OMX decoder, destruction of the current image can indirectly
 147   //  block on main thread I/O. If we let this happen while holding onto
 148   //  |mImageContainer|'s lock, then when the main thread then tries to
 149   //  composite it can then block on |mImageContainer|'s lock, causing a
 150   //  deadlock. We use this hack to defer the destruction of the current image
 151   //  until it is safe.
 152   nsTArray<ImageContainer::OwningImage> oldImages;
 153   mImageContainer->GetCurrentImages(&oldImages);
 154
 155   PrincipalHandle principalHandle = PRINCIPAL_HANDLE_NONE;
 156   ImageContainer::FrameID lastFrameIDForOldPrincipalHandle =
 157       mFrameIDForPendingPrincipalHandle - 1;
 158   if (mPendingPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
 159       ((!oldImages.IsEmpty() &&
 160         oldImages.LastElement().mFrameID >= lastFrameIDForOldPrincipalHandle) ||
 161        (!aImages.IsEmpty() &&
 162         aImages[0].mFrameID > lastFrameIDForOldPrincipalHandle))) {
 163     // We are releasing the last FrameID prior to
 164     // `lastFrameIDForOldPrincipalHandle` OR there are no FrameIDs prior to
 165     // `lastFrameIDForOldPrincipalHandle` in the new set of images. This means
 166     // that the old principal handle has been flushed out and we can notify our
 167     // video element about this change.
 168     principalHandle = mPendingPrincipalHandle;
 169     mLastPrincipalHandle = mPendingPrincipalHandle;
 170     mPendingPrincipalHandle = PRINCIPAL_HANDLE_NONE;
 171     mFrameIDForPendingPrincipalHandle = 0;
 172   }
 173
 174   if (aImages.IsEmpty()) {
 175     mImageContainer->ClearAllImages();
 176   } else {
 177     mImageContainer->SetCurrentImages(aImages);
 178   }
 179   gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize();
 180   bool imageSizeChanged = (oldFrameSize != newFrameSize);
 181
 182   if (principalHandle != PRINCIPAL_HANDLE_NONE || imageSizeChanged) {
 183     RefPtr<VideoFrameContainer> self = this;
 184     mMainThread->Dispatch(NS_NewRunnableFunction(
 185         "PrincipalHandleOrImageSizeChanged",
 186         [this, self, principalHandle, imageSizeChanged]() {
 187           mMainThreadState.mImageSizeChanged = imageSizeChanged;
 188           if (mOwner && principalHandle != PRINCIPAL_HANDLE_NONE) {
 189             mOwner->PrincipalHandleChangedForVideoFrameContainer(
 190                 this, principalHandle);
 191           }
 192         }));
 193   }
 194 }
 195
 196 void VideoFrameContainer::ClearFutureFrames(TimeStamp aNow) {
 197   MutexAutoLock lock(mMutex);
 198
 199   // See comment in SetCurrentFrame for the reasoning behind
 200   // using a kungFuDeathGrip here.
 201   AutoTArray<ImageContainer::OwningImage, 10> kungFuDeathGrip;
 202   mImageContainer->GetCurrentImages(&kungFuDeathGrip);
 203
 204   if (!kungFuDeathGrip.IsEmpty()) {
 205     AutoTArray<ImageContainer::NonOwningImage, 1> currentFrame;
 206     ImageContainer::OwningImage& img = kungFuDeathGrip[0];
 207     // Find the current image in case there are several.
 208     for (const auto& image : kungFuDeathGrip) {
 209       if (image.mTimeStamp > aNow) {
 210         break;
 211       }
 212       img = image;
 213     }
 214     currentFrame.AppendElement(ImageContainer::NonOwningImage(
 215         img.mImage, img.mTimeStamp, img.mFrameID, img.mProducerID));
 216     mImageContainer->SetCurrentImages(currentFrame);
 217   }
 218 }
 219
 220 void VideoFrameContainer::ClearCachedResources() {
 221   MutexAutoLock lock(mMutex);
 222   mImageContainer->ClearCachedResources();
 223 }
 224
 225 ImageContainer* VideoFrameContainer::GetImageContainer() {
 226   // Note - you'll need the lock to manipulate this.  The pointer is not
 227   // modified from multiple threads, just the data pointed to by it.
 228   return mImageContainer;
 229 }
 230
 231 double VideoFrameContainer::GetFrameDelay() {
 232   MutexAutoLock lock(mMutex);
 233   return mImageContainer->GetPaintDelay().ToSeconds();
 234 }
 235
 236 void VideoFrameContainer::InvalidateWithFlags(uint32_t aFlags) {
 237   NS_ASSERTION(NS_IsMainThread(), "Must call on main thread");
 238
 239   if (!mOwner) {
 240     // Owner has been destroyed
 241     return;
 242   }
 243
 244   MediaDecoderOwner::ImageSizeChanged imageSizeChanged{
 245       mMainThreadState.mImageSizeChanged};
 246   mMainThreadState.mImageSizeChanged = false;
 247
 248   auto newIntrinsicSize = std::move(mMainThreadState.mNewIntrinsicSize);
 249
 250   MediaDecoderOwner::ForceInvalidate forceInvalidate{
 251       (aFlags & INVALIDATE_FORCE) != 0};
 252   mOwner->Invalidate(imageSizeChanged, newIntrinsicSize, forceInvalidate);
 253 }
 254
 255 }  // namespace mozilla
 256
 257 #undef NS_DispatchToMainThread