gfx/webrender_bindings/RenderAndroidSurfaceTextureHost.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "RenderAndroidSurfaceTextureHost.h"
   8
   9 #include "GLReadTexImageHelper.h"
  10 #include "mozilla/gfx/Logging.h"
  11 #include "mozilla/webrender/RenderThread.h"
  12 #include "GLContext.h"
  13 #include "AndroidSurfaceTexture.h"
  14
  15 namespace mozilla {
  16 namespace wr {
  17
  18 RenderAndroidSurfaceTextureHost::RenderAndroidSurfaceTextureHost(
  19     const java::GeckoSurfaceTexture::GlobalRef& aSurfTex, gfx::IntSize aSize,
  20     gfx::SurfaceFormat aFormat, bool aContinuousUpdate,
  21     Maybe<gfx::Matrix4x4> aTransformOverride, bool aIsRemoteTexture)
  22     : mSurfTex(aSurfTex),
  23       mSize(aSize),
  24       mFormat(aFormat),
  25       mContinuousUpdate(aContinuousUpdate),
  26       mTransformOverride(aTransformOverride),
  27       mPrepareStatus(STATUS_NONE),
  28       mAttachedToGLContext(false),
  29       mIsRemoteTexture(aIsRemoteTexture) {
  30   MOZ_COUNT_CTOR_INHERITED(RenderAndroidSurfaceTextureHost, RenderTextureHost);
  31
  32   if (mSurfTex) {
  33     mSurfTex->IncrementUse();
  34   }
  35 }
  36
  37 RenderAndroidSurfaceTextureHost::~RenderAndroidSurfaceTextureHost() {
  38   MOZ_ASSERT(RenderThread::IsInRenderThread());
  39   MOZ_COUNT_DTOR_INHERITED(RenderAndroidSurfaceTextureHost, RenderTextureHost);
  40   // The SurfaceTexture gets destroyed when its use count reaches zero.
  41   if (mSurfTex) {
  42     mSurfTex->DecrementUse();
  43   }
  44 }
  45
  46 wr::WrExternalImage RenderAndroidSurfaceTextureHost::Lock(uint8_t aChannelIndex,
  47                                                           gl::GLContext* aGL) {
  48   MOZ_ASSERT(aChannelIndex == 0);
  49   MOZ_ASSERT((mPrepareStatus == STATUS_PREPARED) ||
  50              (!mSurfTex->IsSingleBuffer() &&
  51               mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) ||
  52              mIsRemoteTexture);
  53
  54   if (mIsRemoteTexture) {
  55     EnsureAttachedToGLContext();
  56   }
  57
  58   if (mGL.get() != aGL) {
  59     // This should not happen. On android, SingletonGL is used.
  60     MOZ_ASSERT_UNREACHABLE("Unexpected GL context");
  61     return InvalidToWrExternalImage();
  62   }
  63
  64   if (!mSurfTex || !mGL || !mGL->MakeCurrent()) {
  65     return InvalidToWrExternalImage();
  66   }
  67
  68   MOZ_ASSERT(mAttachedToGLContext);
  69   if (!mAttachedToGLContext) {
  70     return InvalidToWrExternalImage();
  71   }
  72
  73   UpdateTexImageIfNecessary();
  74
  75   const auto uvs = GetUvCoords(mSize);
  76   return NativeTextureToWrExternalImage(mSurfTex->GetTexName(), uvs.first.x,
  77                                         uvs.first.y, uvs.second.x,
  78                                         uvs.second.y);
  79 }
  80
  81 void RenderAndroidSurfaceTextureHost::Unlock() {}
  82
  83 bool RenderAndroidSurfaceTextureHost::EnsureAttachedToGLContext() {
  84   // During handling WebRenderError, GeckoSurfaceTexture should not be attached
  85   // to GLContext.
  86   if (RenderThread::Get()->IsHandlingWebRenderError()) {
  87     return false;
  88   }
  89
  90   if (mAttachedToGLContext) {
  91     return true;
  92   }
  93
  94   if (!mGL) {
  95     mGL = RenderThread::Get()->SingletonGL();
  96   }
  97
  98   if (!mSurfTex || !mGL || !mGL->MakeCurrent()) {
  99     return false;
 100   }
 101
 102   if (!mSurfTex->IsAttachedToGLContext((int64_t)mGL.get())) {
 103     GLuint texName;
 104     mGL->fGenTextures(1, &texName);
 105     ActivateBindAndTexParameteri(mGL, LOCAL_GL_TEXTURE0,
 106                                  LOCAL_GL_TEXTURE_EXTERNAL_OES, texName);
 107
 108     if (NS_FAILED(mSurfTex->AttachToGLContext((int64_t)mGL.get(), texName))) {
 109       MOZ_ASSERT(0);
 110       mGL->fDeleteTextures(1, &texName);
 111       return false;
 112     }
 113   }
 114
 115   mAttachedToGLContext = true;
 116   return true;
 117 }
 118
 119 void RenderAndroidSurfaceTextureHost::PrepareForUse() {
 120   // When SurfaceTexture is single buffer mode, UpdateTexImage needs to be
 121   // called only once for each publish. If UpdateTexImage is called more
 122   // than once, it causes hang on puglish side. And UpdateTexImage needs to
 123   // be called on render thread, since the SurfaceTexture is consumed on render
 124   // thread.
 125   MOZ_ASSERT(RenderThread::IsInRenderThread());
 126   MOZ_ASSERT(mPrepareStatus == STATUS_NONE);
 127
 128   if (mContinuousUpdate || !mSurfTex) {
 129     return;
 130   }
 131
 132   mPrepareStatus = STATUS_MIGHT_BE_USED_BY_WR;
 133
 134   if (mSurfTex->IsSingleBuffer()) {
 135     EnsureAttachedToGLContext();
 136     // When SurfaceTexture is single buffer mode, it is OK to call
 137     // UpdateTexImage() here.
 138     mSurfTex->UpdateTexImage();
 139     mPrepareStatus = STATUS_PREPARED;
 140   }
 141 }
 142
 143 void RenderAndroidSurfaceTextureHost::NotifyForUse() {
 144   MOZ_ASSERT(RenderThread::IsInRenderThread());
 145
 146   if (mPrepareStatus == STATUS_MIGHT_BE_USED_BY_WR) {
 147     // This happens when SurfaceTexture of video is rendered on WebRender.
 148     // There is a case that SurfaceTexture is not rendered on WebRender, instead
 149     // it is rendered to WebGL and the SurfaceTexture should not be attached to
 150     // gl context of WebRender. It is ugly. But it is same as Compositor
 151     // rendering.
 152     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 153     if (!EnsureAttachedToGLContext()) {
 154       return;
 155     }
 156     mPrepareStatus = STATUS_UPDATE_TEX_IMAGE_NEEDED;
 157   }
 158 }
 159
 160 void RenderAndroidSurfaceTextureHost::NotifyNotUsed() {
 161   MOZ_ASSERT(RenderThread::IsInRenderThread());
 162
 163   if (!mSurfTex) {
 164     MOZ_ASSERT(mPrepareStatus == STATUS_NONE);
 165     return;
 166   }
 167
 168   if (mIsRemoteTexture) {
 169     UpdateTexImageIfNecessary();
 170   }
 171
 172   if (mSurfTex->IsSingleBuffer()) {
 173     MOZ_ASSERT(mPrepareStatus == STATUS_PREPARED);
 174     MOZ_ASSERT(mAttachedToGLContext);
 175     // Release SurfaceTexture's buffer to client side.
 176     mGL->MakeCurrent();
 177     mSurfTex->ReleaseTexImage();
 178   } else if (mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) {
 179     MOZ_ASSERT(mAttachedToGLContext);
 180     // This could happen when video frame was skipped. UpdateTexImage() neeeds
 181     // to be called for adjusting SurfaceTexture's buffer status.
 182     mSurfTex->UpdateTexImage();
 183   }
 184
 185   mPrepareStatus = STATUS_NONE;
 186 }
 187
 188 void RenderAndroidSurfaceTextureHost::UpdateTexImageIfNecessary() {
 189   if (mIsRemoteTexture) {
 190     EnsureAttachedToGLContext();
 191     if (mPrepareStatus == STATUS_NONE) {
 192       PrepareForUse();
 193     }
 194     if (mPrepareStatus == STATUS_MIGHT_BE_USED_BY_WR) {
 195       NotifyForUse();
 196     }
 197   }
 198
 199   if (mContinuousUpdate) {
 200     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 201     mSurfTex->UpdateTexImage();
 202   } else if (mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) {
 203     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 204     // When SurfaceTexture is not single buffer mode, call UpdateTexImage() once
 205     // just before rendering. During playing video, one SurfaceTexture is used
 206     // for all RenderAndroidSurfaceTextureHosts of video.
 207     mSurfTex->UpdateTexImage();
 208     mPrepareStatus = STATUS_PREPARED;
 209   }
 210 }
 211
 212 gfx::SurfaceFormat RenderAndroidSurfaceTextureHost::GetFormat() const {
 213   MOZ_ASSERT(mFormat == gfx::SurfaceFormat::R8G8B8A8 ||
 214              mFormat == gfx::SurfaceFormat::R8G8B8X8);
 215
 216   if (mFormat == gfx::SurfaceFormat::R8G8B8A8) {
 217     return gfx::SurfaceFormat::B8G8R8A8;
 218   }
 219
 220   if (mFormat == gfx::SurfaceFormat::R8G8B8X8) {
 221     return gfx::SurfaceFormat::B8G8R8X8;
 222   }
 223
 224   gfxCriticalNoteOnce
 225       << "Unexpected color format of RenderAndroidSurfaceTextureHost";
 226
 227   return gfx::SurfaceFormat::UNKNOWN;
 228 }
 229
 230 already_AddRefed<DataSourceSurface>
 231 RenderAndroidSurfaceTextureHost::ReadTexImage() {
 232   if (!mGL) {
 233     mGL = RenderThread::Get()->SingletonGL();
 234     if (!mGL) {
 235       return nullptr;
 236     }
 237   }
 238
 239   /* Allocate resulting image surface */
 240   int32_t stride = mSize.width * BytesPerPixel(GetFormat());
 241   RefPtr<DataSourceSurface> surf =
 242       Factory::CreateDataSourceSurfaceWithStride(mSize, GetFormat(), stride);
 243   if (!surf) {
 244     return nullptr;
 245   }
 246
 247   layers::ShaderConfigOGL config = layers::ShaderConfigFromTargetAndFormat(
 248       LOCAL_GL_TEXTURE_EXTERNAL, mFormat);
 249   int shaderConfig = config.mFeatures;
 250
 251   bool ret = mGL->ReadTexImageHelper()->ReadTexImage(
 252       surf, mSurfTex->GetTexName(), LOCAL_GL_TEXTURE_EXTERNAL, mSize,
 253       shaderConfig, /* aYInvert */ false);
 254   if (!ret) {
 255     return nullptr;
 256   }
 257
 258   return surf.forget();
 259 }
 260
 261 bool RenderAndroidSurfaceTextureHost::MapPlane(RenderCompositor* aCompositor,
 262                                                uint8_t aChannelIndex,
 263                                                PlaneInfo& aPlaneInfo) {
 264   UpdateTexImageIfNecessary();
 265
 266   RefPtr<gfx::DataSourceSurface> readback = ReadTexImage();
 267   if (!readback) {
 268     return false;
 269   }
 270
 271   DataSourceSurface::MappedSurface map;
 272   if (!readback->Map(DataSourceSurface::MapType::READ, &map)) {
 273     return false;
 274   }
 275
 276   mReadback = readback;
 277   aPlaneInfo.mSize = mSize;
 278   aPlaneInfo.mStride = map.mStride;
 279   aPlaneInfo.mData = map.mData;
 280   return true;
 281 }
 282
 283 void RenderAndroidSurfaceTextureHost::UnmapPlanes() {
 284   if (mReadback) {
 285     mReadback->Unmap();
 286     mReadback = nullptr;
 287   }
 288 }
 289
 290 std::pair<gfx::Point, gfx::Point> RenderAndroidSurfaceTextureHost::GetUvCoords(
 291     gfx::IntSize aTextureSize) const {
 292   gfx::Matrix4x4 transform;
 293
 294   // GetTransformMatrix() returns the transform set by the producer side of the
 295   // SurfaceTexture that must be applied to texture coordinates when
 296   // sampling. In some cases we may have set an override value, such as in
 297   // AndroidNativeWindowTextureData where we own the producer side, or for
 298   // MediaCodec output on devices where where we know the value is incorrect.
 299   if (mTransformOverride) {
 300     transform = *mTransformOverride;
 301   } else if (mSurfTex) {
 302     const auto& surf = java::sdk::SurfaceTexture::LocalRef(
 303         java::sdk::SurfaceTexture::Ref::From(mSurfTex));
 304     gl::AndroidSurfaceTexture::GetTransformMatrix(surf, &transform);
 305   }
 306
 307   // We expect this transform to always be rectilinear, usually just a
 308   // y-flip and sometimes an x and y scale. This allows this function
 309   // to simply transform and return 2 points here instead of 4.
 310   MOZ_ASSERT(transform.IsRectilinear(),
 311              "Unexpected non-rectilinear transform returned from "
 312              "SurfaceTexture.GetTransformMatrix()");
 313
 314   transform.PostScale(aTextureSize.width, aTextureSize.height, 0.0);
 315
 316   gfx::Point uv0 = gfx::Point(0.0, 0.0);
 317   gfx::Point uv1 = gfx::Point(1.0, 1.0);
 318   uv0 = transform.TransformPoint(uv0);
 319   uv1 = transform.TransformPoint(uv1);
 320
 321   return std::make_pair(uv0, uv1);
 322 }
 323
 324 }  // namespace wr
 325 }  // namespace mozilla