dom/media/webrtc/MediaEngineRemoteVideoSource.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "MediaEngineRemoteVideoSource.h"
   7
   8 #include "CamerasChild.h"
   9 #include "Layers.h"
  10 #include "MediaManager.h"
  11 #include "MediaTrackConstraints.h"
  12 #include "mozilla/dom/MediaTrackSettingsBinding.h"
  13 #include "mozilla/ErrorNames.h"
  14 #include "mozilla/gfx/Point.h"
  15 #include "mozilla/RefPtr.h"
  16 #include "Tracing.h"
  17 #include "VideoFrameUtils.h"
  18 #include "VideoUtils.h"
  19 #include "ImageContainer.h"
  20 #include "common_video/include/video_frame_buffer.h"
  21 #include "common_video/libyuv/include/webrtc_libyuv.h"
  22
  23 namespace mozilla {
  24
  25 extern LazyLogModule gMediaManagerLog;
  26 #define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
  27 #define LOG_FRAME(...) \
  28   MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
  29
  30 using dom::ConstrainLongRange;
  31 using dom::MediaSourceEnum;
  32 using dom::MediaTrackConstraints;
  33 using dom::MediaTrackConstraintSet;
  34 using dom::MediaTrackSettings;
  35 using dom::VideoFacingModeEnum;
  36
  37 static Maybe<VideoFacingModeEnum> GetFacingMode(const nsString& aDeviceName) {
  38   // Set facing mode based on device name.
  39 #if defined(ANDROID)
  40   // Names are generated. Example: "Camera 0, Facing back, Orientation 90"
  41   //
  42   // See media/webrtc/trunk/webrtc/modules/video_capture/android/java/src/org/
  43   // webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java
  44
  45   if (aDeviceName.Find(u"Facing back"_ns) != kNotFound) {
  46     return Some(VideoFacingModeEnum::Environment);
  47   }
  48   if (aDeviceName.Find(u"Facing front"_ns) != kNotFound) {
  49     return Some(VideoFacingModeEnum::User);
  50   }
  51 #endif  // ANDROID
  52 #ifdef XP_MACOSX
  53   // Kludge to test user-facing cameras on OSX.
  54   if (aDeviceName.Find(u"Face"_ns) != -1) {
  55     return Some(VideoFacingModeEnum::User);
  56   }
  57 #endif
  58 #ifdef XP_WIN
  59   // The cameras' name of Surface book are "Microsoft Camera Front" and
  60   // "Microsoft Camera Rear" respectively.
  61
  62   if (aDeviceName.Find(u"Front"_ns) != kNotFound) {
  63     return Some(VideoFacingModeEnum::User);
  64   }
  65   if (aDeviceName.Find(u"Rear"_ns) != kNotFound) {
  66     return Some(VideoFacingModeEnum::Environment);
  67   }
  68 #endif  // WINDOWS
  69
  70   return Nothing();
  71 }
  72
  73 MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
  74     const nsAString& aDeviceName, const nsACString& aDeviceUUID,
  75     camera::CaptureEngine aCapEngine, bool aScary)
  76     : mCapEngine(aCapEngine),
  77       mScary(aScary),
  78       mMutex("MediaEngineRemoteVideoSource::mMutex"),
  79       mRescalingBufferPool(/* zero_initialize */ false,
  80                            /* max_number_of_buffers */ 1),
  81       mSettingsUpdatedByFrame(MakeAndAddRef<media::Refcountable<AtomicBool>>()),
  82       mSettings(MakeAndAddRef<media::Refcountable<MediaTrackSettings>>()),
  83       mFirstFramePromise(mFirstFramePromiseHolder.Ensure(__func__)),
  84       mDeviceName(aDeviceName),
  85       mDeviceUUID(aDeviceUUID) {
  86   LOG("%s", __PRETTY_FUNCTION__);
  87   mSettings->mWidth.Construct(0);
  88   mSettings->mHeight.Construct(0);
  89   mSettings->mFrameRate.Construct(0);
  90   if (mCapEngine == camera::CameraEngine) {
  91     // Only cameras can have a facing mode.
  92     Maybe<VideoFacingModeEnum> facingMode = GetFacingMode(mDeviceName);
  93     if (facingMode.isSome()) {
  94       NS_ConvertASCIItoUTF16 facingString(
  95           dom::VideoFacingModeEnumValues::GetString(*facingMode));
  96       mSettings->mFacingMode.Construct(facingString);
  97       mFacingMode.emplace(facingString);
  98     }
  99   }
 100 }
 101
 102 MediaEngineRemoteVideoSource::~MediaEngineRemoteVideoSource() {
 103   mFirstFramePromiseHolder.RejectIfExists(NS_ERROR_ABORT, __func__);
 104 }
 105
 106 dom::MediaSourceEnum MediaEngineRemoteVideoSource::GetMediaSource() const {
 107   switch (mCapEngine) {
 108     case camera::BrowserEngine:
 109       return MediaSourceEnum::Browser;
 110     case camera::CameraEngine:
 111       return MediaSourceEnum::Camera;
 112     case camera::ScreenEngine:
 113       return MediaSourceEnum::Screen;
 114     case camera::WinEngine:
 115       return MediaSourceEnum::Window;
 116     default:
 117       MOZ_CRASH();
 118   }
 119 }
 120
 121 nsString MediaEngineRemoteVideoSource::GetName() const {
 122   AssertIsOnOwningThread();
 123
 124   return mDeviceName;
 125 }
 126
 127 nsCString MediaEngineRemoteVideoSource::GetUUID() const {
 128   AssertIsOnOwningThread();
 129
 130   return mDeviceUUID;
 131 }
 132
 133 nsString MediaEngineRemoteVideoSource::GetGroupId() const {
 134   AssertIsOnOwningThread();
 135
 136   // The remote video backend doesn't implement group id. We return the device
 137   // name and higher layers will correlate this with the name of audio devices.
 138   return mDeviceName;
 139 }
 140
 141 nsresult MediaEngineRemoteVideoSource::Allocate(
 142     const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
 143     uint64_t aWindowID, const char** aOutBadConstraint) {
 144   LOG("%s", __PRETTY_FUNCTION__);
 145   AssertIsOnOwningThread();
 146
 147   MOZ_ASSERT(mState == kReleased);
 148
 149   NormalizedConstraints constraints(aConstraints);
 150   webrtc::CaptureCapability newCapability;
 151   LOG("ChooseCapability(kFitness) for mCapability (Allocate) ++");
 152   if (!ChooseCapability(constraints, aPrefs, newCapability, kFitness)) {
 153     *aOutBadConstraint =
 154         MediaConstraintsHelper::FindBadConstraint(constraints, this);
 155     return NS_ERROR_FAILURE;
 156   }
 157   LOG("ChooseCapability(kFitness) for mCapability (Allocate) --");
 158
 159   mCaptureId =
 160       camera::GetChildAndCall(&camera::CamerasChild::AllocateCapture,
 161                               mCapEngine, mDeviceUUID.get(), aWindowID);
 162   if (mCaptureId < 0) {
 163     return NS_ERROR_FAILURE;
 164   }
 165
 166   {
 167     MutexAutoLock lock(mMutex);
 168     mState = kAllocated;
 169     mCapability = newCapability;
 170   }
 171
 172   LOG("Video device %d allocated", mCaptureId);
 173   return NS_OK;
 174 }
 175
 176 nsresult MediaEngineRemoteVideoSource::Deallocate() {
 177   LOG("%s", __PRETTY_FUNCTION__);
 178   AssertIsOnOwningThread();
 179
 180   MOZ_ASSERT(mState == kStopped || mState == kAllocated);
 181
 182   if (mTrack) {
 183     mTrack->End();
 184   }
 185
 186   {
 187     MutexAutoLock lock(mMutex);
 188
 189     mTrack = nullptr;
 190     mPrincipal = PRINCIPAL_HANDLE_NONE;
 191     mState = kReleased;
 192   }
 193
 194   // Stop() has stopped capture synchronously on the media thread before we get
 195   // here, so there are no longer any callbacks on an IPC thread accessing
 196   // mImageContainer or mRescalingBufferPool.
 197   mImageContainer = nullptr;
 198   mRescalingBufferPool.Release();
 199
 200   LOG("Video device %d deallocated", mCaptureId);
 201
 202   if (camera::GetChildAndCall(&camera::CamerasChild::ReleaseCapture, mCapEngine,
 203                               mCaptureId)) {
 204     MOZ_ASSERT_UNREACHABLE("Couldn't release allocated device");
 205   }
 206   return NS_OK;
 207 }
 208
 209 void MediaEngineRemoteVideoSource::SetTrack(const RefPtr<MediaTrack>& aTrack,
 210                                             const PrincipalHandle& aPrincipal) {
 211   LOG("%s", __PRETTY_FUNCTION__);
 212   AssertIsOnOwningThread();
 213
 214   MOZ_ASSERT(mState == kAllocated);
 215   MOZ_ASSERT(!mTrack);
 216   MOZ_ASSERT(aTrack);
 217   MOZ_ASSERT(aTrack->AsSourceTrack());
 218
 219   if (!mImageContainer) {
 220     mImageContainer = MakeAndAddRef<layers::ImageContainer>(
 221         layers::ImageContainer::ASYNCHRONOUS);
 222   }
 223
 224   {
 225     MutexAutoLock lock(mMutex);
 226     mTrack = aTrack->AsSourceTrack();
 227     mPrincipal = aPrincipal;
 228   }
 229 }
 230
 231 nsresult MediaEngineRemoteVideoSource::Start() {
 232   LOG("%s", __PRETTY_FUNCTION__);
 233   AssertIsOnOwningThread();
 234
 235   MOZ_ASSERT(mState == kAllocated || mState == kStopped);
 236   MOZ_ASSERT(mTrack);
 237
 238   {
 239     MutexAutoLock lock(mMutex);
 240     mState = kStarted;
 241   }
 242
 243   mSettingsUpdatedByFrame->mValue = false;
 244
 245   if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture, mCapEngine,
 246                               mCaptureId, mCapability, this)) {
 247     LOG("StartCapture failed");
 248     MutexAutoLock lock(mMutex);
 249     mState = kStopped;
 250     return NS_ERROR_FAILURE;
 251   }
 252
 253   NS_DispatchToMainThread(NS_NewRunnableFunction(
 254       "MediaEngineRemoteVideoSource::SetLastCapability",
 255       [settings = mSettings, updated = mSettingsUpdatedByFrame,
 256        capEngine = mCapEngine, cap = mCapability]() mutable {
 257         switch (capEngine) {
 258           case camera::ScreenEngine:
 259           case camera::WinEngine:
 260             // Undo the hack where ideal and max constraints are crammed
 261             // together in mCapability for consumption by low-level code. We
 262             // don't actually know the real resolution yet, so report min(ideal,
 263             // max) for now.
 264             // TODO: This can be removed in bug 1453269.
 265             cap.width = std::min(cap.width >> 16, cap.width & 0xffff);
 266             cap.height = std::min(cap.height >> 16, cap.height & 0xffff);
 267             break;
 268           default:
 269             break;
 270         }
 271
 272         if (!updated->mValue) {
 273           settings->mWidth.Value() = cap.width;
 274           settings->mHeight.Value() = cap.height;
 275         }
 276         settings->mFrameRate.Value() = cap.maxFPS;
 277       }));
 278
 279   return NS_OK;
 280 }
 281
 282 nsresult MediaEngineRemoteVideoSource::FocusOnSelectedSource() {
 283   LOG("%s", __PRETTY_FUNCTION__);
 284   AssertIsOnOwningThread();
 285
 286   int result;
 287   result = camera::GetChildAndCall(&camera::CamerasChild::FocusOnSelectedSource,
 288                                    mCapEngine, mCaptureId);
 289   return result == 0 ? NS_OK : NS_ERROR_FAILURE;
 290 }
 291
 292 nsresult MediaEngineRemoteVideoSource::Stop() {
 293   LOG("%s", __PRETTY_FUNCTION__);
 294   AssertIsOnOwningThread();
 295
 296   if (mState == kStopped || mState == kAllocated) {
 297     return NS_OK;
 298   }
 299
 300   MOZ_ASSERT(mState == kStarted);
 301
 302   if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, mCapEngine,
 303                               mCaptureId)) {
 304     MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed");
 305     return NS_ERROR_FAILURE;
 306   }
 307
 308   {
 309     MutexAutoLock lock(mMutex);
 310     mState = kStopped;
 311   }
 312
 313   return NS_OK;
 314 }
 315
 316 nsresult MediaEngineRemoteVideoSource::Reconfigure(
 317     const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
 318     const char** aOutBadConstraint) {
 319   LOG("%s", __PRETTY_FUNCTION__);
 320   AssertIsOnOwningThread();
 321
 322   NormalizedConstraints constraints(aConstraints);
 323   webrtc::CaptureCapability newCapability;
 324   LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) ++");
 325   if (!ChooseCapability(constraints, aPrefs, newCapability, kFitness)) {
 326     *aOutBadConstraint =
 327         MediaConstraintsHelper::FindBadConstraint(constraints, this);
 328     return NS_ERROR_INVALID_ARG;
 329   }
 330   LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) --");
 331
 332   if (mCapability == newCapability) {
 333     return NS_OK;
 334   }
 335
 336   bool started = mState == kStarted;
 337   if (started) {
 338     nsresult rv = Stop();
 339     if (NS_WARN_IF(NS_FAILED(rv))) {
 340       nsAutoCString name;
 341       GetErrorName(rv, name);
 342       LOG("Video source %p for video device %d Reconfigure() failed "
 343           "unexpectedly in Stop(). rv=%s",
 344           this, mCaptureId, name.Data());
 345       return NS_ERROR_UNEXPECTED;
 346     }
 347   }
 348
 349   {
 350     MutexAutoLock lock(mMutex);
 351     // Start() applies mCapability on the device.
 352     mCapability = newCapability;
 353   }
 354
 355   if (started) {
 356     nsresult rv = Start();
 357     if (NS_WARN_IF(NS_FAILED(rv))) {
 358       nsAutoCString name;
 359       GetErrorName(rv, name);
 360       LOG("Video source %p for video device %d Reconfigure() failed "
 361           "unexpectedly in Start(). rv=%s",
 362           this, mCaptureId, name.Data());
 363       return NS_ERROR_UNEXPECTED;
 364     }
 365   }
 366
 367   return NS_OK;
 368 }
 369
 370 size_t MediaEngineRemoteVideoSource::NumCapabilities() const {
 371   AssertIsOnOwningThread();
 372
 373   if (!mCapabilities.IsEmpty()) {
 374     return mCapabilities.Length();
 375   }
 376
 377   int num = camera::GetChildAndCall(&camera::CamerasChild::NumberOfCapabilities,
 378                                     mCapEngine, mDeviceUUID.get());
 379   if (num > 0) {
 380     mCapabilities.SetLength(num);
 381   } else {
 382     // The default for devices that don't return discrete capabilities: treat
 383     // them as supporting all capabilities orthogonally. E.g. screensharing.
 384     // CaptureCapability defaults key values to 0, which means accept any value.
 385     mCapabilities.AppendElement(MakeUnique<webrtc::CaptureCapability>());
 386     mCapabilitiesAreHardcoded = true;
 387   }
 388
 389   return mCapabilities.Length();
 390 }
 391
 392 webrtc::CaptureCapability& MediaEngineRemoteVideoSource::GetCapability(
 393     size_t aIndex) const {
 394   AssertIsOnOwningThread();
 395   MOZ_RELEASE_ASSERT(aIndex < mCapabilities.Length());
 396   if (!mCapabilities[aIndex]) {
 397     mCapabilities[aIndex] = MakeUnique<webrtc::CaptureCapability>();
 398     camera::GetChildAndCall(&camera::CamerasChild::GetCaptureCapability,
 399                             mCapEngine, mDeviceUUID.get(), aIndex,
 400                             mCapabilities[aIndex].get());
 401   }
 402   return *mCapabilities[aIndex];
 403 }
 404
 405 int MediaEngineRemoteVideoSource::DeliverFrame(
 406     uint8_t* aBuffer, const camera::VideoFrameProperties& aProps) {
 407   // Cameras IPC thread - take great care with accessing members!
 408
 409   Maybe<int32_t> req_max_width;
 410   Maybe<int32_t> req_max_height;
 411   Maybe<int32_t> req_ideal_width;
 412   Maybe<int32_t> req_ideal_height;
 413   {
 414     MutexAutoLock lock(mMutex);
 415     MOZ_ASSERT(mState == kStarted);
 416     // TODO: These can be removed in bug 1453269.
 417     const int32_t max_width = mCapability.width & 0xffff;
 418     const int32_t max_height = mCapability.height & 0xffff;
 419     const int32_t ideal_width = (mCapability.width >> 16) & 0xffff;
 420     const int32_t ideal_height = (mCapability.height >> 16) & 0xffff;
 421
 422     req_max_width = max_width ? Some(max_width) : Nothing();
 423     req_max_height = max_height ? Some(max_height) : Nothing();
 424     req_ideal_width = ideal_width ? Some(ideal_width) : Nothing();
 425     req_ideal_height = ideal_height ? Some(ideal_height) : Nothing();
 426   }
 427
 428   // This is only used in the case of screen sharing, see bug 1453269.
 429
 430   if (aProps.rotation() == 90 || aProps.rotation() == 270) {
 431     // This frame is rotated, so what was negotiated as width is now height,
 432     // and vice versa.
 433     std::swap(req_max_width, req_max_height);
 434     std::swap(req_ideal_width, req_ideal_height);
 435   }
 436
 437   int32_t dst_max_width =
 438       std::min(aProps.width(), req_max_width.valueOr(aProps.width()));
 439   int32_t dst_max_height =
 440       std::min(aProps.height(), req_max_height.valueOr(aProps.height()));
 441   // This logic works for both camera and screen sharing case.
 442   // for camera case, req_ideal_width and req_ideal_height are absent.
 443   int32_t dst_width = req_ideal_width.valueOr(aProps.width());
 444   int32_t dst_height = req_ideal_height.valueOr(aProps.height());
 445
 446   if (!req_ideal_width && req_ideal_height) {
 447     dst_width = *req_ideal_height * aProps.width() / aProps.height();
 448   } else if (!req_ideal_height && req_ideal_width) {
 449     dst_height = *req_ideal_width * aProps.height() / aProps.width();
 450   }
 451   dst_width = std::min(dst_width, dst_max_width);
 452   dst_height = std::min(dst_height, dst_max_height);
 453
 454   // Apply scaling for screen sharing, see bug 1453269.
 455   switch (mCapEngine) {
 456     case camera::ScreenEngine:
 457     case camera::WinEngine: {
 458       // scale to average of portrait and landscape
 459       float scale_width = (float)dst_width / (float)aProps.width();
 460       float scale_height = (float)dst_height / (float)aProps.height();
 461       float scale = (scale_width + scale_height) / 2;
 462       // If both req_ideal_width & req_ideal_height are absent, scale is 1, but
 463       // if one is present and the other not, scale precisely to the one present
 464       if (!req_ideal_width) {
 465         scale = scale_height;
 466       } else if (!req_ideal_height) {
 467         scale = scale_width;
 468       }
 469       dst_width = int32_t(scale * (float)aProps.width());
 470       dst_height = int32_t(scale * (float)aProps.height());
 471
 472       // if scaled rectangle exceeds max rectangle, scale to minimum of portrait
 473       // and landscape
 474       if (dst_width > dst_max_width || dst_height > dst_max_height) {
 475         scale_width = (float)dst_max_width / (float)dst_width;
 476         scale_height = (float)dst_max_height / (float)dst_height;
 477         scale = std::min(scale_width, scale_height);
 478         dst_width = int32_t(scale * dst_width);
 479         dst_height = int32_t(scale * dst_height);
 480       }
 481       break;
 482     }
 483     default: {
 484       break;
 485     }
 486   }
 487
 488   // Ensure width and height are at least two. Smaller frames can lead to
 489   // problems with scaling and video encoding.
 490   dst_width = std::max(2, dst_width);
 491   dst_height = std::max(2, dst_height);
 492
 493   rtc::Callback0<void> callback_unused;
 494   rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
 495       webrtc::WrapI420Buffer(
 496           aProps.width(), aProps.height(), aBuffer, aProps.yStride(),
 497           aBuffer + aProps.yAllocatedSize(), aProps.uStride(),
 498           aBuffer + aProps.yAllocatedSize() + aProps.uAllocatedSize(),
 499           aProps.vStride(), callback_unused);
 500
 501   if ((dst_width != aProps.width() || dst_height != aProps.height()) &&
 502       dst_width <= aProps.width() && dst_height <= aProps.height()) {
 503     // Destination resolution is smaller than source buffer. We'll rescale.
 504     rtc::scoped_refptr<webrtc::I420Buffer> scaledBuffer =
 505         mRescalingBufferPool.CreateBuffer(dst_width, dst_height);
 506     if (!scaledBuffer) {
 507       MOZ_ASSERT_UNREACHABLE(
 508           "We might fail to allocate a buffer, but with this "
 509           "being a recycling pool that shouldn't happen");
 510       return 0;
 511     }
 512     scaledBuffer->CropAndScaleFrom(*buffer);
 513     buffer = scaledBuffer;
 514   }
 515
 516   layers::PlanarYCbCrData data;
 517   data.mYChannel = const_cast<uint8_t*>(buffer->DataY());
 518   data.mYSize = gfx::IntSize(buffer->width(), buffer->height());
 519   data.mYStride = buffer->StrideY();
 520   MOZ_ASSERT(buffer->StrideU() == buffer->StrideV());
 521   data.mCbCrStride = buffer->StrideU();
 522   data.mCbChannel = const_cast<uint8_t*>(buffer->DataU());
 523   data.mCrChannel = const_cast<uint8_t*>(buffer->DataV());
 524   data.mCbCrSize =
 525       gfx::IntSize((buffer->width() + 1) / 2, (buffer->height() + 1) / 2);
 526   data.mPicX = 0;
 527   data.mPicY = 0;
 528   data.mPicSize = gfx::IntSize(buffer->width(), buffer->height());
 529   data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
 530
 531   RefPtr<layers::PlanarYCbCrImage> image =
 532       mImageContainer->CreatePlanarYCbCrImage();
 533   if (!image->CopyData(data)) {
 534     MOZ_ASSERT_UNREACHABLE(
 535         "We might fail to allocate a buffer, but with this "
 536         "being a recycling container that shouldn't happen");
 537     return 0;
 538   }
 539
 540 #ifdef DEBUG
 541   static uint32_t frame_num = 0;
 542   LOG_FRAME(
 543       "frame %d (%dx%d)->(%dx%d); rotation %d, timeStamp %u, ntpTimeMs %" PRIu64
 544       ", renderTimeMs %" PRIu64,
 545       frame_num++, aProps.width(), aProps.height(), dst_width, dst_height,
 546       aProps.rotation(), aProps.timeStamp(), aProps.ntpTimeMs(),
 547       aProps.renderTimeMs());
 548 #endif
 549
 550   if (mImageSize.width != dst_width || mImageSize.height != dst_height) {
 551     NS_DispatchToMainThread(NS_NewRunnableFunction(
 552         "MediaEngineRemoteVideoSource::FrameSizeChange",
 553         [settings = mSettings, updated = mSettingsUpdatedByFrame,
 554          holder = std::move(mFirstFramePromiseHolder), dst_width,
 555          dst_height]() mutable {
 556           settings->mWidth.Value() = dst_width;
 557           settings->mHeight.Value() = dst_height;
 558           updated->mValue = true;
 559           // Since mImageSize was initialized to (0,0), we end up here on the
 560           // arrival of the first frame. We resolve the promise representing
 561           // arrival of first frame, after correct settings values have been
 562           // made available (Resolve() is idempotent if already resolved).
 563           holder.ResolveIfExists(true, __func__);
 564         }));
 565   }
 566
 567   {
 568     MutexAutoLock lock(mMutex);
 569     MOZ_ASSERT(mState == kStarted);
 570     VideoSegment segment;
 571     mImageSize = image->GetSize();
 572     segment.AppendFrame(image.forget(), mImageSize, mPrincipal);
 573     mTrack->AppendData(&segment);
 574   }
 575
 576   return 0;
 577 }
 578
 579 uint32_t MediaEngineRemoteVideoSource::GetDistance(
 580     const webrtc::CaptureCapability& aCandidate,
 581     const NormalizedConstraintSet& aConstraints,
 582     const DistanceCalculation aCalculate) const {
 583   if (aCalculate == kFeasibility) {
 584     return GetFeasibilityDistance(aCandidate, aConstraints);
 585   }
 586   return GetFitnessDistance(aCandidate, aConstraints);
 587 }
 588
 589 uint32_t MediaEngineRemoteVideoSource::GetFitnessDistance(
 590     const webrtc::CaptureCapability& aCandidate,
 591     const NormalizedConstraintSet& aConstraints) const {
 592   AssertIsOnOwningThread();
 593
 594   // Treat width|height|frameRate == 0 on capability as "can do any".
 595   // This allows for orthogonal capabilities that are not in discrete steps.
 596
 597   typedef MediaConstraintsHelper H;
 598   uint64_t distance =
 599       uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
 600       uint64_t(aCandidate.width ? H::FitnessDistance(int32_t(aCandidate.width),
 601                                                      aConstraints.mWidth)
 602                                 : 0) +
 603       uint64_t(aCandidate.height
 604                    ? H::FitnessDistance(int32_t(aCandidate.height),
 605                                         aConstraints.mHeight)
 606                    : 0) +
 607       uint64_t(aCandidate.maxFPS ? H::FitnessDistance(double(aCandidate.maxFPS),
 608                                                       aConstraints.mFrameRate)
 609                                  : 0);
 610   return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
 611 }
 612
 613 uint32_t MediaEngineRemoteVideoSource::GetFeasibilityDistance(
 614     const webrtc::CaptureCapability& aCandidate,
 615     const NormalizedConstraintSet& aConstraints) const {
 616   AssertIsOnOwningThread();
 617
 618   // Treat width|height|frameRate == 0 on capability as "can do any".
 619   // This allows for orthogonal capabilities that are not in discrete steps.
 620
 621   typedef MediaConstraintsHelper H;
 622   uint64_t distance =
 623       uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
 624       uint64_t(aCandidate.width
 625                    ? H::FeasibilityDistance(int32_t(aCandidate.width),
 626                                             aConstraints.mWidth)
 627                    : 0) +
 628       uint64_t(aCandidate.height
 629                    ? H::FeasibilityDistance(int32_t(aCandidate.height),
 630                                             aConstraints.mHeight)
 631                    : 0) +
 632       uint64_t(aCandidate.maxFPS
 633                    ? H::FeasibilityDistance(double(aCandidate.maxFPS),
 634                                             aConstraints.mFrameRate)
 635                    : 0);
 636   return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
 637 }
 638
 639 // Find best capability by removing inferiors. May leave >1 of equal distance
 640
 641 /* static */
 642 void MediaEngineRemoteVideoSource::TrimLessFitCandidates(
 643     nsTArray<CapabilityCandidate>& aSet) {
 644   uint32_t best = UINT32_MAX;
 645   for (auto& candidate : aSet) {
 646     if (best > candidate.mDistance) {
 647       best = candidate.mDistance;
 648     }
 649   }
 650   aSet.RemoveElementsBy(
 651       [best](const auto& set) { return set.mDistance > best; });
 652   MOZ_ASSERT(aSet.Length());
 653 }
 654
 655 uint32_t MediaEngineRemoteVideoSource::GetBestFitnessDistance(
 656     const nsTArray<const NormalizedConstraintSet*>& aConstraintSets) const {
 657   AssertIsOnOwningThread();
 658
 659   size_t num = NumCapabilities();
 660   nsTArray<CapabilityCandidate> candidateSet;
 661   for (size_t i = 0; i < num; i++) {
 662     candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
 663   }
 664
 665   bool first = true;
 666   for (const NormalizedConstraintSet* ns : aConstraintSets) {
 667     for (size_t i = 0; i < candidateSet.Length();) {
 668       auto& candidate = candidateSet[i];
 669       uint32_t distance = GetFitnessDistance(candidate.mCapability, *ns);
 670       if (distance == UINT32_MAX) {
 671         candidateSet.RemoveElementAt(i);
 672       } else {
 673         ++i;
 674         if (first) {
 675           candidate.mDistance = distance;
 676         }
 677       }
 678     }
 679     first = false;
 680   }
 681   if (!candidateSet.Length()) {
 682     return UINT32_MAX;
 683   }
 684   TrimLessFitCandidates(candidateSet);
 685   return candidateSet[0].mDistance;
 686 }
 687
 688 static void LogCapability(const char* aHeader,
 689                           const webrtc::CaptureCapability& aCapability,
 690                           uint32_t aDistance) {
 691   static const char* const codec[] = {"VP8",           "VP9",          "H264",
 692                                       "I420",          "RED",          "ULPFEC",
 693                                       "Generic codec", "Unknown codec"};
 694
 695   LOG("%s: %4u x %4u x %2u maxFps, %s. Distance = %" PRIu32, aHeader,
 696       aCapability.width, aCapability.height, aCapability.maxFPS,
 697       codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.videoType)),
 698                      uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
 699       aDistance);
 700 }
 701
 702 bool MediaEngineRemoteVideoSource::ChooseCapability(
 703     const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
 704     webrtc::CaptureCapability& aCapability,
 705     const DistanceCalculation aCalculate) {
 706   LOG("%s", __PRETTY_FUNCTION__);
 707   AssertIsOnOwningThread();
 708
 709   if (MOZ_LOG_TEST(gMediaManagerLog, LogLevel::Debug)) {
 710     LOG("ChooseCapability: prefs: %dx%d @%dfps", aPrefs.GetWidth(),
 711         aPrefs.GetHeight(), aPrefs.mFPS);
 712     MediaConstraintsHelper::LogConstraints(aConstraints);
 713     if (!aConstraints.mAdvanced.empty()) {
 714       LOG("Advanced array[%zu]:", aConstraints.mAdvanced.size());
 715       for (auto& advanced : aConstraints.mAdvanced) {
 716         MediaConstraintsHelper::LogConstraints(advanced);
 717       }
 718     }
 719   }
 720
 721   switch (mCapEngine) {
 722     case camera::ScreenEngine:
 723     case camera::WinEngine: {
 724       FlattenedConstraints c(aConstraints);
 725       // The actual resolution to constrain around is not easy to find ahead of
 726       // time (and may in fact change over time), so as a hack, we push ideal
 727       // and max constraints down to desktop_capture_impl.cc and finish the
 728       // algorithm there.
 729       // TODO: This can be removed in bug 1453269.
 730       aCapability.width =
 731           (std::min(0xffff, c.mWidth.mIdeal.valueOr(0)) & 0xffff) << 16 |
 732           (std::min(0xffff, c.mWidth.mMax) & 0xffff);
 733       aCapability.height =
 734           (std::min(0xffff, c.mHeight.mIdeal.valueOr(0)) & 0xffff) << 16 |
 735           (std::min(0xffff, c.mHeight.mMax) & 0xffff);
 736       aCapability.maxFPS =
 737           c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
 738       return true;
 739     }
 740     default:
 741       break;
 742   }
 743
 744   nsTArray<CapabilityCandidate> candidateSet;
 745   size_t num = NumCapabilities();
 746   for (size_t i = 0; i < num; i++) {
 747     candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
 748   }
 749
 750   if (mCapabilitiesAreHardcoded && mCapEngine == camera::CameraEngine) {
 751     // We have a hardcoded capability, which means this camera didn't report
 752     // discrete capabilities. It might still allow a ranged capability, so we
 753     // add a couple of default candidates based on prefs and constraints.
 754     // The chosen candidate will be propagated to StartCapture() which will fail
 755     // for an invalid candidate.
 756     MOZ_DIAGNOSTIC_ASSERT(mCapabilities.Length() == 1);
 757     MOZ_DIAGNOSTIC_ASSERT(candidateSet.Length() == 1);
 758     candidateSet.Clear();
 759
 760     FlattenedConstraints c(aConstraints);
 761     // Reuse the code across both the low-definition (`false`) pref and
 762     // the high-definition (`true`) pref.
 763     // If there are constraints we try to satisfy them but we default to prefs.
 764     // Note that since constraints are from content and can literally be
 765     // anything we put (rather generous) caps on them.
 766     for (bool isHd : {false, true}) {
 767       webrtc::CaptureCapability cap;
 768       int32_t prefWidth = aPrefs.GetWidth(isHd);
 769       int32_t prefHeight = aPrefs.GetHeight(isHd);
 770
 771       cap.width = c.mWidth.Get(prefWidth);
 772       cap.width = std::max(0, std::min(cap.width, 7680));
 773
 774       cap.height = c.mHeight.Get(prefHeight);
 775       cap.height = std::max(0, std::min(cap.height, 4320));
 776
 777       cap.maxFPS = c.mFrameRate.Get(aPrefs.mFPS);
 778       cap.maxFPS = std::max(0, std::min(cap.maxFPS, 480));
 779
 780       if (cap.width != prefWidth) {
 781         // Width was affected by constraints.
 782         // We'll adjust the height too so the aspect ratio is retained.
 783         cap.height = cap.width * prefHeight / prefWidth;
 784       } else if (cap.height != prefHeight) {
 785         // Height was affected by constraints but not width.
 786         // We'll adjust the width too so the aspect ratio is retained.
 787         cap.width = cap.height * prefWidth / prefHeight;
 788       }
 789
 790       if (candidateSet.Contains(cap, CapabilityComparator())) {
 791         continue;
 792       }
 793       LogCapability("Hardcoded capability", cap, 0);
 794       candidateSet.AppendElement(cap);
 795     }
 796   }
 797
 798   // First, filter capabilities by required constraints (min, max, exact).
 799
 800   for (size_t i = 0; i < candidateSet.Length();) {
 801     auto& candidate = candidateSet[i];
 802     candidate.mDistance =
 803         GetDistance(candidate.mCapability, aConstraints, aCalculate);
 804     LogCapability("Capability", candidate.mCapability, candidate.mDistance);
 805     if (candidate.mDistance == UINT32_MAX) {
 806       candidateSet.RemoveElementAt(i);
 807     } else {
 808       ++i;
 809     }
 810   }
 811
 812   if (candidateSet.IsEmpty()) {
 813     LOG("failed to find capability match from %zu choices",
 814         candidateSet.Length());
 815     return false;
 816   }
 817
 818   // Filter further with all advanced constraints (that don't overconstrain).
 819
 820   for (const auto& cs : aConstraints.mAdvanced) {
 821     nsTArray<CapabilityCandidate> rejects;
 822     for (size_t i = 0; i < candidateSet.Length();) {
 823       if (GetDistance(candidateSet[i].mCapability, cs, aCalculate) ==
 824           UINT32_MAX) {
 825         rejects.AppendElement(candidateSet[i]);
 826         candidateSet.RemoveElementAt(i);
 827       } else {
 828         ++i;
 829       }
 830     }
 831     if (!candidateSet.Length()) {
 832       candidateSet.AppendElements(std::move(rejects));
 833     }
 834   }
 835   MOZ_ASSERT(
 836       candidateSet.Length(),
 837       "advanced constraints filtering step can't reduce candidates to zero");
 838
 839   // Remaining algorithm is up to the UA.
 840
 841   TrimLessFitCandidates(candidateSet);
 842
 843   // Any remaining multiples all have the same distance. A common case of this
 844   // occurs when no ideal is specified. Lean toward defaults.
 845   uint32_t sameDistance = candidateSet[0].mDistance;
 846   {
 847     MediaTrackConstraintSet prefs;
 848     prefs.mWidth.Construct().SetAsLong() = aPrefs.GetWidth();
 849     prefs.mHeight.Construct().SetAsLong() = aPrefs.GetHeight();
 850     prefs.mFrameRate.Construct().SetAsDouble() = aPrefs.mFPS;
 851     NormalizedConstraintSet normPrefs(prefs, false);
 852
 853     for (auto& candidate : candidateSet) {
 854       candidate.mDistance =
 855           GetDistance(candidate.mCapability, normPrefs, aCalculate);
 856     }
 857     TrimLessFitCandidates(candidateSet);
 858   }
 859
 860   aCapability = candidateSet[0].mCapability;
 861
 862   LogCapability("Chosen capability", aCapability, sameDistance);
 863   return true;
 864 }
 865
 866 void MediaEngineRemoteVideoSource::GetSettings(
 867     MediaTrackSettings& aOutSettings) const {
 868   aOutSettings = *mSettings;
 869 }
 870
 871 }  // namespace mozilla