widget/windows/WindowsSMTCProvider.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 /* mingw currently doesn't support windows.media.h, so we disable
   7  * the whole related class until this is fixed.
   8  * @TODO: Maybe contact MinGW Team for inclusion?*/
   9 #ifndef __MINGW32__
  10
  11 #  include "WindowsSMTCProvider.h"
  12
  13 #  include <windows.h>
  14 #  include <windows.media.h>
  15 #  include <wrl.h>
  16
  17 #  include "nsMimeTypes.h"
  18 #  include "mozilla/Assertions.h"
  19 #  include "mozilla/Logging.h"
  20 #  include "mozilla/Maybe.h"
  21 #  include "mozilla/WidgetUtils.h"
  22 #  include "mozilla/ScopeExit.h"
  23 #  include "mozilla/dom/MediaControlUtils.h"
  24 #  include "mozilla/media/MediaUtils.h"
  25 #  include "nsThreadUtils.h"
  26
  27 #  pragma comment(lib, "runtimeobject.lib")
  28
  29 using namespace ABI::Windows::Foundation;
  30 using namespace ABI::Windows::Media;
  31 using namespace ABI::Windows::Storage::Streams;
  32 using namespace Microsoft::WRL;
  33 using namespace Microsoft::WRL::Wrappers;
  34 using namespace mozilla;
  35
  36 #  ifndef RuntimeClass_Windows_Media_SystemMediaTransportControls
  37 #    define RuntimeClass_Windows_Media_SystemMediaTransportControls \
  38       L"Windows.Media.SystemMediaTransportControls"
  39 #  endif
  40
  41 #  ifndef RuntimeClass_Windows_Storage_Streams_RandomAccessStreamReference
  42 #    define RuntimeClass_Windows_Storage_Streams_RandomAccessStreamReference \
  43       L"Windows.Storage.Streams.RandomAccessStreamReference"
  44 #  endif
  45
  46 #  ifndef ISystemMediaTransportControlsInterop
  47 EXTERN_C const IID IID_ISystemMediaTransportControlsInterop;
  48 MIDL_INTERFACE("ddb0472d-c911-4a1f-86d9-dc3d71a95f5a")
  49 ISystemMediaTransportControlsInterop : public IInspectable {
  50  public:
  51   virtual HRESULT STDMETHODCALLTYPE GetForWindow(
  52       /* [in] */ __RPC__in HWND appWindow,
  53       /* [in] */ __RPC__in REFIID riid,
  54       /* [iid_is][retval][out] */
  55       __RPC__deref_out_opt void** mediaTransportControl) = 0;
  56 };
  57 #  endif /* __ISystemMediaTransportControlsInterop_INTERFACE_DEFINED__ */
  58
  59 extern mozilla::LazyLogModule gMediaControlLog;
  60
  61 #  undef LOG
  62 #  define LOG(msg, ...)                        \
  63     MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
  64             ("WindowSMTCProvider=%p, " msg, this, ##__VA_ARGS__))
  65
  66 static inline Maybe<mozilla::dom::MediaControlKey> TranslateKeycode(
  67     SystemMediaTransportControlsButton keycode) {
  68   switch (keycode) {
  69     case SystemMediaTransportControlsButton_Play:
  70       return Some(mozilla::dom::MediaControlKey::Play);
  71     case SystemMediaTransportControlsButton_Pause:
  72       return Some(mozilla::dom::MediaControlKey::Pause);
  73     case SystemMediaTransportControlsButton_Next:
  74       return Some(mozilla::dom::MediaControlKey::Nexttrack);
  75     case SystemMediaTransportControlsButton_Previous:
  76       return Some(mozilla::dom::MediaControlKey::Previoustrack);
  77     case SystemMediaTransportControlsButton_Stop:
  78       return Some(mozilla::dom::MediaControlKey::Stop);
  79     case SystemMediaTransportControlsButton_FastForward:
  80       return Some(mozilla::dom::MediaControlKey::Seekforward);
  81     case SystemMediaTransportControlsButton_Rewind:
  82       return Some(mozilla::dom::MediaControlKey::Seekbackward);
  83     default:
  84       return Nothing();  // Not supported Button
  85   }
  86 }
  87
  88 static IAsyncInfo* GetIAsyncInfo(IAsyncOperation<unsigned int>* aAsyncOp) {
  89   MOZ_ASSERT(aAsyncOp);
  90   IAsyncInfo* asyncInfo;
  91   HRESULT hr = aAsyncOp->QueryInterface(IID_IAsyncInfo,
  92                                         reinterpret_cast<void**>(&asyncInfo));
  93   // The assertion always works since IAsyncOperation implements IAsyncInfo
  94   MOZ_ASSERT(SUCCEEDED(hr));
  95   Unused << hr;
  96   MOZ_ASSERT(asyncInfo);
  97   return asyncInfo;
  98 }
  99
 100 WindowsSMTCProvider::WindowsSMTCProvider() {
 101   LOG("Creating an empty and invisible window");
 102
 103   // In order to create a SMTC-Provider, we need a hWnd, which shall be created
 104   // dynamically from an invisible window. This leads to the following
 105   // boilerplate code.
 106   WNDCLASS wnd{};
 107   wnd.lpszClassName = L"Firefox-MediaKeys";
 108   wnd.hInstance = nullptr;
 109   wnd.lpfnWndProc = DefWindowProc;
 110   GetLastError();  // Clear the error
 111   RegisterClass(&wnd);
 112   MOZ_ASSERT(!GetLastError());
 113
 114   mWindow = CreateWindowExW(0, L"Firefox-MediaKeys", L"Firefox Media Keys", 0,
 115                             CW_USEDEFAULT, CW_USEDEFAULT, 0, 0, nullptr,
 116                             nullptr, nullptr, nullptr);
 117   MOZ_ASSERT(mWindow);
 118   MOZ_ASSERT(!GetLastError());
 119 }
 120
 121 WindowsSMTCProvider::~WindowsSMTCProvider() {
 122   // Dispose the window
 123   MOZ_ASSERT(mWindow);
 124   if (!DestroyWindow(mWindow)) {
 125     LOG("Failed to destroy the hidden window. Error Code: %lu", GetLastError());
 126   }
 127   if (!UnregisterClass(L"Firefox-MediaKeys", nullptr)) {
 128     // Note that this is logged when the class wasn't even registered.
 129     LOG("Failed to unregister the class. Error Code: %lu", GetLastError());
 130   }
 131 }
 132
 133 bool WindowsSMTCProvider::IsOpened() const { return mInitialized; }
 134
 135 bool WindowsSMTCProvider::Open() {
 136   LOG("Opening Source");
 137   MOZ_ASSERT(!mInitialized);
 138
 139   if (!InitDisplayAndControls()) {
 140     LOG("Failed to initialize the SMTC and its display");
 141     return false;
 142   }
 143
 144   if (!UpdateButtons()) {
 145     LOG("Failed to initialize the buttons");
 146     return false;
 147   }
 148
 149   if (!RegisterEvents()) {
 150     LOG("Failed to register SMTC key-event listener");
 151     return false;
 152   }
 153
 154   if (!EnableControl(true)) {
 155     LOG("Failed to enable SMTC control");
 156     return false;
 157   }
 158
 159   mInitialized = true;
 160   SetPlaybackState(mozilla::dom::MediaSessionPlaybackState::None);
 161   return mInitialized;
 162 }
 163
 164 void WindowsSMTCProvider::Close() {
 165   MediaControlKeySource::Close();
 166   // Prevent calling Set methods when init failed
 167   if (mInitialized) {
 168     SetPlaybackState(mozilla::dom::MediaSessionPlaybackState::None);
 169     UnregisterEvents();
 170     ClearMetadata();
 171     // We have observed an Windows issue, if we modify `mControls` , (such as
 172     // setting metadata, disable buttons) before disabling control, and those
 173     // operations are not done sequentially within a same main thread task,
 174     // then it would cause a problem where the SMTC wasn't clean up completely
 175     // and show the executable name.
 176     EnableControl(false);
 177     mInitialized = false;
 178   }
 179 }
 180
 181 void WindowsSMTCProvider::SetPlaybackState(
 182     mozilla::dom::MediaSessionPlaybackState aState) {
 183   MOZ_ASSERT(mInitialized);
 184   MediaControlKeySource::SetPlaybackState(aState);
 185
 186   HRESULT hr;
 187
 188   // Note: we can't return the status of put_PlaybackStatus, but we can at least
 189   // assert it.
 190   switch (aState) {
 191     case mozilla::dom::MediaSessionPlaybackState::Paused:
 192       hr = mControls->put_PlaybackStatus(
 193           ABI::Windows::Media::MediaPlaybackStatus_Paused);
 194       break;
 195     case mozilla::dom::MediaSessionPlaybackState::Playing:
 196       hr = mControls->put_PlaybackStatus(
 197           ABI::Windows::Media::MediaPlaybackStatus_Playing);
 198       break;
 199     case mozilla::dom::MediaSessionPlaybackState::None:
 200       hr = mControls->put_PlaybackStatus(
 201           ABI::Windows::Media::MediaPlaybackStatus_Stopped);
 202       break;
 203       // MediaPlaybackStatus still supports Closed and Changing, which we don't
 204       // use (yet)
 205     default:
 206       MOZ_ASSERT_UNREACHABLE(
 207           "Enum Inconsitency between PlaybackState and WindowsSMTCProvider");
 208       break;
 209   }
 210
 211   MOZ_ASSERT(SUCCEEDED(hr));
 212   Unused << hr;
 213 }
 214
 215 void WindowsSMTCProvider::SetMediaMetadata(
 216     const mozilla::dom::MediaMetadataBase& aMetadata) {
 217   MOZ_ASSERT(mInitialized);
 218   SetMusicMetadata(aMetadata.mArtist, aMetadata.mTitle);
 219   LoadThumbnail(aMetadata.mArtwork);
 220 }
 221
 222 void WindowsSMTCProvider::ClearMetadata() {
 223   MOZ_ASSERT(mDisplay);
 224   if (FAILED(mDisplay->ClearAll())) {
 225     LOG("Failed to clear SMTC display");
 226   }
 227   mImageFetchRequest.DisconnectIfExists();
 228   CancelPendingStoreAsyncOperation();
 229   mThumbnailUrl.Truncate();
 230   mProcessingUrl.Truncate();
 231   mNextImageIndex = 0;
 232   mSupportedKeys = 0;
 233 }
 234
 235 void WindowsSMTCProvider::SetSupportedMediaKeys(
 236     const MediaKeysArray& aSupportedKeys) {
 237   MOZ_ASSERT(mInitialized);
 238
 239   uint32_t supportedKeys = 0;
 240   for (const mozilla::dom::MediaControlKey& key : aSupportedKeys) {
 241     supportedKeys |= GetMediaKeyMask(key);
 242   }
 243
 244   if (supportedKeys == mSupportedKeys) {
 245     LOG("Supported keys stay the same");
 246     return;
 247   }
 248
 249   LOG("Update supported keys");
 250   mSupportedKeys = supportedKeys;
 251   UpdateButtons();
 252 }
 253
 254 void WindowsSMTCProvider::UnregisterEvents() {
 255   if (mControls && mButtonPressedToken.value != 0) {
 256     mControls->remove_ButtonPressed(mButtonPressedToken);
 257   }
 258
 259   if (mControls && mSeekRegistrationToken.value != 0) {
 260     ComPtr<ISystemMediaTransportControls2> smtc2;
 261     HRESULT hr = mControls.As(&smtc2);
 262     if (FAILED(hr)) {
 263       LOG("Failed to cast controls to ISystemMediaTransportControls2 (hr=%lx)",
 264           hr);
 265       return;
 266     }
 267     MOZ_ASSERT(smtc2);
 268
 269     hr = smtc2->remove_PlaybackPositionChangeRequested(mSeekRegistrationToken);
 270     if (FAILED(hr)) {
 271       LOG("SystemMediaTransportControls: Failed unregister position change "
 272           "event (hr=%lx)",
 273           hr);
 274       return;
 275     }
 276   }
 277 }
 278
 279 bool WindowsSMTCProvider::RegisterEvents() {
 280   MOZ_ASSERT(mControls);
 281   auto self = RefPtr<WindowsSMTCProvider>(this);
 282   auto callbackbtnPressed = Callback<
 283       ITypedEventHandler<SystemMediaTransportControls*,
 284                          SystemMediaTransportControlsButtonPressedEventArgs*>>(
 285       [this, self](ISystemMediaTransportControls*,
 286                    ISystemMediaTransportControlsButtonPressedEventArgs* pArgs)
 287           -> HRESULT {
 288         MOZ_ASSERT(pArgs);
 289         SystemMediaTransportControlsButton btn;
 290
 291         if (FAILED(pArgs->get_Button(&btn))) {
 292           LOG("SystemMediaTransportControls: ButtonPressedEvent - Could "
 293               "not get Button.");
 294           return S_OK;  // Propagating the error probably wouldn't help.
 295         }
 296
 297         Maybe<mozilla::dom::MediaControlKey> keyCode = TranslateKeycode(btn);
 298         if (keyCode.isSome() && IsOpened()) {
 299           OnButtonPressed(keyCode.value());
 300         }
 301         return S_OK;
 302       });
 303
 304   if (FAILED(mControls->add_ButtonPressed(callbackbtnPressed.Get(),
 305                                           &mButtonPressedToken))) {
 306     LOG("SystemMediaTransportControls: Failed at "
 307         "registerEvents().add_ButtonPressed()");
 308     return false;
 309   }
 310
 311   ComPtr<ISystemMediaTransportControls2> smtc2;
 312   HRESULT hr = mControls.As(&smtc2);
 313   if (FAILED(hr)) {
 314     LOG("Failed to cast controls to ISystemMediaTransportControls2 (hr=%lx)",
 315         hr);
 316     return false;
 317   }
 318   MOZ_ASSERT(smtc2);
 319
 320   auto positionChangeRequested =
 321       Callback<ITypedEventHandler<SystemMediaTransportControls*,
 322                                   PlaybackPositionChangeRequestedEventArgs*>>(
 323           [this, self](
 324               ISystemMediaTransportControls*,
 325               IPlaybackPositionChangeRequestedEventArgs* pArgs) -> HRESULT {
 326             MOZ_ASSERT(pArgs);
 327
 328             TimeSpan value;
 329             HRESULT hr = pArgs->get_RequestedPlaybackPosition(&value);
 330             if (FAILED(hr)) {
 331               LOG("SystemMediaTransportControls: Playback Position Change - "
 332                   "failed to get requested position (hr=%lx)",
 333                   hr);
 334               return S_OK;  // Propagating the error probably wouldn't help.
 335             }
 336
 337             double position =
 338                 static_cast<double>(value.Duration) / (1e6 * 10.0);
 339             this->OnPositionChangeRequested(position);
 340
 341             return S_OK;
 342           });
 343
 344   hr = smtc2->add_PlaybackPositionChangeRequested(positionChangeRequested.Get(),
 345                                                   &mSeekRegistrationToken);
 346   if (FAILED(hr)) {
 347     LOG("SystemMediaTransportControls: Failed to register position change "
 348         "event (hr=%lx)",
 349         hr);
 350     return false;
 351   }
 352
 353   return true;
 354 }
 355
 356 void WindowsSMTCProvider::OnButtonPressed(
 357     mozilla::dom::MediaControlKey aKey) const {
 358   if (!IsKeySupported(aKey)) {
 359     LOG("key: %s is not supported", dom::GetEnumString(aKey).get());
 360     return;
 361   }
 362
 363   for (auto& listener : mListeners) {
 364     listener->OnActionPerformed(mozilla::dom::MediaControlAction(aKey));
 365   }
 366 }
 367
 368 bool WindowsSMTCProvider::EnableControl(bool aEnabled) const {
 369   MOZ_ASSERT(mControls);
 370   return SUCCEEDED(mControls->put_IsEnabled(aEnabled));
 371 }
 372
 373 bool WindowsSMTCProvider::UpdateButtons() {
 374   static const mozilla::dom::MediaControlKey kKeys[] = {
 375       mozilla::dom::MediaControlKey::Play,
 376       mozilla::dom::MediaControlKey::Pause,
 377       mozilla::dom::MediaControlKey::Previoustrack,
 378       mozilla::dom::MediaControlKey::Nexttrack,
 379       mozilla::dom::MediaControlKey::Stop,
 380       mozilla::dom::MediaControlKey::Seekto};
 381
 382   bool success = true;
 383   for (const mozilla::dom::MediaControlKey& key : kKeys) {
 384     if (!EnableKey(key, IsKeySupported(key))) {
 385       success = false;
 386       LOG("Failed to set %s=%s", dom::GetEnumString(key).get(),
 387           IsKeySupported(key) ? "true" : "false");
 388     }
 389   }
 390
 391   return success;
 392 }
 393
 394 bool WindowsSMTCProvider::IsKeySupported(
 395     mozilla::dom::MediaControlKey aKey) const {
 396   return mSupportedKeys & GetMediaKeyMask(aKey);
 397 }
 398
 399 bool WindowsSMTCProvider::EnableKey(mozilla::dom::MediaControlKey aKey,
 400                                     bool aEnable) const {
 401   MOZ_ASSERT(mControls);
 402   switch (aKey) {
 403     case mozilla::dom::MediaControlKey::Play:
 404       return SUCCEEDED(mControls->put_IsPlayEnabled(aEnable));
 405     case mozilla::dom::MediaControlKey::Pause:
 406       return SUCCEEDED(mControls->put_IsPauseEnabled(aEnable));
 407     case mozilla::dom::MediaControlKey::Previoustrack:
 408       return SUCCEEDED(mControls->put_IsPreviousEnabled(aEnable));
 409     case mozilla::dom::MediaControlKey::Nexttrack:
 410       return SUCCEEDED(mControls->put_IsNextEnabled(aEnable));
 411     case mozilla::dom::MediaControlKey::Stop:
 412       return SUCCEEDED(mControls->put_IsStopEnabled(aEnable));
 413     case mozilla::dom::MediaControlKey::Seekto:
 414       // The callback for the event checks if the key is supported
 415       return mSeekRegistrationToken.value != 0;
 416     default:
 417       LOG("No button for %s", dom::GetEnumString(aKey).get());
 418       return false;
 419   }
 420 }
 421
 422 void WindowsSMTCProvider::OnPositionChangeRequested(double aPosition) const {
 423   if (!IsKeySupported(mozilla::dom::MediaControlKey::Seekto)) {
 424     LOG("Seekto is not supported");
 425     return;
 426   }
 427
 428   for (const auto& listener : mListeners) {
 429     listener->OnActionPerformed(
 430         mozilla::dom::MediaControlAction(mozilla::dom::MediaControlKey::Seekto,
 431                                          mozilla::dom::SeekDetails(aPosition)));
 432   }
 433 }
 434
 435 bool WindowsSMTCProvider::InitDisplayAndControls() {
 436   // As Open() might be called multiple times, "cache" the results of the COM
 437   // API
 438   if (mControls && mDisplay) {
 439     return true;
 440   }
 441   ComPtr<ISystemMediaTransportControlsInterop> interop;
 442   HRESULT hr = GetActivationFactory(
 443       HStringReference(RuntimeClass_Windows_Media_SystemMediaTransportControls)
 444           .Get(),
 445       interop.GetAddressOf());
 446   if (FAILED(hr)) {
 447     LOG("SystemMediaTransportControls: Failed at instantiating the "
 448         "Interop object");
 449     return false;
 450   }
 451   MOZ_ASSERT(interop);
 452
 453   if (!mControls && FAILED(interop->GetForWindow(
 454                         mWindow, IID_PPV_ARGS(mControls.GetAddressOf())))) {
 455     LOG("SystemMediaTransportControls: Failed at GetForWindow()");
 456     return false;
 457   }
 458   MOZ_ASSERT(mControls);
 459
 460   if (!mDisplay &&
 461       FAILED(mControls->get_DisplayUpdater(mDisplay.GetAddressOf()))) {
 462     LOG("SystemMediaTransportControls: Failed at get_DisplayUpdater()");
 463   }
 464
 465   MOZ_ASSERT(mDisplay);
 466   return true;
 467 }
 468
 469 bool WindowsSMTCProvider::SetMusicMetadata(const nsString& aArtist,
 470                                            const nsString& aTitle) {
 471   MOZ_ASSERT(mDisplay);
 472   ComPtr<IMusicDisplayProperties> musicProps;
 473
 474   HRESULT hr = mDisplay->put_Type(MediaPlaybackType::MediaPlaybackType_Music);
 475   MOZ_ASSERT(SUCCEEDED(hr));
 476   Unused << hr;
 477   hr = mDisplay->get_MusicProperties(musicProps.GetAddressOf());
 478   if (FAILED(hr)) {
 479     LOG("Failed to get music properties");
 480     return false;
 481   }
 482
 483   hr = musicProps->put_Artist(HStringReference(aArtist.get()).Get());
 484   if (FAILED(hr)) {
 485     LOG("Failed to set the music's artist");
 486     return false;
 487   }
 488
 489   hr = musicProps->put_Title(HStringReference(aTitle.get()).Get());
 490   if (FAILED(hr)) {
 491     LOG("Failed to set the music's title");
 492     return false;
 493   }
 494
 495   hr = mDisplay->Update();
 496   if (FAILED(hr)) {
 497     LOG("Failed to refresh the display");
 498     return false;
 499   }
 500
 501   return true;
 502 }
 503
 504 void WindowsSMTCProvider::SetPositionState(
 505     const mozilla::Maybe<mozilla::dom::PositionState>& aState) {
 506   ComPtr<ISystemMediaTransportControls2> smtc2;
 507   HRESULT hr = mControls.As(&smtc2);
 508   if (FAILED(hr)) {
 509     LOG("Failed to cast controls to ISystemMediaTransportControls2 (hr=%lx)",
 510         hr);
 511     return;
 512   }
 513   MOZ_ASSERT(smtc2);
 514
 515   ComPtr<ISystemMediaTransportControlsTimelineProperties> properties;
 516   hr = RoActivateInstance(
 517       HStringReference(
 518           RuntimeClass_Windows_Media_SystemMediaTransportControlsTimelineProperties)
 519           .Get(),
 520       &properties);
 521   if (FAILED(hr)) {
 522     LOG("Failed to create timeline properties (hr=%lx)", hr);
 523     return;
 524   }
 525   MOZ_ASSERT(properties);
 526
 527   // Converts a value in seconds to a TimeSpan
 528   // The TimeSpan's Duration is a value in 100ns ticks
 529   // https://learn.microsoft.com/en-us/windows/win32/api/windows.foundation/ns-windows-foundation-timespan#members
 530   auto toTimeSpan = [this](double seconds) {
 531     constexpr double kMaxMicroseconds =
 532         static_cast<double>(std::numeric_limits<LONG64>::max() / 10);
 533
 534     double microseconds = seconds * 1e6;
 535
 536     if (microseconds > kMaxMicroseconds) {
 537       LOG("Failed to convert %f microseconds to TimeSpan (overflow)",
 538           microseconds);
 539       return TimeSpan{0};
 540     }
 541
 542     return TimeSpan{static_cast<LONG64>(microseconds * 10.0)};
 543   };
 544
 545   TimeSpan endTime{0};
 546   TimeSpan position{0};
 547   double playbackRate = 1.0;
 548
 549   if (aState) {
 550     endTime = toTimeSpan(aState->mDuration);
 551     position = toTimeSpan(aState->CurrentPlaybackPosition());
 552     playbackRate = aState->mPlaybackRate;
 553   }
 554
 555   hr = properties->put_StartTime({0});
 556   if (FAILED(hr)) {
 557     LOG("Failed to set the start time (hr=%lx)", hr);
 558     return;
 559   }
 560
 561   hr = properties->put_MinSeekTime({0});
 562   if (FAILED(hr)) {
 563     LOG("Failed to set the min seek time (hr=%lx)", hr);
 564     return;
 565   }
 566
 567   hr = properties->put_EndTime(endTime);
 568   if (FAILED(hr)) {
 569     LOG("Failed to set the end time (hr=%lx)", hr);
 570     return;
 571   }
 572
 573   hr = properties->put_MaxSeekTime(endTime);
 574   if (FAILED(hr)) {
 575     LOG("Failed to set the max seek time (hr=%lx)", hr);
 576     return;
 577   }
 578
 579   hr = properties->put_Position(position);
 580   if (FAILED(hr)) {
 581     LOG("Failed to set the playback position (hr=%lx)", hr);
 582     return;
 583   }
 584
 585   hr = smtc2->UpdateTimelineProperties(properties.Get());
 586   if (FAILED(hr)) {
 587     LOG("Failed to update timeline properties (hr=%lx)", hr);
 588     return;
 589   }
 590
 591   hr = smtc2->put_PlaybackRate(playbackRate);
 592   if (FAILED(hr)) {
 593     LOG("Failed to set the playback rate (hr=%lx)", hr);
 594     return;
 595   }
 596 }
 597
 598 void WindowsSMTCProvider::LoadThumbnail(
 599     const nsTArray<mozilla::dom::MediaImage>& aArtwork) {
 600   MOZ_ASSERT(NS_IsMainThread());
 601
 602   // TODO: Sort the images by the preferred size or format.
 603   mArtwork = aArtwork;
 604   mNextImageIndex = 0;
 605
 606   // Abort the loading if
 607   // 1) thumbnail is being updated, and one in processing is in the artwork
 608   // 2) thumbnail is not being updated, and one in use is in the artwork
 609   if (!mProcessingUrl.IsEmpty()) {
 610     LOG("Load thumbnail while image: %s is being processed",
 611         NS_ConvertUTF16toUTF8(mProcessingUrl).get());
 612     if (mozilla::dom::IsImageIn(mArtwork, mProcessingUrl)) {
 613       LOG("No need to load thumbnail. The one being processed is in the "
 614           "artwork");
 615       return;
 616     }
 617   } else if (!mThumbnailUrl.IsEmpty()) {
 618     if (mozilla::dom::IsImageIn(mArtwork, mThumbnailUrl)) {
 619       LOG("No need to load thumbnail. The one in use is in the artwork");
 620       return;
 621     }
 622   }
 623
 624   // If there is a pending image store operation, that image must be different
 625   // from the new image will be loaded below, so the pending one should be
 626   // cancelled.
 627   CancelPendingStoreAsyncOperation();
 628   // Remove the current thumbnail on the interface
 629   ClearThumbnail();
 630   // Then load the new thumbnail asynchronously
 631   LoadImageAtIndex(mNextImageIndex++);
 632 }
 633
 634 void WindowsSMTCProvider::LoadImageAtIndex(const size_t aIndex) {
 635   MOZ_ASSERT(NS_IsMainThread());
 636
 637   if (aIndex >= mArtwork.Length()) {
 638     LOG("Stop loading thumbnail. No more available images");
 639     mImageFetchRequest.DisconnectIfExists();
 640     mProcessingUrl.Truncate();
 641     return;
 642   }
 643
 644   const mozilla::dom::MediaImage& image = mArtwork[aIndex];
 645
 646   // TODO: No need to fetch the default image and do image processing since the
 647   // the default image is local file and it's trustworthy. For the default
 648   // image, we can use `CreateFromFile` to create the IRandomAccessStream. We
 649   // should probably cache it since it could be used very often (Bug 1643102)
 650
 651   if (!mozilla::dom::IsValidImageUrl(image.mSrc)) {
 652     LOG("Skip the image with invalid URL. Try next image");
 653     mImageFetchRequest.DisconnectIfExists();
 654     LoadImageAtIndex(mNextImageIndex++);
 655     return;
 656   }
 657
 658   mImageFetchRequest.DisconnectIfExists();
 659   mProcessingUrl = image.mSrc;
 660
 661   mImageFetcher = mozilla::MakeUnique<mozilla::dom::FetchImageHelper>(image);
 662   RefPtr<WindowsSMTCProvider> self = this;
 663   mImageFetcher->FetchImage()
 664       ->Then(
 665           AbstractThread::MainThread(), __func__,
 666           [this, self](const nsCOMPtr<imgIContainer>& aImage) {
 667             LOG("The image is fetched successfully");
 668             mImageFetchRequest.Complete();
 669
 670             // Although IMAGE_JPEG or IMAGE_BMP are valid types as well, but a
 671             // png image with transparent background will be converted into a
 672             // jpeg/bmp file with a colored background. IMAGE_PNG format seems
 673             // to be the best choice for now.
 674             uint32_t size = 0;
 675             char* src = nullptr;
 676             // Only used to hold the image data
 677             nsCOMPtr<nsIInputStream> inputStream;
 678             nsresult rv = mozilla::dom::GetEncodedImageBuffer(
 679                 aImage, nsLiteralCString(IMAGE_PNG),
 680                 getter_AddRefs(inputStream), &size, &src);
 681             if (NS_FAILED(rv) || !inputStream || size == 0 || !src) {
 682               LOG("Failed to get the image buffer info. Try next image");
 683               LoadImageAtIndex(mNextImageIndex++);
 684               return;
 685             }
 686
 687             LoadImage(src, size);
 688           },
 689           [this, self](bool) {
 690             LOG("Failed to fetch image. Try next image");
 691             mImageFetchRequest.Complete();
 692             LoadImageAtIndex(mNextImageIndex++);
 693           })
 694       ->Track(mImageFetchRequest);
 695 }
 696
 697 void WindowsSMTCProvider::LoadImage(const char* aImageData,
 698                                     uint32_t aDataSize) {
 699   MOZ_ASSERT(NS_IsMainThread());
 700
 701   // 1. Use mImageDataWriter to write the binary data of image into mImageStream
 702   // 2. Refer the image by mImageStreamReference and then set it to the SMTC
 703   // In case of the race condition between they are being destroyed and the
 704   // async operation for image loading, mImageDataWriter, mImageStream, and
 705   // mImageStreamReference are member variables
 706
 707   HRESULT hr = ActivateInstance(
 708       HStringReference(
 709           RuntimeClass_Windows_Storage_Streams_InMemoryRandomAccessStream)
 710           .Get(),
 711       mImageStream.GetAddressOf());
 712   if (FAILED(hr)) {
 713     LOG("Failed to make mImageStream refer to an instance of "
 714         "InMemoryRandomAccessStream");
 715     return;
 716   }
 717
 718   ComPtr<IOutputStream> outputStream;
 719   hr = mImageStream.As(&outputStream);
 720   if (FAILED(hr)) {
 721     LOG("Failed when query IOutputStream interface from mImageStream");
 722     return;
 723   }
 724
 725   ComPtr<IDataWriterFactory> dataWriterFactory;
 726   hr = GetActivationFactory(
 727       HStringReference(RuntimeClass_Windows_Storage_Streams_DataWriter).Get(),
 728       dataWriterFactory.GetAddressOf());
 729   if (FAILED(hr)) {
 730     LOG("Failed to get an activation factory for IDataWriterFactory");
 731     return;
 732   }
 733
 734   hr = dataWriterFactory->CreateDataWriter(outputStream.Get(),
 735                                            mImageDataWriter.GetAddressOf());
 736   if (FAILED(hr)) {
 737     LOG("Failed to create mImageDataWriter that writes data to mImageStream");
 738     return;
 739   }
 740
 741   hr = mImageDataWriter->WriteBytes(
 742       aDataSize, reinterpret_cast<BYTE*>(const_cast<char*>(aImageData)));
 743   if (FAILED(hr)) {
 744     LOG("Failed to write data to mImageStream");
 745     return;
 746   }
 747
 748   hr = mImageDataWriter->StoreAsync(&mStoreAsyncOperation);
 749   if (FAILED(hr)) {
 750     LOG("Failed to create a DataWriterStoreOperation for mStoreAsyncOperation");
 751     return;
 752   }
 753
 754   // Upon the image is stored in mImageStream, set the image to the SMTC
 755   // interface
 756   auto onStoreCompleted = Callback<
 757       IAsyncOperationCompletedHandler<unsigned int>>(
 758       [this, self = RefPtr<WindowsSMTCProvider>(this),
 759        aImageUrl = nsString(mProcessingUrl)](
 760           IAsyncOperation<unsigned int>* aAsyncOp, AsyncStatus aStatus) {
 761         MOZ_ASSERT(NS_IsMainThread());
 762
 763         if (aStatus != AsyncStatus::Completed) {
 764           LOG("Asynchronous operation is not completed");
 765           return E_ABORT;
 766         }
 767
 768         HRESULT hr = S_OK;
 769         IAsyncInfo* asyncInfo = GetIAsyncInfo(aAsyncOp);
 770         asyncInfo->get_ErrorCode(&hr);
 771         if (FAILED(hr)) {
 772           LOG("Failed to get termination status of the asynchronous operation");
 773           return hr;
 774         }
 775
 776         if (!UpdateThumbnail(aImageUrl)) {
 777           LOG("Failed to update thumbnail");
 778         }
 779
 780         // If an error occurs above:
 781         // - If aImageUrl is not mProcessingUrl. It's fine.
 782         // - If aImageUrl is mProcessingUrl, then mProcessingUrl won't be reset.
 783         //   Therefore the thumbnail will remain empty until a new image whose
 784         //   url is different from mProcessingUrl is loaded.
 785
 786         return S_OK;
 787       });
 788
 789   hr = mStoreAsyncOperation->put_Completed(onStoreCompleted.Get());
 790   if (FAILED(hr)) {
 791     LOG("Failed to set callback on completeing the asynchronous operation");
 792   }
 793 }
 794
 795 bool WindowsSMTCProvider::SetThumbnail(const nsAString& aUrl) {
 796   MOZ_ASSERT(mDisplay);
 797   MOZ_ASSERT(mImageStream);
 798   MOZ_ASSERT(!aUrl.IsEmpty());
 799
 800   ComPtr<IRandomAccessStreamReferenceStatics> streamRefFactory;
 801
 802   HRESULT hr = GetActivationFactory(
 803       HStringReference(
 804           RuntimeClass_Windows_Storage_Streams_RandomAccessStreamReference)
 805           .Get(),
 806       streamRefFactory.GetAddressOf());
 807   auto cleanup =
 808       MakeScopeExit([this, self = RefPtr<WindowsSMTCProvider>(this)] {
 809         LOG("Clean mThumbnailUrl");
 810         mThumbnailUrl.Truncate();
 811       });
 812
 813   if (FAILED(hr)) {
 814     LOG("Failed to get an activation factory for "
 815         "IRandomAccessStreamReferenceStatics type");
 816     return false;
 817   }
 818
 819   hr = streamRefFactory->CreateFromStream(mImageStream.Get(),
 820                                           mImageStreamReference.GetAddressOf());
 821   if (FAILED(hr)) {
 822     LOG("Failed to create mImageStreamReference from mImageStream");
 823     return false;
 824   }
 825
 826   hr = mDisplay->put_Thumbnail(mImageStreamReference.Get());
 827   if (FAILED(hr)) {
 828     LOG("Failed to update thumbnail");
 829     return false;
 830   }
 831
 832   hr = mDisplay->Update();
 833   if (FAILED(hr)) {
 834     LOG("Failed to refresh display");
 835     return false;
 836   }
 837
 838   // No need to clean mThumbnailUrl since thumbnail is set successfully
 839   cleanup.release();
 840   mThumbnailUrl = aUrl;
 841
 842   return true;
 843 }
 844
 845 void WindowsSMTCProvider::ClearThumbnail() {
 846   MOZ_ASSERT(mDisplay);
 847   HRESULT hr = mDisplay->put_Thumbnail(nullptr);
 848   MOZ_ASSERT(SUCCEEDED(hr));
 849   hr = mDisplay->Update();
 850   MOZ_ASSERT(SUCCEEDED(hr));
 851   Unused << hr;
 852   mThumbnailUrl.Truncate();
 853 }
 854
 855 bool WindowsSMTCProvider::UpdateThumbnail(const nsAString& aUrl) {
 856   MOZ_ASSERT(NS_IsMainThread());
 857
 858   if (!IsOpened()) {
 859     LOG("Abort the thumbnail update: SMTC is closed");
 860     return false;
 861   }
 862
 863   if (aUrl != mProcessingUrl) {
 864     LOG("Abort the thumbnail update: The image from %s is out of date",
 865         NS_ConvertUTF16toUTF8(aUrl).get());
 866     return false;
 867   }
 868
 869   mProcessingUrl.Truncate();
 870
 871   if (!SetThumbnail(aUrl)) {
 872     LOG("Failed to update thumbnail");
 873     return false;
 874   }
 875
 876   MOZ_ASSERT(mThumbnailUrl == aUrl);
 877   LOG("The thumbnail is updated to the image from: %s",
 878       NS_ConvertUTF16toUTF8(mThumbnailUrl).get());
 879   return true;
 880 }
 881
 882 void WindowsSMTCProvider::CancelPendingStoreAsyncOperation() const {
 883   if (mStoreAsyncOperation) {
 884     IAsyncInfo* asyncInfo = GetIAsyncInfo(mStoreAsyncOperation.Get());
 885     asyncInfo->Cancel();
 886   }
 887 }
 888
 889 #endif  // __MINGW32__