dom/html/TextTrackManager.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "mozilla/dom/TextTrackManager.h"
   8 #include "mozilla/ClearOnShutdown.h"
   9 #include "mozilla/CycleCollectedJSContext.h"
  10 #include "mozilla/Maybe.h"
  11 #include "mozilla/Telemetry.h"
  12 #include "mozilla/dom/Document.h"
  13 #include "mozilla/dom/Event.h"
  14 #include "mozilla/dom/HTMLMediaElement.h"
  15 #include "mozilla/dom/HTMLTrackElement.h"
  16 #include "mozilla/dom/HTMLVideoElement.h"
  17 #include "mozilla/dom/TextTrack.h"
  18 #include "mozilla/dom/TextTrackCue.h"
  19 #include "nsComponentManagerUtils.h"
  20 #include "nsGlobalWindowInner.h"
  21 #include "nsIFrame.h"
  22 #include "nsIWebVTTParserWrapper.h"
  23 #include "nsVariant.h"
  24 #include "nsVideoFrame.h"
  25
  26 mozilla::LazyLogModule gTextTrackLog("WebVTT");
  27
  28 #define WEBVTT_LOG(msg, ...)              \
  29   MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
  30           ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
  31 #define WEBVTT_LOGV(msg, ...)               \
  32   MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
  33           ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
  34
  35 namespace mozilla::dom {
  36
  37 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
  38
  39 void TextTrackManager::ShutdownObserverProxy::Unregister() {
  40   nsContentUtils::UnregisterShutdownObserver(this);
  41   mManager = nullptr;
  42 }
  43
  44 CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) {
  45   mMediaElement = aMediaElement;
  46 }
  47
  48 Maybe<uint32_t> CompareTextTracks::TrackChildPosition(
  49     TextTrack* aTextTrack) const {
  50   MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
  51   HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
  52   if (!trackElement) {
  53     return Nothing();
  54   }
  55   return mMediaElement->ComputeIndexOf(trackElement);
  56 }
  57
  58 bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
  59   // Two tracks can never be equal. If they have corresponding TrackElements
  60   // they would need to occupy the same tree position (impossible) and in the
  61   // case of tracks coming from AddTextTrack source we put the newest at the
  62   // last position, so they won't be equal as well.
  63   return false;
  64 }
  65
  66 bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const {
  67   // Protect against nullptr TextTrack objects; treat them as
  68   // sorting toward the end.
  69   if (!aOne) {
  70     return false;
  71   }
  72   if (!aTwo) {
  73     return true;
  74   }
  75   TextTrackSource sourceOne = aOne->GetTextTrackSource();
  76   TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
  77   if (sourceOne != sourceTwo) {
  78     return sourceOne == TextTrackSource::Track ||
  79            (sourceOne == TextTrackSource::AddTextTrack &&
  80             sourceTwo == TextTrackSource::MediaResourceSpecific);
  81   }
  82   switch (sourceOne) {
  83     case TextTrackSource::Track: {
  84       Maybe<uint32_t> positionOne = TrackChildPosition(aOne);
  85       Maybe<uint32_t> positionTwo = TrackChildPosition(aTwo);
  86       // If either position one or positiontwo are Nothing then something has
  87       // gone wrong. In this case we should just put them at the back of the
  88       // list.
  89       return positionOne.isSome() && positionTwo.isSome() &&
  90              *positionOne < *positionTwo;
  91     }
  92     case TextTrackSource::AddTextTrack:
  93       // For AddTextTrack sources the tracks will already be in the correct
  94       // relative order in the source array. Assume we're called in iteration
  95       // order and can therefore always report aOne < aTwo to maintain the
  96       // original temporal ordering.
  97       return true;
  98     case TextTrackSource::MediaResourceSpecific:
  99       // No rules for Media Resource Specific tracks yet.
 100       break;
 101   }
 102   return true;
 103 }
 104
 105 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
 106                          mPendingTextTracks, mNewCues)
 107
 108 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
 109   NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
 110 NS_INTERFACE_MAP_END
 111
 112 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
 113 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
 114
 115 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
 116
 117 TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
 118     : mMediaElement(aMediaElement),
 119       mHasSeeked(false),
 120       mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
 121       mTimeMarchesOnDispatched(false),
 122       mUpdateCueDisplayDispatched(false),
 123       performedTrackSelection(false),
 124       mShutdown(false) {
 125   nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
 126
 127   NS_ENSURE_TRUE_VOID(parentObject);
 128   WEBVTT_LOG("Create TextTrackManager");
 129   nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
 130   mNewCues = new TextTrackCueList(window);
 131   mTextTracks = new TextTrackList(window, this);
 132   mPendingTextTracks = new TextTrackList(window, this);
 133
 134   if (!sParserWrapper) {
 135     nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
 136         do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
 137     MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
 138     sParserWrapper = parserWrapper;
 139     ClearOnShutdown(&sParserWrapper);
 140   }
 141   mShutdownProxy = new ShutdownObserverProxy(this);
 142 }
 143
 144 TextTrackManager::~TextTrackManager() {
 145   WEBVTT_LOG("~TextTrackManager");
 146   mShutdownProxy->Unregister();
 147 }
 148
 149 TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; }
 150
 151 already_AddRefed<TextTrack> TextTrackManager::AddTextTrack(
 152     TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage,
 153     TextTrackMode aMode, TextTrackReadyState aReadyState,
 154     TextTrackSource aTextTrackSource) {
 155   if (!mMediaElement || !mTextTracks) {
 156     return nullptr;
 157   }
 158   RefPtr<TextTrack> track = mTextTracks->AddTextTrack(
 159       aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource,
 160       CompareTextTracks(mMediaElement));
 161   WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s",
 162              track.get(), static_cast<uint32_t>(aKind),
 163              NS_ConvertUTF16toUTF8(aLabel).get(),
 164              NS_ConvertUTF16toUTF8(aLanguage).get());
 165   AddCues(track);
 166
 167   if (aTextTrackSource == TextTrackSource::Track) {
 168     RefPtr<nsIRunnable> task = NewRunnableMethod(
 169         "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
 170         &TextTrackManager::HonorUserPreferencesForTrackSelection);
 171     NS_DispatchToMainThread(task.forget());
 172   }
 173
 174   return track.forget();
 175 }
 176
 177 void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) {
 178   if (!mMediaElement || !mTextTracks) {
 179     return;
 180   }
 181   WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack);
 182   mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
 183   AddCues(aTextTrack);
 184
 185   if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
 186     RefPtr<nsIRunnable> task = NewRunnableMethod(
 187         "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
 188         &TextTrackManager::HonorUserPreferencesForTrackSelection);
 189     NS_DispatchToMainThread(task.forget());
 190   }
 191 }
 192
 193 void TextTrackManager::AddCues(TextTrack* aTextTrack) {
 194   if (!mNewCues) {
 195     WEBVTT_LOG("AddCues mNewCues is null");
 196     return;
 197   }
 198
 199   TextTrackCueList* cueList = aTextTrack->GetCues();
 200   if (cueList) {
 201     bool dummy;
 202     WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length());
 203     for (uint32_t i = 0; i < cueList->Length(); ++i) {
 204       mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
 205     }
 206     MaybeRunTimeMarchesOn();
 207   }
 208 }
 209
 210 void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack,
 211                                        bool aPendingListOnly) {
 212   if (!mPendingTextTracks || !mTextTracks) {
 213     return;
 214   }
 215
 216   WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack);
 217   mPendingTextTracks->RemoveTextTrack(aTextTrack);
 218   if (aPendingListOnly) {
 219     return;
 220   }
 221
 222   mTextTracks->RemoveTextTrack(aTextTrack);
 223   // Remove the cues in mNewCues belong to aTextTrack.
 224   TextTrackCueList* removeCueList = aTextTrack->GetCues();
 225   if (removeCueList) {
 226     WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length());
 227     for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
 228       mNewCues->RemoveCue(*((*removeCueList)[i]));
 229     }
 230     MaybeRunTimeMarchesOn();
 231   }
 232 }
 233
 234 void TextTrackManager::DidSeek() {
 235   WEBVTT_LOG("DidSeek");
 236   mHasSeeked = true;
 237 }
 238
 239 void TextTrackManager::UpdateCueDisplay() {
 240   WEBVTT_LOG("UpdateCueDisplay");
 241   mUpdateCueDisplayDispatched = false;
 242
 243   if (!mMediaElement || !mTextTracks || IsShutdown()) {
 244     WEBVTT_LOG("Abort UpdateCueDisplay.");
 245     return;
 246   }
 247
 248   nsIFrame* frame = mMediaElement->GetPrimaryFrame();
 249   nsVideoFrame* videoFrame = do_QueryFrame(frame);
 250   if (!videoFrame) {
 251     WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
 252     return;
 253   }
 254
 255   nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
 256   if (!overlay) {
 257     WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
 258     return;
 259   }
 260
 261   RefPtr<nsPIDOMWindowInner> window =
 262       mMediaElement->OwnerDoc()->GetInnerWindow();
 263   if (!window) {
 264     WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
 265   }
 266
 267   nsTArray<RefPtr<TextTrackCue>> showingCues;
 268   mTextTracks->GetShowingCues(showingCues);
 269
 270   WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
 271              showingCues.Length());
 272   RefPtr<nsVariantCC> jsCues = new nsVariantCC();
 273   jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget),
 274                      showingCues.Length(),
 275                      static_cast<void*>(showingCues.Elements()));
 276   nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
 277
 278   nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
 279       "TextTrackManager::UpdateCueDisplay",
 280       [window, jsCues, overlay, controls]() {
 281         if (sParserWrapper) {
 282           sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
 283         }
 284       }));
 285 }
 286
 287 void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) {
 288   WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue);
 289   if (mNewCues) {
 290     mNewCues->AddCue(aCue);
 291   }
 292   MaybeRunTimeMarchesOn();
 293 }
 294
 295 void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) {
 296   WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue);
 297   if (mNewCues) {
 298     mNewCues->RemoveCue(aCue);
 299   }
 300   MaybeRunTimeMarchesOn();
 301   DispatchUpdateCueDisplay();
 302 }
 303
 304 void TextTrackManager::PopulatePendingList() {
 305   if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
 306     return;
 307   }
 308   uint32_t len = mTextTracks->Length();
 309   bool dummy;
 310   for (uint32_t index = 0; index < len; ++index) {
 311     TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
 312     if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
 313         ttrack->ReadyState() == TextTrackReadyState::Loading) {
 314       mPendingTextTracks->AddTextTrack(ttrack,
 315                                        CompareTextTracks(mMediaElement));
 316     }
 317   }
 318 }
 319
 320 void TextTrackManager::AddListeners() {
 321   if (mMediaElement) {
 322     mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false);
 323     mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false,
 324                                     false);
 325     mMediaElement->AddEventListener(u"seeked"_ns, this, false, false);
 326     mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true);
 327   }
 328 }
 329
 330 void TextTrackManager::HonorUserPreferencesForTrackSelection() {
 331   if (performedTrackSelection || !mTextTracks) {
 332     return;
 333   }
 334   WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
 335   TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles};
 336
 337   // Steps 1 - 3: Perform automatic track selection for different TextTrack
 338   // Kinds.
 339   PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
 340   PerformTrackSelection(TextTrackKind::Descriptions);
 341   PerformTrackSelection(TextTrackKind::Chapters);
 342
 343   // Step 4: Set all TextTracks with a kind of metadata that are disabled
 344   // to hidden.
 345   for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
 346     TextTrack* track = (*mTextTracks)[i];
 347     if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
 348         track->Mode() == TextTrackMode::Disabled) {
 349       track->SetMode(TextTrackMode::Hidden);
 350     }
 351   }
 352
 353   performedTrackSelection = true;
 354 }
 355
 356 bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) {
 357   HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
 358   if (!trackElement) {
 359     return false;
 360   }
 361   return trackElement->Default();
 362 }
 363
 364 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) {
 365   TextTrackKind ttKinds[] = {aTextTrackKind};
 366   PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
 367 }
 368
 369 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
 370                                              uint32_t size) {
 371   nsTArray<TextTrack*> candidates;
 372   GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
 373
 374   // Step 3: If any TextTracks in candidates are showing then abort these steps.
 375   for (uint32_t i = 0; i < candidates.Length(); i++) {
 376     if (candidates[i]->Mode() == TextTrackMode::Showing) {
 377       WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
 378                   static_cast<int>(candidates[i]->Kind()));
 379       return;
 380     }
 381   }
 382
 383   // Step 4: Honor user preferences for track selection, otherwise, set the
 384   // first TextTrack in candidates with a default attribute to showing.
 385   // TODO: Bug 981691 - Honor user preferences for text track selection.
 386   for (uint32_t i = 0; i < candidates.Length(); i++) {
 387     if (TrackIsDefault(candidates[i]) &&
 388         candidates[i]->Mode() == TextTrackMode::Disabled) {
 389       candidates[i]->SetMode(TextTrackMode::Showing);
 390       WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
 391                   static_cast<int>(candidates[i]->Kind()));
 392       return;
 393     }
 394   }
 395 }
 396
 397 void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
 398                                             uint32_t size,
 399                                             nsTArray<TextTrack*>& aTextTracks) {
 400   for (uint32_t i = 0; i < size; i++) {
 401     GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
 402   }
 403 }
 404
 405 void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
 406                                            nsTArray<TextTrack*>& aTextTracks) {
 407   if (!mTextTracks) {
 408     return;
 409   }
 410   for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
 411     TextTrack* textTrack = (*mTextTracks)[i];
 412     if (textTrack->Kind() == aTextTrackKind) {
 413       aTextTracks.AppendElement(textTrack);
 414     }
 415   }
 416 }
 417
 418 NS_IMETHODIMP
 419 TextTrackManager::HandleEvent(Event* aEvent) {
 420   if (!mTextTracks) {
 421     return NS_OK;
 422   }
 423
 424   nsAutoString type;
 425   aEvent->GetType(type);
 426   WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get());
 427
 428   const bool setDirty = type.EqualsLiteral("seeked") ||
 429                         type.EqualsLiteral("resizecaption") ||
 430                         type.EqualsLiteral("resizevideocontrols");
 431   const bool updateDisplay = type.EqualsLiteral("controlbarchange") ||
 432                              type.EqualsLiteral("resizecaption");
 433
 434   if (setDirty) {
 435     for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
 436       ((*mTextTracks)[i])->SetCuesDirty();
 437     }
 438   }
 439   if (updateDisplay) {
 440     UpdateCueDisplay();
 441   }
 442
 443   return NS_OK;
 444 }
 445
 446 class SimpleTextTrackEvent : public Runnable {
 447  public:
 448   friend class CompareSimpleTextTrackEvents;
 449   SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
 450                        TextTrack* aTrack, TextTrackCue* aCue)
 451       : Runnable("dom::SimpleTextTrackEvent"),
 452         mName(aEventName),
 453         mTime(aTime),
 454         mTrack(aTrack),
 455         mCue(aCue) {}
 456
 457   NS_IMETHOD Run() override {
 458     WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(),
 459                 NS_ConvertUTF16toUTF8(mName).get(), mTime);
 460     mCue->DispatchTrustedEvent(mName);
 461     return NS_OK;
 462   }
 463
 464   void Dispatch() {
 465     if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) {
 466       global->Dispatch(do_AddRef(this));
 467     } else {
 468       NS_DispatchToMainThread(do_AddRef(this));
 469     }
 470   }
 471
 472  private:
 473   nsString mName;
 474   double mTime;
 475   TextTrack* mTrack;
 476   RefPtr<TextTrackCue> mCue;
 477 };
 478
 479 class CompareSimpleTextTrackEvents {
 480  private:
 481   Maybe<uint32_t> TrackChildPosition(SimpleTextTrackEvent* aEvent) const {
 482     if (aEvent->mTrack) {
 483       HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
 484       if (trackElement) {
 485         return mMediaElement->ComputeIndexOf(trackElement);
 486       }
 487     }
 488     return Nothing();
 489   }
 490   HTMLMediaElement* mMediaElement;
 491
 492  public:
 493   explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) {
 494     mMediaElement = aMediaElement;
 495   }
 496
 497   bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
 498     return false;
 499   }
 500
 501   bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
 502     // TimeMarchesOn step 13.1.
 503     if (aOne->mTime < aTwo->mTime) {
 504       return true;
 505     }
 506     if (aOne->mTime > aTwo->mTime) {
 507       return false;
 508     }
 509
 510     // TimeMarchesOn step 13.2 text track cue order.
 511     // TextTrack position in TextTrackList
 512     TextTrack* t1 = aOne->mTrack;
 513     TextTrack* t2 = aTwo->mTrack;
 514     MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
 515     MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
 516     if (t1 != t2) {
 517       TextTrackList* tList = t1->GetTextTrackList();
 518       MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
 519       nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
 520       auto index1 = textTracks.IndexOf(t1);
 521       auto index2 = textTracks.IndexOf(t2);
 522       if (index1 < index2) {
 523         return true;
 524       }
 525       if (index1 > index2) {
 526         return false;
 527       }
 528     }
 529
 530     MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
 531     // c1 and c2 are both belongs to t1.
 532     TextTrackCue* c1 = aOne->mCue;
 533     TextTrackCue* c2 = aTwo->mCue;
 534     if (c1 != c2) {
 535       if (c1->StartTime() < c2->StartTime()) {
 536         return true;
 537       }
 538       if (c1->StartTime() > c2->StartTime()) {
 539         return false;
 540       }
 541       if (c1->EndTime() < c2->EndTime()) {
 542         return true;
 543       }
 544       if (c1->EndTime() > c2->EndTime()) {
 545         return false;
 546       }
 547
 548       TextTrackCueList* cueList = t1->GetCues();
 549       MOZ_ASSERT(cueList);
 550       nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
 551       auto index1 = cues.IndexOf(c1);
 552       auto index2 = cues.IndexOf(c2);
 553       if (index1 < index2) {
 554         return true;
 555       }
 556       if (index1 > index2) {
 557         return false;
 558       }
 559     }
 560
 561     // TimeMarchesOn step 13.3.
 562     if (aOne->mName.EqualsLiteral("enter") ||
 563         aTwo->mName.EqualsLiteral("exit")) {
 564       return true;
 565     }
 566     return false;
 567   }
 568 };
 569
 570 class TextTrackListInternal {
 571  public:
 572   void AddTextTrack(TextTrack* aTextTrack,
 573                     const CompareTextTracks& aCompareTT) {
 574     if (!mTextTracks.Contains(aTextTrack)) {
 575       mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
 576     }
 577   }
 578   uint32_t Length() const { return mTextTracks.Length(); }
 579   TextTrack* operator[](uint32_t aIndex) {
 580     return mTextTracks.SafeElementAt(aIndex, nullptr);
 581   }
 582
 583  private:
 584   nsTArray<RefPtr<TextTrack>> mTextTracks;
 585 };
 586
 587 void TextTrackManager::DispatchUpdateCueDisplay() {
 588   if (!mUpdateCueDisplayDispatched && !IsShutdown()) {
 589     WEBVTT_LOG("DispatchUpdateCueDisplay");
 590     if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) {
 591       nsGlobalWindowInner::Cast(win)->Dispatch(
 592           NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
 593                             &TextTrackManager::UpdateCueDisplay));
 594       mUpdateCueDisplayDispatched = true;
 595     }
 596   }
 597 }
 598
 599 void TextTrackManager::DispatchTimeMarchesOn() {
 600   // Run the algorithm if no previous instance is still running, otherwise
 601   // enqueue the current playback position and whether only that changed
 602   // through its usual monotonic increase during normal playback; current
 603   // executing call upon completion will check queue for further 'work'.
 604   if (!mTimeMarchesOnDispatched && !IsShutdown()) {
 605     WEBVTT_LOG("DispatchTimeMarchesOn");
 606     if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) {
 607       nsGlobalWindowInner::Cast(win)->Dispatch(
 608           NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
 609                             &TextTrackManager::TimeMarchesOn));
 610       mTimeMarchesOnDispatched = true;
 611     }
 612   }
 613 }
 614
 615 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
 616 void TextTrackManager::TimeMarchesOn() {
 617   NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
 618   mTimeMarchesOnDispatched = false;
 619
 620   CycleCollectedJSContext* context = CycleCollectedJSContext::Get();
 621   if (context && context->IsInStableOrMetaStableState()) {
 622     // FireTimeUpdate can be called while at stable state following a
 623     // current position change which triggered a state watcher in MediaDecoder
 624     // (see bug 1443429).
 625     // TimeMarchesOn() will modify JS attributes which is forbidden while in
 626     // stable state. So we dispatch a task to perform such operation later
 627     // instead.
 628     DispatchTimeMarchesOn();
 629     return;
 630   }
 631   WEBVTT_LOG("TimeMarchesOn");
 632
 633   // Early return if we don't have any TextTracks or shutting down.
 634   if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() ||
 635       !mMediaElement) {
 636     return;
 637   }
 638
 639   if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) {
 640     WEBVTT_LOG(
 641         "TimeMarchesOn return because media doesn't contain any data yet");
 642     return;
 643   }
 644
 645   if (mMediaElement->Seeking()) {
 646     WEBVTT_LOG("TimeMarchesOn return during seeking");
 647     return;
 648   }
 649
 650   // Step 1, 2.
 651   nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
 652   if (NS_WARN_IF(!parentObject)) {
 653     return;
 654   }
 655   nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
 656   RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window);
 657   RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window);
 658
 659   // Step 3.
 660   auto currentPlaybackTime =
 661       media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
 662   bool hasNormalPlayback = !mHasSeeked;
 663   mHasSeeked = false;
 664   WEBVTT_LOG(
 665       "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
 666       "hasNormalPlayback %d",
 667       mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
 668       hasNormalPlayback);
 669
 670   // The reason we collect other cues is (1) to change active cues to inactive,
 671   // (2) find missing cues, so we actually no need to process all cues. We just
 672   // need to handle cues which are in the time interval [lastTime:currentTime]
 673   // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
 674   // size of other cues, which can improve execution time.
 675   auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
 676   auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
 677   media::TimeInterval interval(start, end);
 678   WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
 679               end.ToSeconds());
 680   for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
 681     TextTrack* track = (*mTextTracks)[idx];
 682     if (track) {
 683       track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval);
 684     }
 685   }
 686
 687   // Step 4.
 688   RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
 689   if (hasNormalPlayback) {
 690     for (uint32_t i = 0; i < otherCues->Length(); ++i) {
 691       TextTrackCue* cue = (*otherCues)[i];
 692       if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() &&
 693           cue->EndTime() <= currentPlaybackTime.ToSeconds()) {
 694         missedCues->AddCue(*cue);
 695       }
 696     }
 697   }
 698
 699   WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
 700   WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
 701   WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
 702   // Step 5. Empty now.
 703   // TODO: Step 6: fire timeupdate?
 704
 705   // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
 706   // 1. All of the cues in current cues have their active flag set.
 707   // 2. None of the cues in other cues have their active flag set.
 708   // 3. Missed cues is empty.
 709   bool c1 = true;
 710   for (uint32_t i = 0; i < currentCues->Length(); ++i) {
 711     if (!(*currentCues)[i]->GetActive()) {
 712       c1 = false;
 713       break;
 714     }
 715   }
 716   bool c2 = true;
 717   for (uint32_t i = 0; i < otherCues->Length(); ++i) {
 718     if ((*otherCues)[i]->GetActive()) {
 719       c2 = false;
 720       break;
 721     }
 722   }
 723   bool c3 = (missedCues->Length() == 0);
 724   if (c1 && c2 && c3) {
 725     mLastTimeMarchesOnCalled = currentPlaybackTime;
 726     WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
 727                mLastTimeMarchesOnCalled.ToSeconds());
 728     return;
 729   }
 730
 731   // Step 8. Respect PauseOnExit flag if not seek.
 732   if (hasNormalPlayback) {
 733     for (uint32_t i = 0; i < otherCues->Length(); ++i) {
 734       TextTrackCue* cue = (*otherCues)[i];
 735       if (cue && cue->PauseOnExit() && cue->GetActive()) {
 736         WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
 737         mMediaElement->Pause();
 738         break;
 739       }
 740     }
 741     for (uint32_t i = 0; i < missedCues->Length(); ++i) {
 742       TextTrackCue* cue = (*missedCues)[i];
 743       if (cue && cue->PauseOnExit()) {
 744         WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
 745         mMediaElement->Pause();
 746         break;
 747       }
 748     }
 749   }
 750
 751   // Step 15.
 752   // Sort text tracks in the same order as the text tracks appear
 753   // in the media element's list of text tracks, and remove
 754   // duplicates.
 755   TextTrackListInternal affectedTracks;
 756   // Step 13, 14.
 757   nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
 758   // Step 9, 10.
 759   // For each text track cue in missed cues, prepare an event named
 760   // enter for the TextTrackCue object with the cue start time.
 761   for (uint32_t i = 0; i < missedCues->Length(); ++i) {
 762     TextTrackCue* cue = (*missedCues)[i];
 763     if (cue) {
 764       WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
 765                  cue, cue->StartTime(), cue->EndTime());
 766       SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
 767           u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
 768       eventList.InsertElementSorted(
 769           event, CompareSimpleTextTrackEvents(mMediaElement));
 770       affectedTracks.AddTextTrack(cue->GetTrack(),
 771                                   CompareTextTracks(mMediaElement));
 772     }
 773   }
 774
 775   // Step 11, 17.
 776   for (uint32_t i = 0; i < otherCues->Length(); ++i) {
 777     TextTrackCue* cue = (*otherCues)[i];
 778     if (cue->GetActive() || missedCues->IsCueExist(cue)) {
 779       double time =
 780           cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime();
 781       WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue,
 782                  cue->StartTime(), cue->EndTime());
 783       SimpleTextTrackEvent* event =
 784           new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue);
 785       eventList.InsertElementSorted(
 786           event, CompareSimpleTextTrackEvents(mMediaElement));
 787       affectedTracks.AddTextTrack(cue->GetTrack(),
 788                                   CompareTextTracks(mMediaElement));
 789     }
 790     cue->SetActive(false);
 791   }
 792
 793   // Step 12, 17.
 794   for (uint32_t i = 0; i < currentCues->Length(); ++i) {
 795     TextTrackCue* cue = (*currentCues)[i];
 796     if (!cue->GetActive()) {
 797       WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
 798                  cue, cue->StartTime(), cue->EndTime());
 799       SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
 800           u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
 801       eventList.InsertElementSorted(
 802           event, CompareSimpleTextTrackEvents(mMediaElement));
 803       affectedTracks.AddTextTrack(cue->GetTrack(),
 804                                   CompareTextTracks(mMediaElement));
 805     }
 806     cue->SetActive(true);
 807   }
 808
 809   // Fire the eventList
 810   for (uint32_t i = 0; i < eventList.Length(); ++i) {
 811     eventList[i]->Dispatch();
 812   }
 813
 814   // Step 16.
 815   for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
 816     TextTrack* ttrack = affectedTracks[i];
 817     if (ttrack) {
 818       ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns);
 819       HTMLTrackElement* trackElement = ttrack->GetTrackElement();
 820       if (trackElement) {
 821         trackElement->DispatchTrackRunnable(u"cuechange"_ns);
 822       }
 823     }
 824   }
 825
 826   mLastTimeMarchesOnCalled = currentPlaybackTime;
 827
 828   // Step 18.
 829   UpdateCueDisplay();
 830 }
 831
 832 void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
 833   // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
 834   WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
 835   MaybeRunTimeMarchesOn();
 836   // For the case "Texttrack.mode = hidden/showing", if the mode
 837   // changing between showing and hidden, TimeMarchesOn
 838   // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
 839   DispatchUpdateCueDisplay();
 840 }
 841
 842 void TextTrackManager::NotifyReset() {
 843   // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
 844   // This will unset all cues' active flag and update the cue display.
 845   WEBVTT_LOG("NotifyReset");
 846   mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
 847   for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
 848     (*mTextTracks)[idx]->SetCuesInactive();
 849   }
 850   UpdateCueDisplay();
 851 }
 852
 853 bool TextTrackManager::IsLoaded() {
 854   return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
 855 }
 856
 857 bool TextTrackManager::IsShutdown() const {
 858   return (mShutdown || !sParserWrapper);
 859 }
 860
 861 void TextTrackManager::MaybeRunTimeMarchesOn() {
 862   MOZ_ASSERT(mMediaElement);
 863   // According to spec, we should check media element's show poster flag before
 864   // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
 865   // (3) cue's start time changes (4) cues's end time changes
 866   // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
 867   // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
 868   if (mMediaElement->GetShowPosterFlag()) {
 869     return;
 870   }
 871   TimeMarchesOn();
 872 }
 873
 874 }  // namespace mozilla::dom