1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "mozilla/dom/TextTrackManager.h"
8 #include "mozilla/ClearOnShutdown.h"
9 #include "mozilla/CycleCollectedJSContext.h"
10 #include "mozilla/Maybe.h"
11 #include "mozilla/Telemetry.h"
12 #include "mozilla/dom/Document.h"
13 #include "mozilla/dom/Event.h"
14 #include "mozilla/dom/HTMLMediaElement.h"
15 #include "mozilla/dom/HTMLTrackElement.h"
16 #include "mozilla/dom/HTMLVideoElement.h"
17 #include "mozilla/dom/TextTrack.h"
18 #include "mozilla/dom/TextTrackCue.h"
19 #include "nsComponentManagerUtils.h"
20 #include "nsGlobalWindowInner.h"
22 #include "nsIWebVTTParserWrapper.h"
23 #include "nsVariant.h"
24 #include "nsVideoFrame.h"
26 mozilla::LazyLogModule
gTextTrackLog("WebVTT");
28 #define WEBVTT_LOG(msg, ...) \
29 MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
30 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
31 #define WEBVTT_LOGV(msg, ...) \
32 MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
33 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
35 namespace mozilla::dom
{
37 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy
, nsIObserver
);
39 void TextTrackManager::ShutdownObserverProxy::Unregister() {
40 nsContentUtils::UnregisterShutdownObserver(this);
44 CompareTextTracks::CompareTextTracks(HTMLMediaElement
* aMediaElement
) {
45 mMediaElement
= aMediaElement
;
48 Maybe
<uint32_t> CompareTextTracks::TrackChildPosition(
49 TextTrack
* aTextTrack
) const {
50 MOZ_DIAGNOSTIC_ASSERT(aTextTrack
);
51 HTMLTrackElement
* trackElement
= aTextTrack
->GetTrackElement();
55 return mMediaElement
->ComputeIndexOf(trackElement
);
58 bool CompareTextTracks::Equals(TextTrack
* aOne
, TextTrack
* aTwo
) const {
59 // Two tracks can never be equal. If they have corresponding TrackElements
60 // they would need to occupy the same tree position (impossible) and in the
61 // case of tracks coming from AddTextTrack source we put the newest at the
62 // last position, so they won't be equal as well.
66 bool CompareTextTracks::LessThan(TextTrack
* aOne
, TextTrack
* aTwo
) const {
67 // Protect against nullptr TextTrack objects; treat them as
68 // sorting toward the end.
75 TextTrackSource sourceOne
= aOne
->GetTextTrackSource();
76 TextTrackSource sourceTwo
= aTwo
->GetTextTrackSource();
77 if (sourceOne
!= sourceTwo
) {
78 return sourceOne
== TextTrackSource::Track
||
79 (sourceOne
== TextTrackSource::AddTextTrack
&&
80 sourceTwo
== TextTrackSource::MediaResourceSpecific
);
83 case TextTrackSource::Track
: {
84 Maybe
<uint32_t> positionOne
= TrackChildPosition(aOne
);
85 Maybe
<uint32_t> positionTwo
= TrackChildPosition(aTwo
);
86 // If either position one or positiontwo are Nothing then something has
87 // gone wrong. In this case we should just put them at the back of the
89 return positionOne
.isSome() && positionTwo
.isSome() &&
90 *positionOne
< *positionTwo
;
92 case TextTrackSource::AddTextTrack
:
93 // For AddTextTrack sources the tracks will already be in the correct
94 // relative order in the source array. Assume we're called in iteration
95 // order and can therefore always report aOne < aTwo to maintain the
96 // original temporal ordering.
98 case TextTrackSource::MediaResourceSpecific
:
99 // No rules for Media Resource Specific tracks yet.
105 NS_IMPL_CYCLE_COLLECTION(TextTrackManager
, mMediaElement
, mTextTracks
,
106 mPendingTextTracks
, mNewCues
)
108 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager
)
109 NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener
)
112 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager
)
113 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager
)
115 StaticRefPtr
<nsIWebVTTParserWrapper
> TextTrackManager::sParserWrapper
;
117 TextTrackManager::TextTrackManager(HTMLMediaElement
* aMediaElement
)
118 : mMediaElement(aMediaElement
),
120 mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
121 mTimeMarchesOnDispatched(false),
122 mUpdateCueDisplayDispatched(false),
123 performedTrackSelection(false),
125 nsISupports
* parentObject
= mMediaElement
->OwnerDoc()->GetParentObject();
127 NS_ENSURE_TRUE_VOID(parentObject
);
128 WEBVTT_LOG("Create TextTrackManager");
129 nsCOMPtr
<nsPIDOMWindowInner
> window
= do_QueryInterface(parentObject
);
130 mNewCues
= new TextTrackCueList(window
);
131 mTextTracks
= new TextTrackList(window
, this);
132 mPendingTextTracks
= new TextTrackList(window
, this);
134 if (!sParserWrapper
) {
135 nsCOMPtr
<nsIWebVTTParserWrapper
> parserWrapper
=
136 do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID
);
137 MOZ_ASSERT(parserWrapper
, "Can't create nsIWebVTTParserWrapper");
138 sParserWrapper
= parserWrapper
;
139 ClearOnShutdown(&sParserWrapper
);
141 mShutdownProxy
= new ShutdownObserverProxy(this);
144 TextTrackManager::~TextTrackManager() {
145 WEBVTT_LOG("~TextTrackManager");
146 mShutdownProxy
->Unregister();
149 TextTrackList
* TextTrackManager::GetTextTracks() const { return mTextTracks
; }
151 already_AddRefed
<TextTrack
> TextTrackManager::AddTextTrack(
152 TextTrackKind aKind
, const nsAString
& aLabel
, const nsAString
& aLanguage
,
153 TextTrackMode aMode
, TextTrackReadyState aReadyState
,
154 TextTrackSource aTextTrackSource
) {
155 if (!mMediaElement
|| !mTextTracks
) {
158 RefPtr
<TextTrack
> track
= mTextTracks
->AddTextTrack(
159 aKind
, aLabel
, aLanguage
, aMode
, aReadyState
, aTextTrackSource
,
160 CompareTextTracks(mMediaElement
));
161 WEBVTT_LOG("AddTextTrack %p kind %" PRIu32
" Label %s Language %s",
162 track
.get(), static_cast<uint32_t>(aKind
),
163 NS_ConvertUTF16toUTF8(aLabel
).get(),
164 NS_ConvertUTF16toUTF8(aLanguage
).get());
167 if (aTextTrackSource
== TextTrackSource::Track
) {
168 RefPtr
<nsIRunnable
> task
= NewRunnableMethod(
169 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
170 &TextTrackManager::HonorUserPreferencesForTrackSelection
);
171 NS_DispatchToMainThread(task
.forget());
174 return track
.forget();
177 void TextTrackManager::AddTextTrack(TextTrack
* aTextTrack
) {
178 if (!mMediaElement
|| !mTextTracks
) {
181 WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack
);
182 mTextTracks
->AddTextTrack(aTextTrack
, CompareTextTracks(mMediaElement
));
185 if (aTextTrack
->GetTextTrackSource() == TextTrackSource::Track
) {
186 RefPtr
<nsIRunnable
> task
= NewRunnableMethod(
187 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
188 &TextTrackManager::HonorUserPreferencesForTrackSelection
);
189 NS_DispatchToMainThread(task
.forget());
193 void TextTrackManager::AddCues(TextTrack
* aTextTrack
) {
195 WEBVTT_LOG("AddCues mNewCues is null");
199 TextTrackCueList
* cueList
= aTextTrack
->GetCues();
202 WEBVTT_LOGV("AddCues, CuesNum=%d", cueList
->Length());
203 for (uint32_t i
= 0; i
< cueList
->Length(); ++i
) {
204 mNewCues
->AddCue(*cueList
->IndexedGetter(i
, dummy
));
206 MaybeRunTimeMarchesOn();
210 void TextTrackManager::RemoveTextTrack(TextTrack
* aTextTrack
,
211 bool aPendingListOnly
) {
212 if (!mPendingTextTracks
|| !mTextTracks
) {
216 WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack
);
217 mPendingTextTracks
->RemoveTextTrack(aTextTrack
);
218 if (aPendingListOnly
) {
222 mTextTracks
->RemoveTextTrack(aTextTrack
);
223 // Remove the cues in mNewCues belong to aTextTrack.
224 TextTrackCueList
* removeCueList
= aTextTrack
->GetCues();
226 WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList
->Length());
227 for (uint32_t i
= 0; i
< removeCueList
->Length(); ++i
) {
228 mNewCues
->RemoveCue(*((*removeCueList
)[i
]));
230 MaybeRunTimeMarchesOn();
234 void TextTrackManager::DidSeek() {
235 WEBVTT_LOG("DidSeek");
239 void TextTrackManager::UpdateCueDisplay() {
240 WEBVTT_LOG("UpdateCueDisplay");
241 mUpdateCueDisplayDispatched
= false;
243 if (!mMediaElement
|| !mTextTracks
|| IsShutdown()) {
244 WEBVTT_LOG("Abort UpdateCueDisplay.");
248 nsIFrame
* frame
= mMediaElement
->GetPrimaryFrame();
249 nsVideoFrame
* videoFrame
= do_QueryFrame(frame
);
251 WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
255 nsCOMPtr
<nsIContent
> overlay
= videoFrame
->GetCaptionOverlay();
257 WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
261 RefPtr
<nsPIDOMWindowInner
> window
=
262 mMediaElement
->OwnerDoc()->GetInnerWindow();
264 WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
267 nsTArray
<RefPtr
<TextTrackCue
>> showingCues
;
268 mTextTracks
->GetShowingCues(showingCues
);
270 WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
271 showingCues
.Length());
272 RefPtr
<nsVariantCC
> jsCues
= new nsVariantCC();
273 jsCues
->SetAsArray(nsIDataType::VTYPE_INTERFACE
, &NS_GET_IID(EventTarget
),
274 showingCues
.Length(),
275 static_cast<void*>(showingCues
.Elements()));
276 nsCOMPtr
<nsIContent
> controls
= videoFrame
->GetVideoControls();
278 nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
279 "TextTrackManager::UpdateCueDisplay",
280 [window
, jsCues
, overlay
, controls
]() {
281 if (sParserWrapper
) {
282 sParserWrapper
->ProcessCues(window
, jsCues
, overlay
, controls
);
287 void TextTrackManager::NotifyCueAdded(TextTrackCue
& aCue
) {
288 WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue
);
290 mNewCues
->AddCue(aCue
);
292 MaybeRunTimeMarchesOn();
295 void TextTrackManager::NotifyCueRemoved(TextTrackCue
& aCue
) {
296 WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue
);
298 mNewCues
->RemoveCue(aCue
);
300 MaybeRunTimeMarchesOn();
301 DispatchUpdateCueDisplay();
304 void TextTrackManager::PopulatePendingList() {
305 if (!mTextTracks
|| !mPendingTextTracks
|| !mMediaElement
) {
308 uint32_t len
= mTextTracks
->Length();
310 for (uint32_t index
= 0; index
< len
; ++index
) {
311 TextTrack
* ttrack
= mTextTracks
->IndexedGetter(index
, dummy
);
312 if (ttrack
&& ttrack
->Mode() != TextTrackMode::Disabled
&&
313 ttrack
->ReadyState() == TextTrackReadyState::Loading
) {
314 mPendingTextTracks
->AddTextTrack(ttrack
,
315 CompareTextTracks(mMediaElement
));
320 void TextTrackManager::AddListeners() {
322 mMediaElement
->AddEventListener(u
"resizecaption"_ns
, this, false, false);
323 mMediaElement
->AddEventListener(u
"resizevideocontrols"_ns
, this, false,
325 mMediaElement
->AddEventListener(u
"seeked"_ns
, this, false, false);
326 mMediaElement
->AddEventListener(u
"controlbarchange"_ns
, this, false, true);
330 void TextTrackManager::HonorUserPreferencesForTrackSelection() {
331 if (performedTrackSelection
|| !mTextTracks
) {
334 WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
335 TextTrackKind ttKinds
[] = {TextTrackKind::Captions
, TextTrackKind::Subtitles
};
337 // Steps 1 - 3: Perform automatic track selection for different TextTrack
339 PerformTrackSelection(ttKinds
, ArrayLength(ttKinds
));
340 PerformTrackSelection(TextTrackKind::Descriptions
);
341 PerformTrackSelection(TextTrackKind::Chapters
);
343 // Step 4: Set all TextTracks with a kind of metadata that are disabled
345 for (uint32_t i
= 0; i
< mTextTracks
->Length(); i
++) {
346 TextTrack
* track
= (*mTextTracks
)[i
];
347 if (track
->Kind() == TextTrackKind::Metadata
&& TrackIsDefault(track
) &&
348 track
->Mode() == TextTrackMode::Disabled
) {
349 track
->SetMode(TextTrackMode::Hidden
);
353 performedTrackSelection
= true;
356 bool TextTrackManager::TrackIsDefault(TextTrack
* aTextTrack
) {
357 HTMLTrackElement
* trackElement
= aTextTrack
->GetTrackElement();
361 return trackElement
->Default();
364 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind
) {
365 TextTrackKind ttKinds
[] = {aTextTrackKind
};
366 PerformTrackSelection(ttKinds
, ArrayLength(ttKinds
));
369 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds
[],
371 nsTArray
<TextTrack
*> candidates
;
372 GetTextTracksOfKinds(aTextTrackKinds
, size
, candidates
);
374 // Step 3: If any TextTracks in candidates are showing then abort these steps.
375 for (uint32_t i
= 0; i
< candidates
.Length(); i
++) {
376 if (candidates
[i
]->Mode() == TextTrackMode::Showing
) {
377 WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
378 static_cast<int>(candidates
[i
]->Kind()));
383 // Step 4: Honor user preferences for track selection, otherwise, set the
384 // first TextTrack in candidates with a default attribute to showing.
385 // TODO: Bug 981691 - Honor user preferences for text track selection.
386 for (uint32_t i
= 0; i
< candidates
.Length(); i
++) {
387 if (TrackIsDefault(candidates
[i
]) &&
388 candidates
[i
]->Mode() == TextTrackMode::Disabled
) {
389 candidates
[i
]->SetMode(TextTrackMode::Showing
);
390 WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
391 static_cast<int>(candidates
[i
]->Kind()));
397 void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds
[],
399 nsTArray
<TextTrack
*>& aTextTracks
) {
400 for (uint32_t i
= 0; i
< size
; i
++) {
401 GetTextTracksOfKind(aTextTrackKinds
[i
], aTextTracks
);
405 void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind
,
406 nsTArray
<TextTrack
*>& aTextTracks
) {
410 for (uint32_t i
= 0; i
< mTextTracks
->Length(); i
++) {
411 TextTrack
* textTrack
= (*mTextTracks
)[i
];
412 if (textTrack
->Kind() == aTextTrackKind
) {
413 aTextTracks
.AppendElement(textTrack
);
419 TextTrackManager::HandleEvent(Event
* aEvent
) {
425 aEvent
->GetType(type
);
426 WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type
).get());
428 const bool setDirty
= type
.EqualsLiteral("seeked") ||
429 type
.EqualsLiteral("resizecaption") ||
430 type
.EqualsLiteral("resizevideocontrols");
431 const bool updateDisplay
= type
.EqualsLiteral("controlbarchange") ||
432 type
.EqualsLiteral("resizecaption");
435 for (uint32_t i
= 0; i
< mTextTracks
->Length(); i
++) {
436 ((*mTextTracks
)[i
])->SetCuesDirty();
446 class SimpleTextTrackEvent
: public Runnable
{
448 friend class CompareSimpleTextTrackEvents
;
449 SimpleTextTrackEvent(const nsAString
& aEventName
, double aTime
,
450 TextTrack
* aTrack
, TextTrackCue
* aCue
)
451 : Runnable("dom::SimpleTextTrackEvent"),
457 NS_IMETHOD
Run() override
{
458 WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue
.get(),
459 NS_ConvertUTF16toUTF8(mName
).get(), mTime
);
460 mCue
->DispatchTrustedEvent(mName
);
465 if (nsCOMPtr
<nsIGlobalObject
> global
= mCue
->GetOwnerGlobal()) {
466 global
->Dispatch(do_AddRef(this));
468 NS_DispatchToMainThread(do_AddRef(this));
476 RefPtr
<TextTrackCue
> mCue
;
479 class CompareSimpleTextTrackEvents
{
481 Maybe
<uint32_t> TrackChildPosition(SimpleTextTrackEvent
* aEvent
) const {
482 if (aEvent
->mTrack
) {
483 HTMLTrackElement
* trackElement
= aEvent
->mTrack
->GetTrackElement();
485 return mMediaElement
->ComputeIndexOf(trackElement
);
490 HTMLMediaElement
* mMediaElement
;
493 explicit CompareSimpleTextTrackEvents(HTMLMediaElement
* aMediaElement
) {
494 mMediaElement
= aMediaElement
;
497 bool Equals(SimpleTextTrackEvent
* aOne
, SimpleTextTrackEvent
* aTwo
) const {
501 bool LessThan(SimpleTextTrackEvent
* aOne
, SimpleTextTrackEvent
* aTwo
) const {
502 // TimeMarchesOn step 13.1.
503 if (aOne
->mTime
< aTwo
->mTime
) {
506 if (aOne
->mTime
> aTwo
->mTime
) {
510 // TimeMarchesOn step 13.2 text track cue order.
511 // TextTrack position in TextTrackList
512 TextTrack
* t1
= aOne
->mTrack
;
513 TextTrack
* t2
= aTwo
->mTrack
;
514 MOZ_ASSERT(t1
, "CompareSimpleTextTrackEvents t1 is null");
515 MOZ_ASSERT(t2
, "CompareSimpleTextTrackEvents t2 is null");
517 TextTrackList
* tList
= t1
->GetTextTrackList();
518 MOZ_ASSERT(tList
, "CompareSimpleTextTrackEvents tList is null");
519 nsTArray
<RefPtr
<TextTrack
>>& textTracks
= tList
->GetTextTrackArray();
520 auto index1
= textTracks
.IndexOf(t1
);
521 auto index2
= textTracks
.IndexOf(t2
);
522 if (index1
< index2
) {
525 if (index1
> index2
) {
530 MOZ_ASSERT(t1
== t2
, "CompareSimpleTextTrackEvents t1 != t2");
531 // c1 and c2 are both belongs to t1.
532 TextTrackCue
* c1
= aOne
->mCue
;
533 TextTrackCue
* c2
= aTwo
->mCue
;
535 if (c1
->StartTime() < c2
->StartTime()) {
538 if (c1
->StartTime() > c2
->StartTime()) {
541 if (c1
->EndTime() < c2
->EndTime()) {
544 if (c1
->EndTime() > c2
->EndTime()) {
548 TextTrackCueList
* cueList
= t1
->GetCues();
550 nsTArray
<RefPtr
<TextTrackCue
>>& cues
= cueList
->GetCuesArray();
551 auto index1
= cues
.IndexOf(c1
);
552 auto index2
= cues
.IndexOf(c2
);
553 if (index1
< index2
) {
556 if (index1
> index2
) {
561 // TimeMarchesOn step 13.3.
562 if (aOne
->mName
.EqualsLiteral("enter") ||
563 aTwo
->mName
.EqualsLiteral("exit")) {
570 class TextTrackListInternal
{
572 void AddTextTrack(TextTrack
* aTextTrack
,
573 const CompareTextTracks
& aCompareTT
) {
574 if (!mTextTracks
.Contains(aTextTrack
)) {
575 mTextTracks
.InsertElementSorted(aTextTrack
, aCompareTT
);
578 uint32_t Length() const { return mTextTracks
.Length(); }
579 TextTrack
* operator[](uint32_t aIndex
) {
580 return mTextTracks
.SafeElementAt(aIndex
, nullptr);
584 nsTArray
<RefPtr
<TextTrack
>> mTextTracks
;
587 void TextTrackManager::DispatchUpdateCueDisplay() {
588 if (!mUpdateCueDisplayDispatched
&& !IsShutdown()) {
589 WEBVTT_LOG("DispatchUpdateCueDisplay");
590 if (nsPIDOMWindowInner
* win
= mMediaElement
->OwnerDoc()->GetInnerWindow()) {
591 nsGlobalWindowInner::Cast(win
)->Dispatch(
592 NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
593 &TextTrackManager::UpdateCueDisplay
));
594 mUpdateCueDisplayDispatched
= true;
599 void TextTrackManager::DispatchTimeMarchesOn() {
600 // Run the algorithm if no previous instance is still running, otherwise
601 // enqueue the current playback position and whether only that changed
602 // through its usual monotonic increase during normal playback; current
603 // executing call upon completion will check queue for further 'work'.
604 if (!mTimeMarchesOnDispatched
&& !IsShutdown()) {
605 WEBVTT_LOG("DispatchTimeMarchesOn");
606 if (nsPIDOMWindowInner
* win
= mMediaElement
->OwnerDoc()->GetInnerWindow()) {
607 nsGlobalWindowInner::Cast(win
)->Dispatch(
608 NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
609 &TextTrackManager::TimeMarchesOn
));
610 mTimeMarchesOnDispatched
= true;
615 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
616 void TextTrackManager::TimeMarchesOn() {
617 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
618 mTimeMarchesOnDispatched
= false;
620 CycleCollectedJSContext
* context
= CycleCollectedJSContext::Get();
621 if (context
&& context
->IsInStableOrMetaStableState()) {
622 // FireTimeUpdate can be called while at stable state following a
623 // current position change which triggered a state watcher in MediaDecoder
624 // (see bug 1443429).
625 // TimeMarchesOn() will modify JS attributes which is forbidden while in
626 // stable state. So we dispatch a task to perform such operation later
628 DispatchTimeMarchesOn();
631 WEBVTT_LOG("TimeMarchesOn");
633 // Early return if we don't have any TextTracks or shutting down.
634 if (!mTextTracks
|| mTextTracks
->Length() == 0 || IsShutdown() ||
639 if (mMediaElement
->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING
) {
641 "TimeMarchesOn return because media doesn't contain any data yet");
645 if (mMediaElement
->Seeking()) {
646 WEBVTT_LOG("TimeMarchesOn return during seeking");
651 nsISupports
* parentObject
= mMediaElement
->OwnerDoc()->GetParentObject();
652 if (NS_WARN_IF(!parentObject
)) {
655 nsCOMPtr
<nsPIDOMWindowInner
> window
= do_QueryInterface(parentObject
);
656 RefPtr
<TextTrackCueList
> currentCues
= new TextTrackCueList(window
);
657 RefPtr
<TextTrackCueList
> otherCues
= new TextTrackCueList(window
);
660 auto currentPlaybackTime
=
661 media::TimeUnit::FromSeconds(mMediaElement
->CurrentTime());
662 bool hasNormalPlayback
= !mHasSeeked
;
665 "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
666 "hasNormalPlayback %d",
667 mLastTimeMarchesOnCalled
.ToSeconds(), currentPlaybackTime
.ToSeconds(),
670 // The reason we collect other cues is (1) to change active cues to inactive,
671 // (2) find missing cues, so we actually no need to process all cues. We just
672 // need to handle cues which are in the time interval [lastTime:currentTime]
673 // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
674 // size of other cues, which can improve execution time.
675 auto start
= std::min(mLastTimeMarchesOnCalled
, currentPlaybackTime
);
676 auto end
= std::max(mLastTimeMarchesOnCalled
, currentPlaybackTime
);
677 media::TimeInterval
interval(start
, end
);
678 WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start
.ToSeconds(),
680 for (uint32_t idx
= 0; idx
< mTextTracks
->Length(); ++idx
) {
681 TextTrack
* track
= (*mTextTracks
)[idx
];
683 track
->GetCurrentCuesAndOtherCues(currentCues
, otherCues
, interval
);
688 RefPtr
<TextTrackCueList
> missedCues
= new TextTrackCueList(window
);
689 if (hasNormalPlayback
) {
690 for (uint32_t i
= 0; i
< otherCues
->Length(); ++i
) {
691 TextTrackCue
* cue
= (*otherCues
)[i
];
692 if (cue
->StartTime() >= mLastTimeMarchesOnCalled
.ToSeconds() &&
693 cue
->EndTime() <= currentPlaybackTime
.ToSeconds()) {
694 missedCues
->AddCue(*cue
);
699 WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues
->Length());
700 WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues
->Length());
701 WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues
->Length());
702 // Step 5. Empty now.
703 // TODO: Step 6: fire timeupdate?
705 // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
706 // 1. All of the cues in current cues have their active flag set.
707 // 2. None of the cues in other cues have their active flag set.
708 // 3. Missed cues is empty.
710 for (uint32_t i
= 0; i
< currentCues
->Length(); ++i
) {
711 if (!(*currentCues
)[i
]->GetActive()) {
717 for (uint32_t i
= 0; i
< otherCues
->Length(); ++i
) {
718 if ((*otherCues
)[i
]->GetActive()) {
723 bool c3
= (missedCues
->Length() == 0);
724 if (c1
&& c2
&& c3
) {
725 mLastTimeMarchesOnCalled
= currentPlaybackTime
;
726 WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
727 mLastTimeMarchesOnCalled
.ToSeconds());
731 // Step 8. Respect PauseOnExit flag if not seek.
732 if (hasNormalPlayback
) {
733 for (uint32_t i
= 0; i
< otherCues
->Length(); ++i
) {
734 TextTrackCue
* cue
= (*otherCues
)[i
];
735 if (cue
&& cue
->PauseOnExit() && cue
->GetActive()) {
736 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
737 mMediaElement
->Pause();
741 for (uint32_t i
= 0; i
< missedCues
->Length(); ++i
) {
742 TextTrackCue
* cue
= (*missedCues
)[i
];
743 if (cue
&& cue
->PauseOnExit()) {
744 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
745 mMediaElement
->Pause();
752 // Sort text tracks in the same order as the text tracks appear
753 // in the media element's list of text tracks, and remove
755 TextTrackListInternal affectedTracks
;
757 nsTArray
<RefPtr
<SimpleTextTrackEvent
>> eventList
;
759 // For each text track cue in missed cues, prepare an event named
760 // enter for the TextTrackCue object with the cue start time.
761 for (uint32_t i
= 0; i
< missedCues
->Length(); ++i
) {
762 TextTrackCue
* cue
= (*missedCues
)[i
];
764 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
765 cue
, cue
->StartTime(), cue
->EndTime());
766 SimpleTextTrackEvent
* event
= new SimpleTextTrackEvent(
767 u
"enter"_ns
, cue
->StartTime(), cue
->GetTrack(), cue
);
768 eventList
.InsertElementSorted(
769 event
, CompareSimpleTextTrackEvents(mMediaElement
));
770 affectedTracks
.AddTextTrack(cue
->GetTrack(),
771 CompareTextTracks(mMediaElement
));
776 for (uint32_t i
= 0; i
< otherCues
->Length(); ++i
) {
777 TextTrackCue
* cue
= (*otherCues
)[i
];
778 if (cue
->GetActive() || missedCues
->IsCueExist(cue
)) {
780 cue
->StartTime() > cue
->EndTime() ? cue
->StartTime() : cue
->EndTime();
781 WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue
,
782 cue
->StartTime(), cue
->EndTime());
783 SimpleTextTrackEvent
* event
=
784 new SimpleTextTrackEvent(u
"exit"_ns
, time
, cue
->GetTrack(), cue
);
785 eventList
.InsertElementSorted(
786 event
, CompareSimpleTextTrackEvents(mMediaElement
));
787 affectedTracks
.AddTextTrack(cue
->GetTrack(),
788 CompareTextTracks(mMediaElement
));
790 cue
->SetActive(false);
794 for (uint32_t i
= 0; i
< currentCues
->Length(); ++i
) {
795 TextTrackCue
* cue
= (*currentCues
)[i
];
796 if (!cue
->GetActive()) {
797 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
798 cue
, cue
->StartTime(), cue
->EndTime());
799 SimpleTextTrackEvent
* event
= new SimpleTextTrackEvent(
800 u
"enter"_ns
, cue
->StartTime(), cue
->GetTrack(), cue
);
801 eventList
.InsertElementSorted(
802 event
, CompareSimpleTextTrackEvents(mMediaElement
));
803 affectedTracks
.AddTextTrack(cue
->GetTrack(),
804 CompareTextTracks(mMediaElement
));
806 cue
->SetActive(true);
809 // Fire the eventList
810 for (uint32_t i
= 0; i
< eventList
.Length(); ++i
) {
811 eventList
[i
]->Dispatch();
815 for (uint32_t i
= 0; i
< affectedTracks
.Length(); ++i
) {
816 TextTrack
* ttrack
= affectedTracks
[i
];
818 ttrack
->DispatchAsyncTrustedEvent(u
"cuechange"_ns
);
819 HTMLTrackElement
* trackElement
= ttrack
->GetTrackElement();
821 trackElement
->DispatchTrackRunnable(u
"cuechange"_ns
);
826 mLastTimeMarchesOnCalled
= currentPlaybackTime
;
832 void TextTrackManager::NotifyCueUpdated(TextTrackCue
* aCue
) {
833 // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
834 WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue
);
835 MaybeRunTimeMarchesOn();
836 // For the case "Texttrack.mode = hidden/showing", if the mode
837 // changing between showing and hidden, TimeMarchesOn
838 // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
839 DispatchUpdateCueDisplay();
842 void TextTrackManager::NotifyReset() {
843 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
844 // This will unset all cues' active flag and update the cue display.
845 WEBVTT_LOG("NotifyReset");
846 mLastTimeMarchesOnCalled
= media::TimeUnit::Zero();
847 for (uint32_t idx
= 0; idx
< mTextTracks
->Length(); ++idx
) {
848 (*mTextTracks
)[idx
]->SetCuesInactive();
853 bool TextTrackManager::IsLoaded() {
854 return mTextTracks
? mTextTracks
->AreTextTracksLoaded() : true;
857 bool TextTrackManager::IsShutdown() const {
858 return (mShutdown
|| !sParserWrapper
);
861 void TextTrackManager::MaybeRunTimeMarchesOn() {
862 MOZ_ASSERT(mMediaElement
);
863 // According to spec, we should check media element's show poster flag before
864 // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
865 // (3) cue's start time changes (4) cues's end time changes
866 // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
867 // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
868 if (mMediaElement
->GetShowPosterFlag()) {
874 } // namespace mozilla::dom