Bug 1880216 - Migrate Fenix docs into Sphinx. r=owlish,geckoview-reviewers,android...
[gecko.git] / dom / html / HTMLMediaElement.cpp
blob4ea52ccdf653b59f27e2fdf78b2a7f6b24ef7114
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifdef XP_WIN
8 # include "objbase.h"
9 #endif
11 #include "mozilla/dom/HTMLMediaElement.h"
13 #include <unordered_map>
15 #include "AudioDeviceInfo.h"
16 #include "AudioStreamTrack.h"
17 #include "AutoplayPolicy.h"
18 #include "ChannelMediaDecoder.h"
19 #include "CrossGraphPort.h"
20 #include "DOMMediaStream.h"
21 #include "DecoderDoctorDiagnostics.h"
22 #include "DecoderDoctorLogger.h"
23 #include "DecoderTraits.h"
24 #include "FrameStatistics.h"
25 #include "GMPCrashHelper.h"
26 #include "GVAutoplayPermissionRequest.h"
27 #ifdef MOZ_ANDROID_HLS_SUPPORT
28 # include "HLSDecoder.h"
29 #endif
30 #include "HTMLMediaElement.h"
31 #include "ImageContainer.h"
32 #include "MP4Decoder.h"
33 #include "MediaContainerType.h"
34 #include "MediaError.h"
35 #include "MediaManager.h"
36 #include "MediaMetadataManager.h"
37 #include "MediaResource.h"
38 #include "MediaShutdownManager.h"
39 #include "MediaSourceDecoder.h"
40 #include "MediaStreamError.h"
41 #include "MediaTrackGraphImpl.h"
42 #include "MediaTrackListener.h"
43 #include "MediaStreamWindowCapturer.h"
44 #include "MediaTrack.h"
45 #include "MediaTrackList.h"
46 #include "Navigator.h"
47 #include "TimeRanges.h"
48 #include "VideoFrameContainer.h"
49 #include "VideoOutput.h"
50 #include "VideoStreamTrack.h"
51 #include "base/basictypes.h"
52 #include "jsapi.h"
53 #include "js/PropertyAndElement.h" // JS_DefineProperty
54 #include "mozilla/AppShutdown.h"
55 #include "mozilla/ArrayUtils.h"
56 #include "mozilla/AsyncEventDispatcher.h"
57 #include "mozilla/EMEUtils.h"
58 #include "mozilla/EventDispatcher.h"
59 #include "mozilla/FloatingPoint.h"
60 #include "mozilla/MathAlgorithms.h"
61 #include "mozilla/NotNull.h"
62 #include "mozilla/Preferences.h"
63 #include "mozilla/PresShell.h"
64 #include "mozilla/ScopeExit.h"
65 #include "mozilla/SchedulerGroup.h"
66 #include "mozilla/Sprintf.h"
67 #include "mozilla/StaticPrefs_media.h"
68 #include "mozilla/SVGObserverUtils.h"
69 #include "mozilla/Telemetry.h"
70 #include "mozilla/dom/AudioTrack.h"
71 #include "mozilla/dom/AudioTrackList.h"
72 #include "mozilla/dom/BlobURLProtocolHandler.h"
73 #include "mozilla/dom/ContentMediaController.h"
74 #include "mozilla/dom/ElementInlines.h"
75 #include "mozilla/dom/FeaturePolicyUtils.h"
76 #include "mozilla/dom/HTMLAudioElement.h"
77 #include "mozilla/dom/HTMLInputElement.h"
78 #include "mozilla/dom/HTMLMediaElementBinding.h"
79 #include "mozilla/dom/HTMLSourceElement.h"
80 #include "mozilla/dom/HTMLVideoElement.h"
81 #include "mozilla/dom/MediaControlUtils.h"
82 #include "mozilla/dom/MediaDevices.h"
83 #include "mozilla/dom/MediaEncryptedEvent.h"
84 #include "mozilla/dom/MediaErrorBinding.h"
85 #include "mozilla/dom/MediaSource.h"
86 #include "mozilla/dom/PlayPromise.h"
87 #include "mozilla/dom/Promise.h"
88 #include "mozilla/dom/TextTrack.h"
89 #include "mozilla/dom/UserActivation.h"
90 #include "mozilla/dom/VideoPlaybackQuality.h"
91 #include "mozilla/dom/VideoTrack.h"
92 #include "mozilla/dom/VideoTrackList.h"
93 #include "mozilla/dom/WakeLock.h"
94 #include "mozilla/dom/WindowGlobalChild.h"
95 #include "mozilla/dom/power/PowerManagerService.h"
96 #include "mozilla/net/UrlClassifierFeatureFactory.h"
97 #include "nsAttrValueInlines.h"
98 #include "nsContentPolicyUtils.h"
99 #include "nsContentUtils.h"
100 #include "nsCycleCollectionParticipant.h"
101 #include "nsDisplayList.h"
102 #include "nsDocShell.h"
103 #include "nsError.h"
104 #include "nsGenericHTMLElement.h"
105 #include "nsGkAtoms.h"
106 #include "nsIAsyncVerifyRedirectCallback.h"
107 #include "nsICachingChannel.h"
108 #include "nsIClassOfService.h"
109 #include "nsIContentPolicy.h"
110 #include "nsIDocShell.h"
111 #include "mozilla/dom/Document.h"
112 #include "nsIFrame.h"
113 #include "nsIHttpChannel.h"
114 #include "nsIObserverService.h"
115 #include "nsIRequest.h"
116 #include "nsIScriptError.h"
117 #include "nsISupportsPrimitives.h"
118 #include "nsIThreadRetargetableStreamListener.h"
119 #include "nsITimer.h"
120 #include "nsJSUtils.h"
121 #include "nsLayoutUtils.h"
122 #include "nsMediaFragmentURIParser.h"
123 #include "nsMimeTypes.h"
124 #include "nsNetUtil.h"
125 #include "nsNodeInfoManager.h"
126 #include "nsPresContext.h"
127 #include "nsQueryObject.h"
128 #include "nsRange.h"
129 #include "nsSize.h"
130 #include "nsThreadUtils.h"
131 #include "nsURIHashKey.h"
132 #include "nsURLHelper.h"
133 #include "nsVideoFrame.h"
134 #include "ReferrerInfo.h"
135 #include "TimeUnits.h"
136 #include "xpcpublic.h"
137 #include <algorithm>
138 #include <cmath>
139 #include <limits>
140 #include <type_traits>
142 mozilla::LazyLogModule gMediaElementLog("HTMLMediaElement");
143 mozilla::LazyLogModule gMediaElementEventsLog("HTMLMediaElementEvents");
145 extern mozilla::LazyLogModule gAutoplayPermissionLog;
146 #define AUTOPLAY_LOG(msg, ...) \
147 MOZ_LOG(gAutoplayPermissionLog, LogLevel::Debug, (msg, ##__VA_ARGS__))
149 // avoid redefined macro in unified build
150 #undef MEDIACONTROL_LOG
151 #define MEDIACONTROL_LOG(msg, ...) \
152 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
153 ("HTMLMediaElement=%p, " msg, this, ##__VA_ARGS__))
155 #undef CONTROLLER_TIMER_LOG
156 #define CONTROLLER_TIMER_LOG(element, msg, ...) \
157 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
158 ("HTMLMediaElement=%p, " msg, element, ##__VA_ARGS__))
160 #define LOG(type, msg) MOZ_LOG(gMediaElementLog, type, msg)
161 #define LOG_EVENT(type, msg) MOZ_LOG(gMediaElementEventsLog, type, msg)
163 using namespace mozilla::layers;
164 using mozilla::net::nsMediaFragmentURIParser;
165 using namespace mozilla::dom::HTMLMediaElement_Binding;
167 namespace mozilla::dom {
169 using AudibleState = AudioChannelService::AudibleState;
170 using SinkInfoPromise = MediaDevices::SinkInfoPromise;
172 // Number of milliseconds between progress events as defined by spec
173 static const uint32_t PROGRESS_MS = 350;
175 // Number of milliseconds of no data before a stall event is fired as defined by
176 // spec
177 static const uint32_t STALL_MS = 3000;
179 // Used by AudioChannel for suppresssing the volume to this ratio.
180 #define FADED_VOLUME_RATIO 0.25
182 // These constants are arbitrary
183 // Minimum playbackRate for a media
184 static const double MIN_PLAYBACKRATE = 1.0 / 16;
185 // Maximum playbackRate for a media
186 static const double MAX_PLAYBACKRATE = 16.0;
188 static double ClampPlaybackRate(double aPlaybackRate) {
189 MOZ_ASSERT(aPlaybackRate >= 0.0);
191 if (aPlaybackRate == 0.0) {
192 return aPlaybackRate;
194 if (aPlaybackRate < MIN_PLAYBACKRATE) {
195 return MIN_PLAYBACKRATE;
197 if (aPlaybackRate > MAX_PLAYBACKRATE) {
198 return MAX_PLAYBACKRATE;
200 return aPlaybackRate;
203 // Media error values. These need to match the ones in MediaError.webidl.
204 static const unsigned short MEDIA_ERR_ABORTED = 1;
205 static const unsigned short MEDIA_ERR_NETWORK = 2;
206 static const unsigned short MEDIA_ERR_DECODE = 3;
207 static const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
210 * EventBlocker helps media element to postpone the event delivery by storing
211 * the event runner, and execute them once media element decides not to postpone
212 * the event delivery. If media element never resumes the event delivery, then
213 * those runner would be cancelled.
214 * For example, we postpone the event delivery when media element entering to
215 * the bf-cache.
217 class HTMLMediaElement::EventBlocker final : public nsISupports {
218 public:
219 NS_DECL_CYCLE_COLLECTING_ISUPPORTS_FINAL
220 NS_DECL_CYCLE_COLLECTION_CLASS(EventBlocker)
222 explicit EventBlocker(HTMLMediaElement* aElement) : mElement(aElement) {}
224 void SetBlockEventDelivery(bool aShouldBlock) {
225 MOZ_ASSERT(NS_IsMainThread());
226 if (mShouldBlockEventDelivery == aShouldBlock) {
227 return;
229 LOG_EVENT(LogLevel::Debug,
230 ("%p %s event delivery", mElement.get(),
231 mShouldBlockEventDelivery ? "block" : "unblock"));
232 mShouldBlockEventDelivery = aShouldBlock;
233 if (!mShouldBlockEventDelivery) {
234 DispatchPendingMediaEvents();
238 void PostponeEvent(nsMediaEventRunner* aRunner) {
239 MOZ_ASSERT(NS_IsMainThread());
240 // Element has been CCed, which would break the weak pointer.
241 if (!mElement) {
242 return;
244 MOZ_ASSERT(mShouldBlockEventDelivery);
245 MOZ_ASSERT(mElement);
246 LOG_EVENT(LogLevel::Debug,
247 ("%p postpone runner %s for %s", mElement.get(),
248 NS_ConvertUTF16toUTF8(aRunner->Name()).get(),
249 NS_ConvertUTF16toUTF8(aRunner->EventName()).get()));
250 mPendingEventRunners.AppendElement(aRunner);
253 void Shutdown() {
254 MOZ_ASSERT(NS_IsMainThread());
255 for (auto& runner : mPendingEventRunners) {
256 runner->Cancel();
258 mPendingEventRunners.Clear();
261 bool ShouldBlockEventDelivery() const {
262 MOZ_ASSERT(NS_IsMainThread());
263 return mShouldBlockEventDelivery;
266 size_t SizeOfExcludingThis(MallocSizeOf& aMallocSizeOf) const {
267 MOZ_ASSERT(NS_IsMainThread());
268 size_t total = 0;
269 for (const auto& runner : mPendingEventRunners) {
270 total += aMallocSizeOf(runner);
272 return total;
275 private:
276 ~EventBlocker() = default;
278 void DispatchPendingMediaEvents() {
279 MOZ_ASSERT(mElement);
280 for (auto& runner : mPendingEventRunners) {
281 LOG_EVENT(LogLevel::Debug,
282 ("%p execute runner %s for %s", mElement.get(),
283 NS_ConvertUTF16toUTF8(runner->Name()).get(),
284 NS_ConvertUTF16toUTF8(runner->EventName()).get()));
285 GetMainThreadSerialEventTarget()->Dispatch(runner.forget());
287 mPendingEventRunners.Clear();
290 WeakPtr<HTMLMediaElement> mElement;
291 bool mShouldBlockEventDelivery = false;
292 // Contains event runners which should not be run for now because we want
293 // to block all events delivery. They would be dispatched once media element
294 // decides unblocking them.
295 nsTArray<RefPtr<nsMediaEventRunner>> mPendingEventRunners;
298 NS_IMPL_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker, mPendingEventRunners)
299 NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::EventBlocker)
300 NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::EventBlocker)
301 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker)
302 NS_INTERFACE_MAP_ENTRY(nsISupports)
303 NS_INTERFACE_MAP_END
306 * We use MediaControlKeyListener to listen to media control key in order to
307 * play and pause media element when user press media control keys and update
308 * media's playback and audible state to the media controller.
310 * Use `Start()` to start listening event and use `Stop()` to stop listening
311 * event. In addition, notifying any change to media controller MUST be done
312 * after successfully calling `Start()`.
314 class HTMLMediaElement::MediaControlKeyListener final
315 : public ContentMediaControlKeyReceiver {
316 public:
317 NS_INLINE_DECL_REFCOUNTING(MediaControlKeyListener, override)
319 MOZ_INIT_OUTSIDE_CTOR explicit MediaControlKeyListener(
320 HTMLMediaElement* aElement)
321 : mElement(aElement) {
322 MOZ_ASSERT(NS_IsMainThread());
323 MOZ_ASSERT(aElement);
327 * Start listening to the media control keys which would make media being able
328 * to be controlled via pressing media control keys.
330 void Start() {
331 MOZ_ASSERT(NS_IsMainThread());
332 if (IsStarted()) {
333 // We have already been started, do not notify start twice.
334 return;
337 // Fail to init media agent, we are not able to notify the media controller
338 // any update and also are not able to receive media control key events.
339 if (!InitMediaAgent()) {
340 MEDIACONTROL_LOG("Failed to start due to not able to init media agent!");
341 return;
344 NotifyPlaybackStateChanged(MediaPlaybackState::eStarted);
345 // If owner has started playing before the listener starts, we should update
346 // the playing state as well. Eg. media starts inaudily and becomes audible
347 // later.
348 if (!Owner()->Paused()) {
349 NotifyMediaStartedPlaying();
351 if (StaticPrefs::media_mediacontrol_testingevents_enabled()) {
352 auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
353 Owner(), u"MozStartMediaControl"_ns, CanBubble::eYes,
354 ChromeOnlyDispatch::eYes);
355 dispatcher->PostDOMEvent();
360 * Stop listening to the media control keys which would make media not be able
361 * to be controlled via pressing media control keys. If we haven't started
362 * listening to the media control keys, then nothing would happen.
364 void StopIfNeeded() {
365 MOZ_ASSERT(NS_IsMainThread());
366 if (!IsStarted()) {
367 // We have already been stopped, do not notify stop twice.
368 return;
370 NotifyMediaStoppedPlaying();
371 NotifyPlaybackStateChanged(MediaPlaybackState::eStopped);
373 // Remove ourselves from media agent, which would stop receiving event.
374 mControlAgent->RemoveReceiver(this);
375 mControlAgent = nullptr;
378 bool IsStarted() const { return mState != MediaPlaybackState::eStopped; }
380 bool IsPlaying() const override {
381 return Owner() ? !Owner()->Paused() : false;
385 * Following methods should only be used after starting listener.
387 void NotifyMediaStartedPlaying() {
388 MOZ_ASSERT(NS_IsMainThread());
389 MOZ_ASSERT(IsStarted());
390 if (mState == MediaPlaybackState::eStarted ||
391 mState == MediaPlaybackState::ePaused) {
392 NotifyPlaybackStateChanged(MediaPlaybackState::ePlayed);
393 // If media is `inaudible` in the beginning, then we don't need to notify
394 // the state, because notifying `inaudible` should always come after
395 // notifying `audible`.
396 if (mIsOwnerAudible) {
397 NotifyAudibleStateChanged(MediaAudibleState::eAudible);
402 void NotifyMediaStoppedPlaying() {
403 MOZ_ASSERT(NS_IsMainThread());
404 MOZ_ASSERT(IsStarted());
405 if (mState == MediaPlaybackState::ePlayed) {
406 NotifyPlaybackStateChanged(MediaPlaybackState::ePaused);
407 // As media are going to be paused, so no sound is possible to be heard.
408 if (mIsOwnerAudible) {
409 NotifyAudibleStateChanged(MediaAudibleState::eInaudible);
414 // This method can be called before the listener starts, which would cache
415 // the audible state and update after the listener starts.
416 void UpdateMediaAudibleState(bool aIsOwnerAudible) {
417 MOZ_ASSERT(NS_IsMainThread());
418 if (mIsOwnerAudible == aIsOwnerAudible) {
419 return;
421 mIsOwnerAudible = aIsOwnerAudible;
422 MEDIACONTROL_LOG("Media becomes %s",
423 mIsOwnerAudible ? "audible" : "inaudible");
424 // If media hasn't started playing, it doesn't make sense to update media
425 // audible state. Therefore, in that case we would noitfy the audible state
426 // when media starts playing.
427 if (mState == MediaPlaybackState::ePlayed) {
428 NotifyAudibleStateChanged(mIsOwnerAudible
429 ? MediaAudibleState::eAudible
430 : MediaAudibleState::eInaudible);
434 void SetPictureInPictureModeEnabled(bool aIsEnabled) {
435 MOZ_ASSERT(NS_IsMainThread());
436 if (mIsPictureInPictureEnabled == aIsEnabled) {
437 return;
439 // PIP state changes might happen before the listener starts or stops where
440 // we haven't call `InitMediaAgent()` yet. Eg. Reset the PIP video's src,
441 // then cancel the PIP. In addition, not like playback and audible state
442 // which should be restricted to update via the same agent in order to keep
443 // those states correct in each `ContextMediaInfo`, PIP state can be updated
444 // through any browsing context, so we would use `ContentMediaAgent::Get()`
445 // directly to update PIP state.
446 mIsPictureInPictureEnabled = aIsEnabled;
447 if (RefPtr<IMediaInfoUpdater> updater =
448 ContentMediaAgent::Get(GetCurrentBrowsingContext())) {
449 updater->SetIsInPictureInPictureMode(mOwnerBrowsingContextId,
450 mIsPictureInPictureEnabled);
454 void HandleMediaKey(MediaControlKey aKey) override {
455 MOZ_ASSERT(NS_IsMainThread());
456 MOZ_ASSERT(IsStarted());
457 MEDIACONTROL_LOG("HandleEvent '%s'", ToMediaControlKeyStr(aKey));
458 if (aKey == MediaControlKey::Play) {
459 Owner()->Play();
460 } else if (aKey == MediaControlKey::Pause) {
461 Owner()->Pause();
462 } else {
463 MOZ_ASSERT(aKey == MediaControlKey::Stop,
464 "Not supported key for media element!");
465 Owner()->Pause();
466 StopIfNeeded();
470 void UpdateOwnerBrowsingContextIfNeeded() {
471 // Has not notified any information about the owner context yet.
472 if (!IsStarted()) {
473 return;
476 BrowsingContext* currentBC = GetCurrentBrowsingContext();
477 MOZ_ASSERT(currentBC);
478 // Still in the same browsing context, no need to update.
479 if (currentBC->Id() == mOwnerBrowsingContextId) {
480 return;
482 MEDIACONTROL_LOG("Change browsing context from %" PRIu64 " to %" PRIu64,
483 mOwnerBrowsingContextId, currentBC->Id());
484 // This situation would happen when we start a media in an original browsing
485 // context, then we move it to another browsing context, such as an iframe,
486 // so its owner browsing context would be changed. Therefore, we should
487 // reset the media status for the previous browsing context by calling
488 // `Stop()`, in which the listener would notify `ePaused` (if it's playing)
489 // and `eStop`. Then calls `Start()`, in which the listener would notify
490 // `eStart` to the new browsing context. If the media was playing before,
491 // we would also notify `ePlayed`.
492 bool wasInPlayingState = mState == MediaPlaybackState::ePlayed;
493 StopIfNeeded();
494 Start();
495 if (wasInPlayingState) {
496 NotifyMediaStartedPlaying();
500 private:
501 ~MediaControlKeyListener() = default;
503 // The media can be moved around different browsing contexts, so this context
504 // might be different from the one that we used to initialize
505 // `ContentMediaAgent`.
506 BrowsingContext* GetCurrentBrowsingContext() const {
507 // Owner has been CCed, which would break the link of the weaker pointer.
508 if (!Owner()) {
509 return nullptr;
511 nsPIDOMWindowInner* window = Owner()->OwnerDoc()->GetInnerWindow();
512 return window ? window->GetBrowsingContext() : nullptr;
515 bool InitMediaAgent() {
516 MOZ_ASSERT(NS_IsMainThread());
517 BrowsingContext* currentBC = GetCurrentBrowsingContext();
518 mControlAgent = ContentMediaAgent::Get(currentBC);
519 if (!mControlAgent) {
520 return false;
522 MOZ_ASSERT(currentBC);
523 mOwnerBrowsingContextId = currentBC->Id();
524 MEDIACONTROL_LOG("Init agent in browsing context %" PRIu64,
525 mOwnerBrowsingContextId);
526 mControlAgent->AddReceiver(this);
527 return true;
530 HTMLMediaElement* Owner() const {
531 // `mElement` would be clear during CC unlinked, but it would only happen
532 // after stopping the listener.
533 MOZ_ASSERT(mElement || !IsStarted());
534 return mElement.get();
537 void NotifyPlaybackStateChanged(MediaPlaybackState aState) {
538 MOZ_ASSERT(NS_IsMainThread());
539 MOZ_ASSERT(mControlAgent);
540 MEDIACONTROL_LOG("NotifyMediaState from state='%s' to state='%s'",
541 ToMediaPlaybackStateStr(mState),
542 ToMediaPlaybackStateStr(aState));
543 MOZ_ASSERT(mState != aState, "Should not notify same state again!");
544 mState = aState;
545 mControlAgent->NotifyMediaPlaybackChanged(mOwnerBrowsingContextId, mState);
548 void NotifyAudibleStateChanged(MediaAudibleState aState) {
549 MOZ_ASSERT(NS_IsMainThread());
550 MOZ_ASSERT(IsStarted());
551 mControlAgent->NotifyMediaAudibleChanged(mOwnerBrowsingContextId, aState);
554 MediaPlaybackState mState = MediaPlaybackState::eStopped;
555 WeakPtr<HTMLMediaElement> mElement;
556 RefPtr<ContentMediaAgent> mControlAgent;
557 bool mIsPictureInPictureEnabled = false;
558 bool mIsOwnerAudible = false;
559 MOZ_INIT_OUTSIDE_CTOR uint64_t mOwnerBrowsingContextId;
562 class HTMLMediaElement::MediaStreamTrackListener
563 : public DOMMediaStream::TrackListener {
564 public:
565 explicit MediaStreamTrackListener(HTMLMediaElement* aElement)
566 : mElement(aElement) {}
568 void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) override {
569 if (!mElement) {
570 return;
572 mElement->NotifyMediaStreamTrackAdded(aTrack);
575 void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack) override {
576 if (!mElement) {
577 return;
579 mElement->NotifyMediaStreamTrackRemoved(aTrack);
582 void OnActive() {
583 MOZ_ASSERT(mElement);
585 // mediacapture-main says:
586 // Note that once ended equals true the HTMLVideoElement will not play media
587 // even if new MediaStreamTracks are added to the MediaStream (causing it to
588 // return to the active state) unless autoplay is true or the web
589 // application restarts the element, e.g., by calling play().
591 // This is vague on exactly how to go from becoming active to playing, when
592 // autoplaying. However, per the media element spec, to play an autoplaying
593 // media element, we must load the source and reach readyState
594 // HAVE_ENOUGH_DATA [1]. Hence, a MediaStream being assigned to a media
595 // element and becoming active runs the load algorithm, so that it can
596 // eventually be played.
598 // [1]
599 // https://html.spec.whatwg.org/multipage/media.html#ready-states:event-media-play
601 LOG(LogLevel::Debug, ("%p, mSrcStream %p became active, checking if we "
602 "need to run the load algorithm",
603 mElement.get(), mElement->mSrcStream.get()));
604 if (!mElement->IsPlaybackEnded()) {
605 return;
607 if (!mElement->Autoplay()) {
608 return;
610 LOG(LogLevel::Info, ("%p, mSrcStream %p became active on autoplaying, "
611 "ended element. Reloading.",
612 mElement.get(), mElement->mSrcStream.get()));
613 mElement->DoLoad();
616 void NotifyActive() override {
617 if (!mElement) {
618 return;
621 if (!mElement->IsVideo()) {
622 // Audio elements use NotifyAudible().
623 return;
626 OnActive();
629 void NotifyAudible() override {
630 if (!mElement) {
631 return;
634 if (mElement->IsVideo()) {
635 // Video elements use NotifyActive().
636 return;
639 OnActive();
642 void OnInactive() {
643 MOZ_ASSERT(mElement);
645 if (mElement->IsPlaybackEnded()) {
646 return;
648 LOG(LogLevel::Debug, ("%p, mSrcStream %p became inactive", mElement.get(),
649 mElement->mSrcStream.get()));
651 mElement->PlaybackEnded();
654 void NotifyInactive() override {
655 if (!mElement) {
656 return;
659 if (!mElement->IsVideo()) {
660 // Audio elements use NotifyInaudible().
661 return;
664 OnInactive();
667 void NotifyInaudible() override {
668 if (!mElement) {
669 return;
672 if (mElement->IsVideo()) {
673 // Video elements use NotifyInactive().
674 return;
677 OnInactive();
680 protected:
681 const WeakPtr<HTMLMediaElement> mElement;
685 * Helper class that manages audio and video outputs for all enabled tracks in a
686 * media element. It also manages calculating the current time when playing a
687 * MediaStream.
689 class HTMLMediaElement::MediaStreamRenderer
690 : public DOMMediaStream::TrackListener {
691 public:
692 NS_INLINE_DECL_REFCOUNTING(MediaStreamRenderer)
694 MediaStreamRenderer(AbstractThread* aMainThread,
695 VideoFrameContainer* aVideoContainer,
696 FirstFrameVideoOutput* aFirstFrameVideoOutput,
697 void* aAudioOutputKey)
698 : mVideoContainer(aVideoContainer),
699 mAudioOutputKey(aAudioOutputKey),
700 mWatchManager(this, aMainThread),
701 mFirstFrameVideoOutput(aFirstFrameVideoOutput) {
702 if (mFirstFrameVideoOutput) {
703 mWatchManager.Watch(mFirstFrameVideoOutput->mFirstFrameRendered,
704 &MediaStreamRenderer::SetFirstFrameRendered);
708 void Shutdown() {
709 for (const auto& t : mAudioTracks.Clone()) {
710 if (t) {
711 RemoveTrack(t->AsAudioStreamTrack());
714 if (mVideoTrack) {
715 RemoveTrack(mVideoTrack->AsVideoStreamTrack());
717 mWatchManager.Shutdown();
718 mFirstFrameVideoOutput = nullptr;
721 void UpdateGraphTime() {
722 mGraphTime =
723 mGraphTimeDummy->mTrack->Graph()->CurrentTime() - *mGraphTimeOffset;
726 void SetFirstFrameRendered() {
727 if (!mFirstFrameVideoOutput) {
728 return;
730 if (mVideoTrack) {
731 mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(
732 mFirstFrameVideoOutput);
734 mWatchManager.Unwatch(mFirstFrameVideoOutput->mFirstFrameRendered,
735 &MediaStreamRenderer::SetFirstFrameRendered);
736 mFirstFrameVideoOutput = nullptr;
739 void SetProgressingCurrentTime(bool aProgress) {
740 if (aProgress == mProgressingCurrentTime) {
741 return;
744 MOZ_DIAGNOSTIC_ASSERT(mGraphTimeDummy);
745 mProgressingCurrentTime = aProgress;
746 MediaTrackGraph* graph = mGraphTimeDummy->mTrack->Graph();
747 if (mProgressingCurrentTime) {
748 mGraphTimeOffset = Some(graph->CurrentTime().Ref() - mGraphTime);
749 mWatchManager.Watch(graph->CurrentTime(),
750 &MediaStreamRenderer::UpdateGraphTime);
751 } else {
752 mWatchManager.Unwatch(graph->CurrentTime(),
753 &MediaStreamRenderer::UpdateGraphTime);
757 void Start() {
758 if (mRendering) {
759 return;
762 LOG(LogLevel::Info, ("MediaStreamRenderer=%p Start", this));
763 mRendering = true;
765 if (!mGraphTimeDummy) {
766 return;
769 for (const auto& t : mAudioTracks) {
770 if (t) {
771 t->AsAudioStreamTrack()->AddAudioOutput(mAudioOutputKey,
772 mAudioOutputSink);
773 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
774 mAudioOutputVolume);
778 if (mVideoTrack) {
779 mVideoTrack->AsVideoStreamTrack()->AddVideoOutput(mVideoContainer);
783 void Stop() {
784 if (!mRendering) {
785 return;
788 LOG(LogLevel::Info, ("MediaStreamRenderer=%p Stop", this));
789 mRendering = false;
791 if (!mGraphTimeDummy) {
792 return;
795 for (const auto& t : mAudioTracks) {
796 if (t) {
797 t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
800 // There is no longer an audio output that needs the device so the
801 // device may not start. Ensure the promise is resolved.
802 ResolveAudioDevicePromiseIfExists(__func__);
804 if (mVideoTrack) {
805 mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(mVideoContainer);
809 void SetAudioOutputVolume(float aVolume) {
810 if (mAudioOutputVolume == aVolume) {
811 return;
813 mAudioOutputVolume = aVolume;
814 if (!mRendering) {
815 return;
817 for (const auto& t : mAudioTracks) {
818 if (t) {
819 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
820 mAudioOutputVolume);
825 RefPtr<GenericPromise> SetAudioOutputDevice(AudioDeviceInfo* aSink) {
826 MOZ_ASSERT(aSink);
827 MOZ_ASSERT(mAudioOutputSink != aSink);
828 LOG(LogLevel::Info,
829 ("MediaStreamRenderer=%p SetAudioOutputDevice name=%s\n", this,
830 NS_ConvertUTF16toUTF8(aSink->Name()).get()));
832 mAudioOutputSink = aSink;
834 if (!mRendering) {
835 MOZ_ASSERT(mSetAudioDevicePromise.IsEmpty());
836 return GenericPromise::CreateAndResolve(true, __func__);
839 nsTArray<RefPtr<GenericPromise>> promises;
840 for (const auto& t : mAudioTracks) {
841 t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
842 promises.AppendElement(t->AsAudioStreamTrack()->AddAudioOutput(
843 mAudioOutputKey, mAudioOutputSink));
844 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
845 mAudioOutputVolume);
847 if (!promises.Length()) {
848 // Not active track, save it for later
849 MOZ_ASSERT(mSetAudioDevicePromise.IsEmpty());
850 return GenericPromise::CreateAndResolve(true, __func__);
853 // Resolve any existing promise for a previous device so that promises
854 // resolve in order of setSinkId() invocation.
855 ResolveAudioDevicePromiseIfExists(__func__);
857 RefPtr promise = mSetAudioDevicePromise.Ensure(__func__);
858 GenericPromise::AllSettled(GetCurrentSerialEventTarget(), promises)
859 ->Then(GetMainThreadSerialEventTarget(), __func__,
860 [self = RefPtr{this},
861 this](const GenericPromise::AllSettledPromiseType::
862 ResolveOrRejectValue& aValue) {
863 // This handler should have been disconnected if
864 // mSetAudioDevicePromise has been settled.
865 MOZ_ASSERT(!mSetAudioDevicePromise.IsEmpty());
866 mDeviceStartedRequest.Complete();
867 // The AudioStreamTrack::AddAudioOutput() promise is rejected
868 // either when the graph no longer needs the device, in which
869 // case this handler would have already been disconnected, or
870 // the graph is force shutdown.
871 // mSetAudioDevicePromise is resolved regardless of whether
872 // the AddAudioOutput() promises resolve or reject because
873 // the underlying device has been changed.
874 LOG(LogLevel::Info,
875 ("MediaStreamRenderer=%p SetAudioOutputDevice settled",
876 this));
877 mSetAudioDevicePromise.Resolve(true, __func__);
879 ->Track(mDeviceStartedRequest);
881 return promise;
884 void AddTrack(AudioStreamTrack* aTrack) {
885 MOZ_DIAGNOSTIC_ASSERT(!mAudioTracks.Contains(aTrack));
886 mAudioTracks.AppendElement(aTrack);
887 EnsureGraphTimeDummy();
888 if (mRendering) {
889 aTrack->AddAudioOutput(mAudioOutputKey, mAudioOutputSink);
890 aTrack->SetAudioOutputVolume(mAudioOutputKey, mAudioOutputVolume);
893 void AddTrack(VideoStreamTrack* aTrack) {
894 MOZ_DIAGNOSTIC_ASSERT(!mVideoTrack);
895 if (!mVideoContainer) {
896 return;
898 mVideoTrack = aTrack;
899 EnsureGraphTimeDummy();
900 if (mFirstFrameVideoOutput) {
901 // Add the first frame output even if we are rendering. It will only
902 // accept one frame. If we are rendering, then the main output will
903 // overwrite that with the same frame (and possibly more frames).
904 aTrack->AddVideoOutput(mFirstFrameVideoOutput);
906 if (mRendering) {
907 aTrack->AddVideoOutput(mVideoContainer);
911 void RemoveTrack(AudioStreamTrack* aTrack) {
912 MOZ_DIAGNOSTIC_ASSERT(mAudioTracks.Contains(aTrack));
913 if (mRendering) {
914 aTrack->RemoveAudioOutput(mAudioOutputKey);
916 mAudioTracks.RemoveElement(aTrack);
918 if (mAudioTracks.IsEmpty()) {
919 // There is no longer an audio output that needs the device so the
920 // device may not start. Ensure the promise is resolved.
921 ResolveAudioDevicePromiseIfExists(__func__);
924 void RemoveTrack(VideoStreamTrack* aTrack) {
925 MOZ_DIAGNOSTIC_ASSERT(mVideoTrack == aTrack);
926 if (!mVideoContainer) {
927 return;
929 if (mFirstFrameVideoOutput) {
930 aTrack->RemoveVideoOutput(mFirstFrameVideoOutput);
932 if (mRendering) {
933 aTrack->RemoveVideoOutput(mVideoContainer);
935 mVideoTrack = nullptr;
938 double CurrentTime() const {
939 if (!mGraphTimeDummy) {
940 return 0.0;
943 return mGraphTimeDummy->mTrack->GraphImpl()->MediaTimeToSeconds(mGraphTime);
946 Watchable<GraphTime>& CurrentGraphTime() { return mGraphTime; }
948 // Set if we're rendering video.
949 const RefPtr<VideoFrameContainer> mVideoContainer;
951 // Set if we're rendering audio, nullptr otherwise.
952 void* const mAudioOutputKey;
954 private:
955 ~MediaStreamRenderer() { Shutdown(); }
957 void EnsureGraphTimeDummy() {
958 if (mGraphTimeDummy) {
959 return;
962 MediaTrackGraph* graph = nullptr;
963 for (const auto& t : mAudioTracks) {
964 if (t && !t->Ended()) {
965 graph = t->Graph();
966 break;
970 if (!graph && mVideoTrack && !mVideoTrack->Ended()) {
971 graph = mVideoTrack->Graph();
974 if (!graph) {
975 return;
978 // This dummy keeps `graph` alive and ensures access to it.
979 mGraphTimeDummy = MakeRefPtr<SharedDummyTrack>(
980 graph->CreateSourceTrack(MediaSegment::AUDIO));
983 void ResolveAudioDevicePromiseIfExists(const char* aMethodName) {
984 if (mSetAudioDevicePromise.IsEmpty()) {
985 return;
987 LOG(LogLevel::Info,
988 ("MediaStreamRenderer=%p resolve audio device promise", this));
989 mSetAudioDevicePromise.Resolve(true, aMethodName);
990 mDeviceStartedRequest.Disconnect();
993 // True when all tracks are being rendered, i.e., when the media element is
994 // playing.
995 bool mRendering = false;
997 // True while we're progressing mGraphTime. False otherwise.
998 bool mProgressingCurrentTime = false;
1000 // The audio output volume for all audio tracks.
1001 float mAudioOutputVolume = 1.0f;
1003 // The sink device for all audio tracks.
1004 RefPtr<AudioDeviceInfo> mAudioOutputSink;
1005 // The promise returned from SetAudioOutputDevice() when an output is
1006 // active.
1007 MozPromiseHolder<GenericPromise> mSetAudioDevicePromise;
1008 // Request tracking the promise to indicate when the device passed to
1009 // SetAudioOutputDevice() is running.
1010 MozPromiseRequestHolder<GenericPromise::AllSettledPromiseType>
1011 mDeviceStartedRequest;
1013 // WatchManager for mGraphTime.
1014 WatchManager<MediaStreamRenderer> mWatchManager;
1016 // A dummy MediaTrack to guarantee a MediaTrackGraph is kept alive while
1017 // we're actively rendering, so we can track the graph's current time. Set
1018 // when the first track is added, never unset.
1019 RefPtr<SharedDummyTrack> mGraphTimeDummy;
1021 // Watchable that relays the graph's currentTime updates to the media element
1022 // only while we're rendering. This is the current time of the rendering in
1023 // GraphTime units.
1024 Watchable<GraphTime> mGraphTime = {0, "MediaStreamRenderer::mGraphTime"};
1026 // Nothing until a track has been added. Then, the current GraphTime at the
1027 // time when we were last Start()ed.
1028 Maybe<GraphTime> mGraphTimeOffset;
1030 // Currently enabled (and rendered) audio tracks.
1031 nsTArray<WeakPtr<MediaStreamTrack>> mAudioTracks;
1033 // Currently selected (and rendered) video track.
1034 WeakPtr<MediaStreamTrack> mVideoTrack;
1036 // Holds a reference to the first-frame-getting video output attached to
1037 // mVideoTrack. Set by the constructor, unset when the media element tells us
1038 // it has rendered the first frame.
1039 RefPtr<FirstFrameVideoOutput> mFirstFrameVideoOutput;
1042 static uint32_t sDecoderCaptureSourceId = 0;
1043 static uint32_t sStreamCaptureSourceId = 0;
1044 class HTMLMediaElement::MediaElementTrackSource
1045 : public MediaStreamTrackSource,
1046 public MediaStreamTrackSource::Sink,
1047 public MediaStreamTrackConsumer {
1048 public:
1049 NS_DECL_ISUPPORTS_INHERITED
1050 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,
1051 MediaStreamTrackSource)
1053 /* MediaDecoder track source */
1054 MediaElementTrackSource(ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal,
1055 OutputMuteState aMuteState, bool aHasAlpha)
1056 : MediaStreamTrackSource(
1057 aPrincipal, nsString(),
1058 TrackingId(TrackingId::Source::MediaElementDecoder,
1059 sDecoderCaptureSourceId++,
1060 TrackingId::TrackAcrossProcesses::Yes)),
1061 mTrack(aTrack),
1062 mIntendedElementMuteState(aMuteState),
1063 mElementMuteState(aMuteState),
1064 mMediaDecoderHasAlpha(Some(aHasAlpha)) {
1065 MOZ_ASSERT(mTrack);
1068 /* MediaStream track source */
1069 MediaElementTrackSource(MediaStreamTrack* aCapturedTrack,
1070 MediaStreamTrackSource* aCapturedTrackSource,
1071 ProcessedMediaTrack* aTrack, MediaInputPort* aPort,
1072 OutputMuteState aMuteState)
1073 : MediaStreamTrackSource(
1074 aCapturedTrackSource->GetPrincipal(), nsString(),
1075 TrackingId(TrackingId::Source::MediaElementStream,
1076 sStreamCaptureSourceId++,
1077 TrackingId::TrackAcrossProcesses::Yes)),
1078 mCapturedTrack(aCapturedTrack),
1079 mCapturedTrackSource(aCapturedTrackSource),
1080 mTrack(aTrack),
1081 mPort(aPort),
1082 mIntendedElementMuteState(aMuteState),
1083 mElementMuteState(aMuteState) {
1084 MOZ_ASSERT(mTrack);
1085 MOZ_ASSERT(mCapturedTrack);
1086 MOZ_ASSERT(mCapturedTrackSource);
1087 MOZ_ASSERT(mPort);
1089 mCapturedTrack->AddConsumer(this);
1090 mCapturedTrackSource->RegisterSink(this);
1093 void SetEnabled(bool aEnabled) {
1094 if (!mTrack) {
1095 return;
1097 mTrack->SetDisabledTrackMode(aEnabled ? DisabledTrackMode::ENABLED
1098 : DisabledTrackMode::SILENCE_FREEZE);
1101 void SetPrincipal(RefPtr<nsIPrincipal> aPrincipal) {
1102 mPrincipal = std::move(aPrincipal);
1103 MediaStreamTrackSource::PrincipalChanged();
1106 void SetMutedByElement(OutputMuteState aMuteState) {
1107 if (mIntendedElementMuteState == aMuteState) {
1108 return;
1110 mIntendedElementMuteState = aMuteState;
1111 GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
1112 "MediaElementTrackSource::SetMutedByElement",
1113 [self = RefPtr<MediaElementTrackSource>(this), this, aMuteState] {
1114 mElementMuteState = aMuteState;
1115 MediaStreamTrackSource::MutedChanged(Muted());
1116 }));
1119 void Destroy() override {
1120 if (mCapturedTrack) {
1121 mCapturedTrack->RemoveConsumer(this);
1122 mCapturedTrack = nullptr;
1124 if (mCapturedTrackSource) {
1125 mCapturedTrackSource->UnregisterSink(this);
1126 mCapturedTrackSource = nullptr;
1128 if (mTrack && !mTrack->IsDestroyed()) {
1129 mTrack->Destroy();
1131 if (mPort) {
1132 mPort->Destroy();
1133 mPort = nullptr;
1137 MediaSourceEnum GetMediaSource() const override {
1138 return MediaSourceEnum::Other;
1141 void Stop() override {
1142 // Do nothing. There may appear new output streams
1143 // that need tracks sourced from this source, so we
1144 // cannot destroy things yet.
1148 * Do not keep the track source alive. The source lifetime is controlled by
1149 * its associated tracks.
1151 bool KeepsSourceAlive() const override { return false; }
1154 * Do not keep the track source on. It is controlled by its associated tracks.
1156 bool Enabled() const override { return false; }
1158 void Disable() override {}
1160 void Enable() override {}
1162 void PrincipalChanged() override {
1163 if (!mCapturedTrackSource) {
1164 // This could happen during shutdown.
1165 return;
1168 SetPrincipal(mCapturedTrackSource->GetPrincipal());
1171 void MutedChanged(bool aNewState) override {
1172 MediaStreamTrackSource::MutedChanged(Muted());
1175 void OverrideEnded() override {
1176 Destroy();
1177 MediaStreamTrackSource::OverrideEnded();
1180 void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) override {
1181 MediaStreamTrackSource::MutedChanged(Muted());
1184 bool Muted() const {
1185 return mElementMuteState == OutputMuteState::Muted ||
1186 (mCapturedTrack &&
1187 (mCapturedTrack->Muted() || !mCapturedTrack->Enabled()));
1190 bool HasAlpha() const override {
1191 if (mCapturedTrack) {
1192 return mCapturedTrack->AsVideoStreamTrack()
1193 ? mCapturedTrack->AsVideoStreamTrack()->HasAlpha()
1194 : false;
1196 return mMediaDecoderHasAlpha.valueOr(false);
1199 ProcessedMediaTrack* Track() const { return mTrack; }
1201 private:
1202 virtual ~MediaElementTrackSource() { Destroy(); };
1204 RefPtr<MediaStreamTrack> mCapturedTrack;
1205 RefPtr<MediaStreamTrackSource> mCapturedTrackSource;
1206 const RefPtr<ProcessedMediaTrack> mTrack;
1207 RefPtr<MediaInputPort> mPort;
1208 // The mute state as intended by the media element.
1209 OutputMuteState mIntendedElementMuteState;
1210 // The mute state as applied to this track source. It is applied async, so
1211 // needs to be tracked separately from the intended state.
1212 OutputMuteState mElementMuteState;
1213 // Some<bool> if this is a MediaDecoder track source.
1214 const Maybe<bool> mMediaDecoderHasAlpha;
1217 HTMLMediaElement::OutputMediaStream::OutputMediaStream(
1218 RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
1219 bool aFinishWhenEnded)
1220 : mStream(std::move(aStream)),
1221 mCapturingAudioOnly(aCapturingAudioOnly),
1222 mFinishWhenEnded(aFinishWhenEnded) {}
1223 HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default;
1225 void ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback& aCallback,
1226 HTMLMediaElement::OutputMediaStream& aField,
1227 const char* aName, uint32_t aFlags) {
1228 ImplCycleCollectionTraverse(aCallback, aField.mStream, "mStream", aFlags);
1229 ImplCycleCollectionTraverse(aCallback, aField.mLiveTracks, "mLiveTracks",
1230 aFlags);
1231 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedLoadingSrc,
1232 "mFinishWhenEndedLoadingSrc", aFlags);
1233 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedAttrStream,
1234 "mFinishWhenEndedAttrStream", aFlags);
1235 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedMediaSource,
1236 "mFinishWhenEndedMediaSource", aFlags);
1239 void ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream& aField) {
1240 ImplCycleCollectionUnlink(aField.mStream);
1241 ImplCycleCollectionUnlink(aField.mLiveTracks);
1242 ImplCycleCollectionUnlink(aField.mFinishWhenEndedLoadingSrc);
1243 ImplCycleCollectionUnlink(aField.mFinishWhenEndedAttrStream);
1244 ImplCycleCollectionUnlink(aField.mFinishWhenEndedMediaSource);
1247 NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1248 MediaStreamTrackSource)
1249 NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1250 MediaStreamTrackSource)
1251 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1252 HTMLMediaElement::MediaElementTrackSource)
1253 NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource)
1254 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource)
1255 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(
1256 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1257 tmp->Destroy();
1258 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrack)
1259 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource)
1260 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1261 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
1262 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1263 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrack)
1264 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource)
1265 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1268 * There is a reference cycle involving this class: MediaLoadListener
1269 * holds a reference to the HTMLMediaElement, which holds a reference
1270 * to an nsIChannel, which holds a reference to this listener.
1271 * We break the reference cycle in OnStartRequest by clearing mElement.
1273 class HTMLMediaElement::MediaLoadListener final
1274 : public nsIChannelEventSink,
1275 public nsIInterfaceRequestor,
1276 public nsIObserver,
1277 public nsIThreadRetargetableStreamListener {
1278 ~MediaLoadListener() = default;
1280 NS_DECL_THREADSAFE_ISUPPORTS
1281 NS_DECL_NSIREQUESTOBSERVER
1282 NS_DECL_NSISTREAMLISTENER
1283 NS_DECL_NSICHANNELEVENTSINK
1284 NS_DECL_NSIOBSERVER
1285 NS_DECL_NSIINTERFACEREQUESTOR
1286 NS_DECL_NSITHREADRETARGETABLESTREAMLISTENER
1288 public:
1289 explicit MediaLoadListener(HTMLMediaElement* aElement)
1290 : mElement(aElement), mLoadID(aElement->GetCurrentLoadID()) {
1291 MOZ_ASSERT(mElement, "Must pass an element to call back");
1294 private:
1295 RefPtr<HTMLMediaElement> mElement;
1296 nsCOMPtr<nsIStreamListener> mNextListener;
1297 const uint32_t mLoadID;
1300 NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener, nsIRequestObserver,
1301 nsIStreamListener, nsIChannelEventSink, nsIInterfaceRequestor,
1302 nsIObserver, nsIThreadRetargetableStreamListener)
1304 NS_IMETHODIMP
1305 HTMLMediaElement::MediaLoadListener::Observe(nsISupports* aSubject,
1306 const char* aTopic,
1307 const char16_t* aData) {
1308 nsContentUtils::UnregisterShutdownObserver(this);
1310 // Clear mElement to break cycle so we don't leak on shutdown
1311 mElement = nullptr;
1312 return NS_OK;
1315 NS_IMETHODIMP
1316 HTMLMediaElement::MediaLoadListener::OnStartRequest(nsIRequest* aRequest) {
1317 nsContentUtils::UnregisterShutdownObserver(this);
1319 if (!mElement) {
1320 // We've been notified by the shutdown observer, and are shutting down.
1321 return NS_BINDING_ABORTED;
1324 // The element is only needed until we've had a chance to call
1325 // InitializeDecoderForChannel. So make sure mElement is cleared here.
1326 RefPtr<HTMLMediaElement> element;
1327 element.swap(mElement);
1329 if (mLoadID != element->GetCurrentLoadID()) {
1330 // The channel has been cancelled before we had a chance to create
1331 // a decoder. Abort, don't dispatch an "error" event, as the new load
1332 // may not be in an error state.
1333 return NS_BINDING_ABORTED;
1336 // Don't continue to load if the request failed or has been canceled.
1337 nsresult status;
1338 nsresult rv = aRequest->GetStatus(&status);
1339 NS_ENSURE_SUCCESS(rv, rv);
1340 if (NS_FAILED(status)) {
1341 if (element) {
1342 // Handle media not loading error because source was a tracking URL (or
1343 // fingerprinting, cryptomining, etc).
1344 // We make a note of this media node by including it in a dedicated
1345 // array of blocked tracking nodes under its parent document.
1346 if (net::UrlClassifierFeatureFactory::IsClassifierBlockingErrorCode(
1347 status)) {
1348 element->OwnerDoc()->AddBlockedNodeByClassifier(element);
1350 element->NotifyLoadError(
1351 nsPrintfCString("%u: %s", uint32_t(status), "Request failed"));
1353 return status;
1356 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(aRequest);
1357 bool succeeded;
1358 if (hc && NS_SUCCEEDED(hc->GetRequestSucceeded(&succeeded)) && !succeeded) {
1359 uint32_t responseStatus = 0;
1360 Unused << hc->GetResponseStatus(&responseStatus);
1361 nsAutoCString statusText;
1362 Unused << hc->GetResponseStatusText(statusText);
1363 // we need status text for resist fingerprinting mode's message allowlist
1364 if (statusText.IsEmpty()) {
1365 net_GetDefaultStatusTextForCode(responseStatus, statusText);
1367 element->NotifyLoadError(
1368 nsPrintfCString("%u: %s", responseStatus, statusText.get()));
1370 nsAutoString code;
1371 code.AppendInt(responseStatus);
1372 nsAutoString src;
1373 element->GetCurrentSrc(src);
1374 AutoTArray<nsString, 2> params = {code, src};
1375 element->ReportLoadError("MediaLoadHttpError", params);
1376 return NS_BINDING_ABORTED;
1379 nsCOMPtr<nsIChannel> channel = do_QueryInterface(aRequest);
1380 if (channel &&
1381 NS_SUCCEEDED(rv = element->InitializeDecoderForChannel(
1382 channel, getter_AddRefs(mNextListener))) &&
1383 mNextListener) {
1384 rv = mNextListener->OnStartRequest(aRequest);
1385 } else {
1386 // If InitializeDecoderForChannel() returned an error, fire a network error.
1387 if (NS_FAILED(rv) && !mNextListener) {
1388 // Load failed, attempt to load the next candidate resource. If there
1389 // are none, this will trigger a MEDIA_ERR_SRC_NOT_SUPPORTED error.
1390 element->NotifyLoadError("Failed to init decoder"_ns);
1392 // If InitializeDecoderForChannel did not return a listener (but may
1393 // have otherwise succeeded), we abort the connection since we aren't
1394 // interested in keeping the channel alive ourselves.
1395 rv = NS_BINDING_ABORTED;
1398 return rv;
1401 NS_IMETHODIMP
1402 HTMLMediaElement::MediaLoadListener::OnStopRequest(nsIRequest* aRequest,
1403 nsresult aStatus) {
1404 if (mNextListener) {
1405 return mNextListener->OnStopRequest(aRequest, aStatus);
1407 return NS_OK;
1410 NS_IMETHODIMP
1411 HTMLMediaElement::MediaLoadListener::OnDataAvailable(nsIRequest* aRequest,
1412 nsIInputStream* aStream,
1413 uint64_t aOffset,
1414 uint32_t aCount) {
1415 if (!mNextListener) {
1416 NS_ERROR(
1417 "Must have a chained listener; OnStartRequest should have "
1418 "canceled this request");
1419 return NS_BINDING_ABORTED;
1421 return mNextListener->OnDataAvailable(aRequest, aStream, aOffset, aCount);
1424 NS_IMETHODIMP
1425 HTMLMediaElement::MediaLoadListener::OnDataFinished(nsresult aStatus) {
1426 if (!mNextListener) {
1427 return NS_ERROR_FAILURE;
1429 nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
1430 do_QueryInterface(mNextListener);
1431 if (retargetable) {
1432 return retargetable->OnDataFinished(aStatus);
1435 return NS_OK;
1438 NS_IMETHODIMP
1439 HTMLMediaElement::MediaLoadListener::AsyncOnChannelRedirect(
1440 nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t aFlags,
1441 nsIAsyncVerifyRedirectCallback* cb) {
1442 // TODO is this really correct?? See bug #579329.
1443 if (mElement) {
1444 mElement->OnChannelRedirect(aOldChannel, aNewChannel, aFlags);
1446 nsCOMPtr<nsIChannelEventSink> sink = do_QueryInterface(mNextListener);
1447 if (sink) {
1448 return sink->AsyncOnChannelRedirect(aOldChannel, aNewChannel, aFlags, cb);
1450 cb->OnRedirectVerifyCallback(NS_OK);
1451 return NS_OK;
1454 NS_IMETHODIMP
1455 HTMLMediaElement::MediaLoadListener::CheckListenerChain() {
1456 MOZ_ASSERT(mNextListener);
1457 nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
1458 do_QueryInterface(mNextListener);
1459 if (retargetable) {
1460 return retargetable->CheckListenerChain();
1462 return NS_ERROR_NO_INTERFACE;
1465 NS_IMETHODIMP
1466 HTMLMediaElement::MediaLoadListener::GetInterface(const nsIID& aIID,
1467 void** aResult) {
1468 return QueryInterface(aIID, aResult);
1471 void HTMLMediaElement::ReportLoadError(const char* aMsg,
1472 const nsTArray<nsString>& aParams) {
1473 ReportToConsole(nsIScriptError::warningFlag, aMsg, aParams);
1476 void HTMLMediaElement::ReportToConsole(
1477 uint32_t aErrorFlags, const char* aMsg,
1478 const nsTArray<nsString>& aParams) const {
1479 nsContentUtils::ReportToConsole(aErrorFlags, "Media"_ns, OwnerDoc(),
1480 nsContentUtils::eDOM_PROPERTIES, aMsg,
1481 aParams);
1484 class HTMLMediaElement::AudioChannelAgentCallback final
1485 : public nsIAudioChannelAgentCallback {
1486 public:
1487 NS_DECL_CYCLE_COLLECTING_ISUPPORTS
1488 NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)
1490 explicit AudioChannelAgentCallback(HTMLMediaElement* aOwner)
1491 : mOwner(aOwner),
1492 mAudioChannelVolume(1.0),
1493 mPlayingThroughTheAudioChannel(false),
1494 mIsOwnerAudible(IsOwnerAudible()),
1495 mIsShutDown(false) {
1496 MOZ_ASSERT(mOwner);
1497 MaybeCreateAudioChannelAgent();
1500 void UpdateAudioChannelPlayingState() {
1501 MOZ_ASSERT(!mIsShutDown);
1502 bool playingThroughTheAudioChannel = IsPlayingThroughTheAudioChannel();
1504 if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) {
1505 if (!MaybeCreateAudioChannelAgent()) {
1506 return;
1509 mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel;
1510 if (mPlayingThroughTheAudioChannel) {
1511 StartAudioChannelAgent();
1512 } else {
1513 StopAudioChanelAgent();
1518 void NotifyPlayStateChanged() {
1519 MOZ_ASSERT(!mIsShutDown);
1520 UpdateAudioChannelPlayingState();
1523 NS_IMETHODIMP WindowVolumeChanged(float aVolume, bool aMuted) override {
1524 MOZ_ASSERT(mAudioChannelAgent);
1526 MOZ_LOG(
1527 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1528 ("HTMLMediaElement::AudioChannelAgentCallback, WindowVolumeChanged, "
1529 "this = %p, aVolume = %f, aMuted = %s\n",
1530 this, aVolume, aMuted ? "true" : "false"));
1532 if (mAudioChannelVolume != aVolume) {
1533 mAudioChannelVolume = aVolume;
1534 mOwner->SetVolumeInternal();
1537 const uint32_t muted = mOwner->mMuted;
1538 if (aMuted && !mOwner->ComputedMuted()) {
1539 mOwner->SetMutedInternal(muted | MUTED_BY_AUDIO_CHANNEL);
1540 } else if (!aMuted && mOwner->ComputedMuted()) {
1541 mOwner->SetMutedInternal(muted & ~MUTED_BY_AUDIO_CHANNEL);
1544 return NS_OK;
1547 NS_IMETHODIMP WindowSuspendChanged(SuspendTypes aSuspend) override {
1548 // Currently this method is only be used for delaying autoplay, and we've
1549 // separated related codes to `MediaPlaybackDelayPolicy`.
1550 return NS_OK;
1553 NS_IMETHODIMP WindowAudioCaptureChanged(bool aCapture) override {
1554 MOZ_ASSERT(mAudioChannelAgent);
1555 AudioCaptureTrackChangeIfNeeded();
1556 return NS_OK;
1559 void AudioCaptureTrackChangeIfNeeded() {
1560 MOZ_ASSERT(!mIsShutDown);
1561 if (!IsPlayingStarted()) {
1562 return;
1565 MOZ_ASSERT(mAudioChannelAgent);
1566 bool isCapturing = mAudioChannelAgent->IsWindowAudioCapturingEnabled();
1567 mOwner->AudioCaptureTrackChange(isCapturing);
1570 void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason) {
1571 MOZ_ASSERT(!mIsShutDown);
1572 AudibleState newAudibleState = IsOwnerAudible();
1573 MOZ_LOG(AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1574 ("HTMLMediaElement::AudioChannelAgentCallback, "
1575 "NotifyAudioPlaybackChanged, this=%p, current=%s, new=%s",
1576 this, AudibleStateToStr(mIsOwnerAudible),
1577 AudibleStateToStr(newAudibleState)));
1578 if (mIsOwnerAudible == newAudibleState) {
1579 return;
1582 mIsOwnerAudible = newAudibleState;
1583 if (IsPlayingStarted()) {
1584 mAudioChannelAgent->NotifyStartedAudible(mIsOwnerAudible, aReason);
1588 void Shutdown() {
1589 MOZ_ASSERT(!mIsShutDown);
1590 if (mAudioChannelAgent && mAudioChannelAgent->IsPlayingStarted()) {
1591 StopAudioChanelAgent();
1593 mAudioChannelAgent = nullptr;
1594 mIsShutDown = true;
1597 float GetEffectiveVolume() const {
1598 MOZ_ASSERT(!mIsShutDown);
1599 return static_cast<float>(mOwner->Volume()) * mAudioChannelVolume;
1602 private:
1603 ~AudioChannelAgentCallback() { MOZ_ASSERT(mIsShutDown); };
1605 bool MaybeCreateAudioChannelAgent() {
1606 if (mAudioChannelAgent) {
1607 return true;
1610 mAudioChannelAgent = new AudioChannelAgent();
1611 nsresult rv =
1612 mAudioChannelAgent->Init(mOwner->OwnerDoc()->GetInnerWindow(), this);
1613 if (NS_WARN_IF(NS_FAILED(rv))) {
1614 mAudioChannelAgent = nullptr;
1615 MOZ_LOG(
1616 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1617 ("HTMLMediaElement::AudioChannelAgentCallback, Fail to initialize "
1618 "the audio channel agent, this = %p\n",
1619 this));
1620 return false;
1623 return true;
1626 void StartAudioChannelAgent() {
1627 MOZ_ASSERT(mAudioChannelAgent);
1628 MOZ_ASSERT(!mAudioChannelAgent->IsPlayingStarted());
1629 if (NS_WARN_IF(NS_FAILED(
1630 mAudioChannelAgent->NotifyStartedPlaying(IsOwnerAudible())))) {
1631 return;
1633 mAudioChannelAgent->PullInitialUpdate();
1636 void StopAudioChanelAgent() {
1637 MOZ_ASSERT(mAudioChannelAgent);
1638 MOZ_ASSERT(mAudioChannelAgent->IsPlayingStarted());
1639 mAudioChannelAgent->NotifyStoppedPlaying();
1640 // If we have started audio capturing before, we have to tell media element
1641 // to clear the output capturing track.
1642 mOwner->AudioCaptureTrackChange(false);
1645 bool IsPlayingStarted() {
1646 if (MaybeCreateAudioChannelAgent()) {
1647 return mAudioChannelAgent->IsPlayingStarted();
1649 return false;
1652 AudibleState IsOwnerAudible() const {
1653 // paused media doesn't produce any sound.
1654 if (mOwner->mPaused) {
1655 return AudibleState::eNotAudible;
1657 return mOwner->IsAudible() ? AudibleState::eAudible
1658 : AudibleState::eNotAudible;
1661 bool IsPlayingThroughTheAudioChannel() const {
1662 // If we have an error, we are not playing.
1663 if (mOwner->GetError()) {
1664 return false;
1667 // We should consider any bfcached page or inactive document as non-playing.
1668 if (!mOwner->OwnerDoc()->IsActive()) {
1669 return false;
1672 // Media is suspended by the docshell.
1673 if (mOwner->ShouldBeSuspendedByInactiveDocShell()) {
1674 return false;
1677 // Are we paused
1678 if (mOwner->mPaused) {
1679 return false;
1682 // No audio track
1683 if (!mOwner->HasAudio()) {
1684 return false;
1687 // A loop always is playing
1688 if (mOwner->HasAttr(nsGkAtoms::loop)) {
1689 return true;
1692 // If we are actually playing...
1693 if (mOwner->IsCurrentlyPlaying()) {
1694 return true;
1697 // If we are playing an external stream.
1698 if (mOwner->mSrcAttrStream) {
1699 return true;
1702 return false;
1705 RefPtr<AudioChannelAgent> mAudioChannelAgent;
1706 HTMLMediaElement* mOwner;
1708 // The audio channel volume
1709 float mAudioChannelVolume;
1710 // Is this media element playing?
1711 bool mPlayingThroughTheAudioChannel;
1712 // Indicate whether media element is audible for users.
1713 AudibleState mIsOwnerAudible;
1714 bool mIsShutDown;
1717 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::AudioChannelAgentCallback)
1719 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(
1720 HTMLMediaElement::AudioChannelAgentCallback)
1721 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelAgent)
1722 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1724 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(
1725 HTMLMediaElement::AudioChannelAgentCallback)
1726 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelAgent)
1727 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1729 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1730 HTMLMediaElement::AudioChannelAgentCallback)
1731 NS_INTERFACE_MAP_ENTRY(nsIAudioChannelAgentCallback)
1732 NS_INTERFACE_MAP_END
1734 NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::AudioChannelAgentCallback)
1735 NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::AudioChannelAgentCallback)
1737 class HTMLMediaElement::ChannelLoader final {
1738 public:
1739 NS_INLINE_DECL_REFCOUNTING(ChannelLoader);
1741 void LoadInternal(HTMLMediaElement* aElement) {
1742 if (mCancelled) {
1743 return;
1746 // determine what security checks need to be performed in AsyncOpen().
1747 nsSecurityFlags securityFlags =
1748 aElement->ShouldCheckAllowOrigin()
1749 ? nsILoadInfo::SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT
1750 : nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT;
1752 if (aElement->GetCORSMode() == CORS_USE_CREDENTIALS) {
1753 securityFlags |= nsILoadInfo::SEC_COOKIES_INCLUDE;
1756 securityFlags |= nsILoadInfo::SEC_ALLOW_CHROME;
1758 MOZ_ASSERT(
1759 aElement->IsAnyOfHTMLElements(nsGkAtoms::audio, nsGkAtoms::video));
1760 nsContentPolicyType contentPolicyType =
1761 aElement->IsHTMLElement(nsGkAtoms::audio)
1762 ? nsIContentPolicy::TYPE_INTERNAL_AUDIO
1763 : nsIContentPolicy::TYPE_INTERNAL_VIDEO;
1765 // If aElement has 'triggeringprincipal' attribute, we will use the value as
1766 // triggeringPrincipal for the channel, otherwise it will default to use
1767 // aElement->NodePrincipal().
1768 // This function returns true when aElement has 'triggeringprincipal', so if
1769 // setAttrs is true we will override the origin attributes on the channel
1770 // later.
1771 nsCOMPtr<nsIPrincipal> triggeringPrincipal;
1772 bool setAttrs = nsContentUtils::QueryTriggeringPrincipal(
1773 aElement, aElement->mLoadingSrcTriggeringPrincipal,
1774 getter_AddRefs(triggeringPrincipal));
1776 nsCOMPtr<nsILoadGroup> loadGroup = aElement->GetDocumentLoadGroup();
1777 nsCOMPtr<nsIChannel> channel;
1778 nsresult rv = NS_NewChannelWithTriggeringPrincipal(
1779 getter_AddRefs(channel), aElement->mLoadingSrc,
1780 static_cast<Element*>(aElement), triggeringPrincipal, securityFlags,
1781 contentPolicyType,
1782 nullptr, // aPerformanceStorage
1783 loadGroup,
1784 nullptr, // aCallbacks
1785 nsICachingChannel::LOAD_BYPASS_LOCAL_CACHE_IF_BUSY |
1786 nsIChannel::LOAD_MEDIA_SNIFFER_OVERRIDES_CONTENT_TYPE |
1787 nsIChannel::LOAD_CALL_CONTENT_SNIFFERS);
1789 if (NS_FAILED(rv)) {
1790 // Notify load error so the element will try next resource candidate.
1791 aElement->NotifyLoadError("Fail to create channel"_ns);
1792 return;
1795 nsCOMPtr<nsILoadInfo> loadInfo = channel->LoadInfo();
1796 if (setAttrs) {
1797 // The function simply returns NS_OK, so we ignore the return value.
1798 Unused << loadInfo->SetOriginAttributes(
1799 triggeringPrincipal->OriginAttributesRef());
1801 loadInfo->SetIsMediaRequest(true);
1802 loadInfo->SetIsMediaInitialRequest(true);
1804 nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(channel));
1805 if (cos) {
1806 if (aElement->mUseUrgentStartForChannel) {
1807 cos->AddClassFlags(nsIClassOfService::UrgentStart);
1809 // Reset the flag to avoid loading again without initiated by user
1810 // interaction.
1811 aElement->mUseUrgentStartForChannel = false;
1814 // Unconditionally disable throttling since we want the media to fluently
1815 // play even when we switch the tab to background.
1816 cos->AddClassFlags(nsIClassOfService::DontThrottle);
1819 // The listener holds a strong reference to us. This creates a
1820 // reference cycle, once we've set mChannel, which is manually broken
1821 // in the listener's OnStartRequest method after it is finished with
1822 // the element. The cycle will also be broken if we get a shutdown
1823 // notification before OnStartRequest fires. Necko guarantees that
1824 // OnStartRequest will eventually fire if we don't shut down first.
1825 RefPtr<MediaLoadListener> loadListener = new MediaLoadListener(aElement);
1827 channel->SetNotificationCallbacks(loadListener);
1829 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(channel);
1830 if (hc) {
1831 // Use a byte range request from the start of the resource.
1832 // This enables us to detect if the stream supports byte range
1833 // requests, and therefore seeking, early.
1834 rv = hc->SetRequestHeader("Range"_ns, "bytes=0-"_ns, false);
1835 MOZ_ASSERT(NS_SUCCEEDED(rv));
1836 aElement->SetRequestHeaders(hc);
1839 rv = channel->AsyncOpen(loadListener);
1840 if (NS_FAILED(rv)) {
1841 // Notify load error so the element will try next resource candidate.
1842 aElement->NotifyLoadError("Failed to open channel"_ns);
1843 return;
1846 // Else the channel must be open and starting to download. If it encounters
1847 // a non-catastrophic failure, it will set a new task to continue loading
1848 // another candidate. It's safe to set it as mChannel now.
1849 mChannel = channel;
1851 // loadListener will be unregistered either on shutdown or when
1852 // OnStartRequest for the channel we just opened fires.
1853 nsContentUtils::RegisterShutdownObserver(loadListener);
1856 nsresult Load(HTMLMediaElement* aElement) {
1857 MOZ_ASSERT(aElement);
1858 // Per bug 1235183 comment 8, we can't spin the event loop from stable
1859 // state. Defer NS_NewChannel() to a new regular runnable.
1860 return aElement->OwnerDoc()->Dispatch(NewRunnableMethod<HTMLMediaElement*>(
1861 "ChannelLoader::LoadInternal", this, &ChannelLoader::LoadInternal,
1862 aElement));
1865 void Cancel() {
1866 mCancelled = true;
1867 if (mChannel) {
1868 mChannel->CancelWithReason(NS_BINDING_ABORTED,
1869 "HTMLMediaElement::ChannelLoader::Cancel"_ns);
1870 mChannel = nullptr;
1874 void Done() {
1875 MOZ_ASSERT(mChannel);
1876 // Decoder successfully created, the decoder now owns the MediaResource
1877 // which owns the channel.
1878 mChannel = nullptr;
1881 nsresult Redirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
1882 uint32_t aFlags) {
1883 NS_ASSERTION(aChannel == mChannel, "Channels should match!");
1884 mChannel = aNewChannel;
1886 // Handle forwarding of Range header so that the intial detection
1887 // of seeking support (via result code 206) works across redirects.
1888 nsCOMPtr<nsIHttpChannel> http = do_QueryInterface(aChannel);
1889 NS_ENSURE_STATE(http);
1891 constexpr auto rangeHdr = "Range"_ns;
1893 nsAutoCString rangeVal;
1894 if (NS_SUCCEEDED(http->GetRequestHeader(rangeHdr, rangeVal))) {
1895 NS_ENSURE_STATE(!rangeVal.IsEmpty());
1897 http = do_QueryInterface(aNewChannel);
1898 NS_ENSURE_STATE(http);
1900 nsresult rv = http->SetRequestHeader(rangeHdr, rangeVal, false);
1901 NS_ENSURE_SUCCESS(rv, rv);
1904 return NS_OK;
1907 private:
1908 ~ChannelLoader() { MOZ_ASSERT(!mChannel); }
1909 // Holds a reference to the first channel we open to the media resource.
1910 // Once the decoder is created, control over the channel passes to the
1911 // decoder, and we null out this reference. We must store this in case
1912 // we need to cancel the channel before control of it passes to the decoder.
1913 nsCOMPtr<nsIChannel> mChannel;
1915 bool mCancelled = false;
1918 class HTMLMediaElement::ErrorSink {
1919 public:
1920 explicit ErrorSink(HTMLMediaElement* aOwner) : mOwner(aOwner) {
1921 MOZ_ASSERT(mOwner);
1924 void SetError(uint16_t aErrorCode, const nsACString& aErrorDetails) {
1925 // Since we have multiple paths calling into DecodeError, e.g.
1926 // MediaKeys::Terminated and EMEH264Decoder::Error. We should take the 1st
1927 // one only in order not to fire multiple 'error' events.
1928 if (mError) {
1929 return;
1932 if (!IsValidErrorCode(aErrorCode)) {
1933 NS_ASSERTION(false, "Undefined MediaError codes!");
1934 return;
1937 mError = new MediaError(mOwner, aErrorCode, aErrorDetails);
1938 mOwner->DispatchAsyncEvent(u"error"_ns);
1939 if (mOwner->ReadyState() == HAVE_NOTHING &&
1940 aErrorCode == MEDIA_ERR_ABORTED) {
1941 // https://html.spec.whatwg.org/multipage/embedded-content.html#media-data-processing-steps-list
1942 // "If the media data fetching process is aborted by the user"
1943 mOwner->DispatchAsyncEvent(u"abort"_ns);
1944 mOwner->ChangeNetworkState(NETWORK_EMPTY);
1945 mOwner->DispatchAsyncEvent(u"emptied"_ns);
1946 if (mOwner->mDecoder) {
1947 mOwner->ShutdownDecoder();
1949 } else if (aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED) {
1950 mOwner->ChangeNetworkState(NETWORK_NO_SOURCE);
1951 } else {
1952 mOwner->ChangeNetworkState(NETWORK_IDLE);
1956 void ResetError() { mError = nullptr; }
1958 RefPtr<MediaError> mError;
1960 private:
1961 bool IsValidErrorCode(const uint16_t& aErrorCode) const {
1962 return (aErrorCode == MEDIA_ERR_DECODE || aErrorCode == MEDIA_ERR_NETWORK ||
1963 aErrorCode == MEDIA_ERR_ABORTED ||
1964 aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED);
1967 // Media elememt's life cycle would be longer than error sink, so we use the
1968 // raw pointer and this class would only be referenced by media element.
1969 HTMLMediaElement* mOwner;
1972 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement)
1974 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement,
1975 nsGenericHTMLElement)
1976 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource)
1977 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource)
1978 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream)
1979 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream)
1980 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer)
1981 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc)
1982 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourceLoadCandidate)
1983 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelWrapper)
1984 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError)
1985 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams)
1986 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputTrackSources);
1987 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed);
1988 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager)
1989 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioTrackList)
1990 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoTrackList)
1991 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
1992 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mIncomingMediaKeys)
1993 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
1994 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPendingPlayPromises)
1995 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSeekDOMPromise)
1996 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSetMediaKeysDOMPromise)
1997 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mEventBlocker)
1998 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
2000 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement,
2001 nsGenericHTMLElement)
2002 tmp->RemoveMutationObserver(tmp);
2003 if (tmp->mSrcStream) {
2004 // Need to unhook everything that EndSrcMediaStreamPlayback would normally
2005 // do, without creating any new strong references.
2006 if (tmp->mSelectedVideoStreamTrack) {
2007 tmp->mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(tmp);
2009 if (tmp->mMediaStreamRenderer) {
2010 tmp->mMediaStreamRenderer->Shutdown();
2011 // We null out mMediaStreamRenderer here since Shutdown() will shut down
2012 // its WatchManager, and UpdateSrcStreamPotentiallyPlaying() contains a
2013 // guard for this.
2014 tmp->mMediaStreamRenderer = nullptr;
2016 if (tmp->mSecondaryMediaStreamRenderer) {
2017 tmp->mSecondaryMediaStreamRenderer->Shutdown();
2018 tmp->mSecondaryMediaStreamRenderer = nullptr;
2020 if (tmp->mMediaStreamTrackListener) {
2021 tmp->mSrcStream->UnregisterTrackListener(
2022 tmp->mMediaStreamTrackListener.get());
2025 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcStream)
2026 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcAttrStream)
2027 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaSource)
2028 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcMediaSource)
2029 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourcePointer)
2030 NS_IMPL_CYCLE_COLLECTION_UNLINK(mLoadBlockedDoc)
2031 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourceLoadCandidate)
2032 if (tmp->mAudioChannelWrapper) {
2033 tmp->mAudioChannelWrapper->Shutdown();
2035 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelWrapper)
2036 NS_IMPL_CYCLE_COLLECTION_UNLINK(mErrorSink->mError)
2037 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputStreams)
2038 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputTrackSources)
2039 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlayed)
2040 NS_IMPL_CYCLE_COLLECTION_UNLINK(mTextTrackManager)
2041 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioTrackList)
2042 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoTrackList)
2043 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
2044 NS_IMPL_CYCLE_COLLECTION_UNLINK(mIncomingMediaKeys)
2045 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
2046 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPendingPlayPromises)
2047 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSeekDOMPromise)
2048 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSetMediaKeysDOMPromise)
2049 if (tmp->mMediaControlKeyListener) {
2050 tmp->mMediaControlKeyListener->StopIfNeeded();
2052 if (tmp->mEventBlocker) {
2053 tmp->mEventBlocker->Shutdown();
2055 NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
2056 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
2058 NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,
2059 nsGenericHTMLElement)
2061 void HTMLMediaElement::AddSizeOfExcludingThis(nsWindowSizes& aSizes,
2062 size_t* aNodeSize) const {
2063 nsGenericHTMLElement::AddSizeOfExcludingThis(aSizes, aNodeSize);
2065 // There are many other fields that might be worth reporting, but as seen in
2066 // bug 1595603, the event we postpone to dispatch can grow to be very large
2067 // sometimes, so at least report that.
2068 if (mEventBlocker) {
2069 *aNodeSize +=
2070 mEventBlocker->SizeOfExcludingThis(aSizes.mState.mMallocSizeOf);
2074 void HTMLMediaElement::ContentRemoved(nsIContent* aChild,
2075 nsIContent* aPreviousSibling) {
2076 if (aChild == mSourcePointer) {
2077 mSourcePointer = aPreviousSibling;
2081 already_AddRefed<MediaSource> HTMLMediaElement::GetMozMediaSourceObject()
2082 const {
2083 RefPtr<MediaSource> source = mMediaSource;
2084 return source.forget();
2087 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugInfo(
2088 ErrorResult& aRv) {
2089 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2090 if (NS_WARN_IF(aRv.Failed())) {
2091 return nullptr;
2093 auto result = MakeUnique<dom::HTMLMediaElementDebugInfo>();
2094 if (mMediaKeys) {
2095 GetEMEInfo(result->mEMEInfo);
2097 if (mVideoFrameContainer) {
2098 result->mCompositorDroppedFrames =
2099 mVideoFrameContainer->GetDroppedImageCount();
2101 if (mDecoder) {
2102 mDecoder->RequestDebugInfo(result->mDecoder)
2103 ->Then(
2104 AbstractMainThread(), __func__,
2105 [promise, ptr = std::move(result)]() {
2106 promise->MaybeResolve(ptr.get());
2108 []() {
2109 MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
2111 } else {
2112 promise->MaybeResolve(result.get());
2114 return promise.forget();
2117 /* static */
2118 void HTMLMediaElement::MozEnableDebugLog(const GlobalObject&) {
2119 DecoderDoctorLogger::EnableLogging();
2122 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugLog(
2123 ErrorResult& aRv) {
2124 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2125 if (NS_WARN_IF(aRv.Failed())) {
2126 return nullptr;
2129 DecoderDoctorLogger::RetrieveMessages(this)->Then(
2130 AbstractMainThread(), __func__,
2131 [promise](const nsACString& aString) {
2132 promise->MaybeResolve(NS_ConvertUTF8toUTF16(aString));
2134 [promise](nsresult rv) { promise->MaybeReject(rv); });
2136 return promise.forget();
2139 void HTMLMediaElement::SetVisible(bool aVisible) {
2140 mForcedHidden = !aVisible;
2141 if (mDecoder) {
2142 mDecoder->SetForcedHidden(!aVisible);
2146 bool HTMLMediaElement::IsVideoDecodingSuspended() const {
2147 return mDecoder && mDecoder->IsVideoDecodingSuspended();
2150 double HTMLMediaElement::TotalVideoPlayTime() const {
2151 return mDecoder ? mDecoder->GetTotalVideoPlayTimeInSeconds() : -1.0;
2154 double HTMLMediaElement::TotalVideoHDRPlayTime() const {
2155 return mDecoder ? mDecoder->GetTotalVideoHDRPlayTimeInSeconds() : -1.0;
2158 double HTMLMediaElement::VisiblePlayTime() const {
2159 return mDecoder ? mDecoder->GetVisibleVideoPlayTimeInSeconds() : -1.0;
2162 double HTMLMediaElement::InvisiblePlayTime() const {
2163 return mDecoder ? mDecoder->GetInvisibleVideoPlayTimeInSeconds() : -1.0;
2166 double HTMLMediaElement::TotalAudioPlayTime() const {
2167 return mDecoder ? mDecoder->GetTotalAudioPlayTimeInSeconds() : -1.0;
2170 double HTMLMediaElement::AudiblePlayTime() const {
2171 return mDecoder ? mDecoder->GetAudiblePlayTimeInSeconds() : -1.0;
2174 double HTMLMediaElement::InaudiblePlayTime() const {
2175 return mDecoder ? mDecoder->GetInaudiblePlayTimeInSeconds() : -1.0;
2178 double HTMLMediaElement::MutedPlayTime() const {
2179 return mDecoder ? mDecoder->GetMutedPlayTimeInSeconds() : -1.0;
2182 double HTMLMediaElement::VideoDecodeSuspendedTime() const {
2183 return mDecoder ? mDecoder->GetVideoDecodeSuspendedTimeInSeconds() : -1.0;
2186 void HTMLMediaElement::SetFormatDiagnosticsReportForMimeType(
2187 const nsAString& aMimeType, DecoderDoctorReportType aType) {
2188 DecoderDoctorDiagnostics diagnostics;
2189 diagnostics.SetDecoderDoctorReportType(aType);
2190 diagnostics.StoreFormatDiagnostics(OwnerDoc(), aMimeType, false /* can play*/,
2191 __func__);
2194 void HTMLMediaElement::SetDecodeError(const nsAString& aError,
2195 ErrorResult& aRv) {
2196 // The reason we use this map-ish structure is because we can't use
2197 // `CR.NS_ERROR.*` directly in test. In order to use them in test, we have to
2198 // add them into `xpc.msg`. As we won't use `CR.NS_ERROR.*` in the production
2199 // code, adding them to `xpc.msg` seems an overdesign and adding maintenance
2200 // effort (exposing them in CR also needs to add a description, which is
2201 // useless because we won't show them to users)
2202 static struct {
2203 const char* mName;
2204 nsresult mResult;
2205 } kSupportedErrorList[] = {
2206 {"NS_ERROR_DOM_MEDIA_ABORT_ERR", NS_ERROR_DOM_MEDIA_ABORT_ERR},
2207 {"NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR",
2208 NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR},
2209 {"NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR",
2210 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR},
2211 {"NS_ERROR_DOM_MEDIA_DECODE_ERR", NS_ERROR_DOM_MEDIA_DECODE_ERR},
2212 {"NS_ERROR_DOM_MEDIA_FATAL_ERR", NS_ERROR_DOM_MEDIA_FATAL_ERR},
2213 {"NS_ERROR_DOM_MEDIA_METADATA_ERR", NS_ERROR_DOM_MEDIA_METADATA_ERR},
2214 {"NS_ERROR_DOM_MEDIA_OVERFLOW_ERR", NS_ERROR_DOM_MEDIA_OVERFLOW_ERR},
2215 {"NS_ERROR_DOM_MEDIA_MEDIASINK_ERR", NS_ERROR_DOM_MEDIA_MEDIASINK_ERR},
2216 {"NS_ERROR_DOM_MEDIA_DEMUXER_ERR", NS_ERROR_DOM_MEDIA_DEMUXER_ERR},
2217 {"NS_ERROR_DOM_MEDIA_CDM_ERR", NS_ERROR_DOM_MEDIA_CDM_ERR},
2218 {"NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR",
2219 NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR}};
2220 for (auto& error : kSupportedErrorList) {
2221 if (strcmp(error.mName, NS_ConvertUTF16toUTF8(aError).get()) == 0) {
2222 DecoderDoctorDiagnostics diagnostics;
2223 diagnostics.StoreDecodeError(OwnerDoc(), error.mResult, u""_ns, __func__);
2224 return;
2227 aRv.Throw(NS_ERROR_FAILURE);
2230 void HTMLMediaElement::SetAudioSinkFailedStartup() {
2231 DecoderDoctorDiagnostics diagnostics;
2232 diagnostics.StoreEvent(OwnerDoc(),
2233 {DecoderDoctorEvent::eAudioSinkStartup,
2234 NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR},
2235 __func__);
2238 already_AddRefed<layers::Image> HTMLMediaElement::GetCurrentImage() {
2239 MarkAsTainted();
2241 // TODO: In bug 1345404, handle case when video decoder is already suspended.
2242 ImageContainer* container = GetImageContainer();
2243 if (!container) {
2244 return nullptr;
2247 AutoLockImage lockImage(container);
2248 RefPtr<layers::Image> image = lockImage.GetImage(TimeStamp::Now());
2249 return image.forget();
2252 bool HTMLMediaElement::HasSuspendTaint() const {
2253 MOZ_ASSERT(!mDecoder || (mDecoder->HasSuspendTaint() == mHasSuspendTaint));
2254 return mHasSuspendTaint;
2257 already_AddRefed<DOMMediaStream> HTMLMediaElement::GetSrcObject() const {
2258 return do_AddRef(mSrcAttrStream);
2261 void HTMLMediaElement::SetSrcObject(DOMMediaStream& aValue) {
2262 SetSrcObject(&aValue);
2265 void HTMLMediaElement::SetSrcObject(DOMMediaStream* aValue) {
2266 for (auto& outputStream : mOutputStreams) {
2267 if (aValue == outputStream.mStream) {
2268 ReportToConsole(nsIScriptError::warningFlag,
2269 "MediaElementStreamCaptureCycle");
2270 return;
2273 mSrcAttrStream = aValue;
2274 UpdateAudioChannelPlayingState();
2275 DoLoad();
2278 bool HTMLMediaElement::Ended() {
2279 return (mDecoder && mDecoder->IsEnded()) ||
2280 (mSrcStream && mSrcStreamReportPlaybackEnded);
2283 void HTMLMediaElement::GetCurrentSrc(nsAString& aCurrentSrc) {
2284 nsAutoCString src;
2285 GetCurrentSpec(src);
2286 CopyUTF8toUTF16(src, aCurrentSrc);
2289 nsresult HTMLMediaElement::OnChannelRedirect(nsIChannel* aChannel,
2290 nsIChannel* aNewChannel,
2291 uint32_t aFlags) {
2292 MOZ_ASSERT(mChannelLoader);
2293 return mChannelLoader->Redirect(aChannel, aNewChannel, aFlags);
2296 void HTMLMediaElement::ShutdownDecoder() {
2297 RemoveMediaElementFromURITable();
2298 NS_ASSERTION(mDecoder, "Must have decoder to shut down");
2300 mWaitingForKeyListener.DisconnectIfExists();
2301 if (mMediaSource) {
2302 mMediaSource->CompletePendingTransactions();
2304 mDecoder->Shutdown();
2305 DDUNLINKCHILD(mDecoder.get());
2306 mDecoder = nullptr;
2309 void HTMLMediaElement::AbortExistingLoads() {
2310 // Abort any already-running instance of the resource selection algorithm.
2311 mLoadWaitStatus = NOT_WAITING;
2313 // Set a new load ID. This will cause events which were enqueued
2314 // with a different load ID to silently be cancelled.
2315 mCurrentLoadID++;
2317 // Immediately reject or resolve the already-dispatched
2318 // nsResolveOrRejectPendingPlayPromisesRunners. These runners won't be
2319 // executed again later since the mCurrentLoadID had been changed.
2320 for (auto& runner : mPendingPlayPromisesRunners) {
2321 runner->ResolveOrReject();
2323 mPendingPlayPromisesRunners.Clear();
2325 if (mChannelLoader) {
2326 mChannelLoader->Cancel();
2327 mChannelLoader = nullptr;
2330 bool fireTimeUpdate = false;
2332 if (mDecoder) {
2333 fireTimeUpdate = mDecoder->GetCurrentTime() != 0.0;
2334 ShutdownDecoder();
2336 if (mSrcStream) {
2337 EndSrcMediaStreamPlayback();
2340 if (mMediaSource) {
2341 OwnerDoc()->RemoveMediaElementWithMSE();
2344 RemoveMediaElementFromURITable();
2345 mLoadingSrcTriggeringPrincipal = nullptr;
2346 DDLOG(DDLogCategory::Property, "loading_src", "");
2347 DDUNLINKCHILD(mMediaSource.get());
2348 mMediaSource = nullptr;
2350 if (mNetworkState == NETWORK_LOADING || mNetworkState == NETWORK_IDLE) {
2351 DispatchAsyncEvent(u"abort"_ns);
2354 bool hadVideo = HasVideo();
2355 mErrorSink->ResetError();
2356 mCurrentPlayRangeStart = -1.0;
2357 mPlayed = new TimeRanges(ToSupports(OwnerDoc()));
2358 mLoadedDataFired = false;
2359 mCanAutoplayFlag = true;
2360 mIsLoadingFromSourceChildren = false;
2361 mSuspendedAfterFirstFrame = false;
2362 mAllowSuspendAfterFirstFrame = true;
2363 mHaveQueuedSelectResource = false;
2364 mSuspendedForPreloadNone = false;
2365 mDownloadSuspendedByCache = false;
2366 mMediaInfo = MediaInfo();
2367 mIsEncrypted = false;
2368 mPendingEncryptedInitData.Reset();
2369 mWaitingForKey = NOT_WAITING_FOR_KEY;
2370 mSourcePointer = nullptr;
2371 mIsBlessed = false;
2372 SetAudibleState(false);
2374 mTags = nullptr;
2376 if (mNetworkState != NETWORK_EMPTY) {
2377 NS_ASSERTION(!mDecoder && !mSrcStream,
2378 "How did someone setup a new stream/decoder already?");
2380 DispatchAsyncEvent(u"emptied"_ns);
2382 // ChangeNetworkState() will call UpdateAudioChannelPlayingState()
2383 // indirectly which depends on mPaused. So we need to update mPaused first.
2384 if (!mPaused) {
2385 mPaused = true;
2386 PlayPromise::RejectPromises(TakePendingPlayPromises(),
2387 NS_ERROR_DOM_MEDIA_ABORT_ERR);
2389 ChangeNetworkState(NETWORK_EMPTY);
2390 RemoveMediaTracks();
2391 UpdateOutputTrackSources();
2392 ChangeReadyState(HAVE_NOTHING);
2394 // TODO: Apply the rules for text track cue rendering Bug 865407
2395 if (mTextTrackManager) {
2396 mTextTrackManager->GetTextTracks()->SetCuesInactive();
2399 if (fireTimeUpdate) {
2400 // Since we destroyed the decoder above, the current playback position
2401 // will now be reported as 0. The playback position was non-zero when
2402 // we destroyed the decoder, so fire a timeupdate event so that the
2403 // change will be reflected in the controls.
2404 FireTimeUpdate(TimeupdateType::eMandatory);
2406 UpdateAudioChannelPlayingState();
2409 if (IsVideo() && hadVideo) {
2410 // Ensure we render transparent black after resetting video resolution.
2411 Maybe<nsIntSize> size = Some(nsIntSize(0, 0));
2412 Invalidate(ImageSizeChanged::Yes, size, ForceInvalidate::No);
2415 // As aborting current load would stop current playback, so we have no need to
2416 // resume a paused media element.
2417 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
2419 mMediaControlKeyListener->StopIfNeeded();
2421 // We may have changed mPaused, mCanAutoplayFlag, and other
2422 // things which can affect AddRemoveSelfReference
2423 AddRemoveSelfReference();
2425 mIsRunningSelectResource = false;
2427 AssertReadyStateIsNothing();
2430 void HTMLMediaElement::NoSupportedMediaSourceError(
2431 const nsACString& aErrorDetails) {
2432 if (mDecoder) {
2433 ShutdownDecoder();
2436 bool isSameOriginLoad = false;
2437 nsresult rv = NS_ERROR_NOT_AVAILABLE;
2438 if (mSrcAttrTriggeringPrincipal && mLoadingSrc) {
2439 rv = mSrcAttrTriggeringPrincipal->IsSameOrigin(mLoadingSrc,
2440 &isSameOriginLoad);
2443 if (NS_SUCCEEDED(rv) && !isSameOriginLoad) {
2444 // aErrorDetails can include sensitive details like MimeType or HTTP Status
2445 // Code. In case we're loading a 3rd party resource we should not leak this
2446 // and pass a Generic Error Message
2447 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED,
2448 "Failed to open media"_ns);
2449 } else {
2450 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED, aErrorDetails);
2453 RemoveMediaTracks();
2454 ChangeDelayLoadStatus(false);
2455 UpdateAudioChannelPlayingState();
2456 PlayPromise::RejectPromises(TakePendingPlayPromises(),
2457 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
2460 // Runs a "synchronous section", a function that must run once the event loop
2461 // has reached a "stable state"
2462 // http://www.whatwg.org/specs/web-apps/current-work/multipage/webappapis.html#synchronous-section
2463 void HTMLMediaElement::RunInStableState(nsIRunnable* aRunnable) {
2464 if (mShuttingDown) {
2465 return;
2468 nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
2469 "HTMLMediaElement::RunInStableState",
2470 [self = RefPtr<HTMLMediaElement>(this), loadId = GetCurrentLoadID(),
2471 runnable = RefPtr<nsIRunnable>(aRunnable)]() {
2472 if (self->GetCurrentLoadID() != loadId) {
2473 return;
2475 runnable->Run();
2477 nsContentUtils::RunInStableState(task.forget());
2480 void HTMLMediaElement::QueueLoadFromSourceTask() {
2481 if (!mIsLoadingFromSourceChildren || mShuttingDown) {
2482 return;
2485 if (mDecoder) {
2486 // Reset readyState to HAVE_NOTHING since we're going to load a new decoder.
2487 ShutdownDecoder();
2488 ChangeReadyState(HAVE_NOTHING);
2491 AssertReadyStateIsNothing();
2493 ChangeDelayLoadStatus(true);
2494 ChangeNetworkState(NETWORK_LOADING);
2495 RefPtr<Runnable> r =
2496 NewRunnableMethod("HTMLMediaElement::LoadFromSourceChildren", this,
2497 &HTMLMediaElement::LoadFromSourceChildren);
2498 RunInStableState(r);
2501 void HTMLMediaElement::QueueSelectResourceTask() {
2502 // Don't allow multiple async select resource calls to be queued.
2503 if (mHaveQueuedSelectResource) return;
2504 mHaveQueuedSelectResource = true;
2505 ChangeNetworkState(NETWORK_NO_SOURCE);
2506 RefPtr<Runnable> r =
2507 NewRunnableMethod("HTMLMediaElement::SelectResourceWrapper", this,
2508 &HTMLMediaElement::SelectResourceWrapper);
2509 RunInStableState(r);
2512 static bool HasSourceChildren(nsIContent* aElement) {
2513 for (nsIContent* child = aElement->GetFirstChild(); child;
2514 child = child->GetNextSibling()) {
2515 if (child->IsHTMLElement(nsGkAtoms::source)) {
2516 return true;
2519 return false;
2522 static nsCString DocumentOrigin(Document* aDoc) {
2523 if (!aDoc) {
2524 return "null"_ns;
2526 nsCOMPtr<nsIPrincipal> principal = aDoc->NodePrincipal();
2527 if (!principal) {
2528 return "null"_ns;
2530 nsCString origin;
2531 if (NS_FAILED(principal->GetOrigin(origin))) {
2532 return "null"_ns;
2534 return origin;
2537 void HTMLMediaElement::Load() {
2538 LOG(LogLevel::Debug,
2539 ("%p Load() hasSrcAttrStream=%d hasSrcAttr=%d hasSourceChildren=%d "
2540 "handlingInput=%d hasAutoplayAttr=%d AllowedToPlay=%d "
2541 "ownerDoc=%p (%s) ownerDocUserActivated=%d "
2542 "muted=%d volume=%f",
2543 this, !!mSrcAttrStream, HasAttr(nsGkAtoms::src), HasSourceChildren(this),
2544 UserActivation::IsHandlingUserInput(), HasAttr(nsGkAtoms::autoplay),
2545 AllowedToPlay(), OwnerDoc(), DocumentOrigin(OwnerDoc()).get(),
2546 OwnerDoc()->HasBeenUserGestureActivated(), mMuted, mVolume));
2548 if (mIsRunningLoadMethod) {
2549 return;
2552 mIsDoingExplicitLoad = true;
2553 DoLoad();
2556 void HTMLMediaElement::DoLoad() {
2557 // Check if media is allowed for the docshell.
2558 nsCOMPtr<nsIDocShell> docShell = OwnerDoc()->GetDocShell();
2559 if (docShell && !docShell->GetAllowMedia()) {
2560 LOG(LogLevel::Debug, ("%p Media not allowed", this));
2561 return;
2564 if (mIsRunningLoadMethod) {
2565 return;
2568 if (UserActivation::IsHandlingUserInput()) {
2569 // Detect if user has interacted with element so that play will not be
2570 // blocked when initiated by a script. This enables sites to capture user
2571 // intent to play by calling load() in the click handler of a "catalog
2572 // view" of a gallery of videos.
2573 mIsBlessed = true;
2574 // Mark the channel as urgent-start when autoplay so that it will play the
2575 // media from src after loading enough resource.
2576 if (HasAttr(nsGkAtoms::autoplay)) {
2577 mUseUrgentStartForChannel = true;
2581 SetPlayedOrSeeked(false);
2582 mIsRunningLoadMethod = true;
2583 AbortExistingLoads();
2584 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
2585 QueueSelectResourceTask();
2586 ResetState();
2587 mIsRunningLoadMethod = false;
2590 void HTMLMediaElement::ResetState() {
2591 // There might be a pending MediaDecoder::PlaybackPositionChanged() which
2592 // will overwrite |mMediaInfo.mVideo.mDisplay| in UpdateMediaSize() to give
2593 // staled videoWidth and videoHeight. We have to call ForgetElement() here
2594 // such that the staled callbacks won't reach us.
2595 if (mVideoFrameContainer) {
2596 mVideoFrameContainer->ForgetElement();
2597 mVideoFrameContainer = nullptr;
2599 if (mMediaStreamRenderer) {
2600 // mMediaStreamRenderer, has a strong reference to mVideoFrameContainer.
2601 mMediaStreamRenderer->Shutdown();
2602 mMediaStreamRenderer = nullptr;
2604 if (mSecondaryMediaStreamRenderer) {
2605 // mSecondaryMediaStreamRenderer, has a strong reference to
2606 // the secondary VideoFrameContainer.
2607 mSecondaryMediaStreamRenderer->Shutdown();
2608 mSecondaryMediaStreamRenderer = nullptr;
2612 void HTMLMediaElement::SelectResourceWrapper() {
2613 SelectResource();
2614 MaybeBeginCloningVisually();
2615 mIsRunningSelectResource = false;
2616 mHaveQueuedSelectResource = false;
2617 mIsDoingExplicitLoad = false;
2620 void HTMLMediaElement::SelectResource() {
2621 if (!mSrcAttrStream && !HasAttr(nsGkAtoms::src) && !HasSourceChildren(this)) {
2622 // The media element has neither a src attribute nor any source
2623 // element children, abort the load.
2624 ChangeNetworkState(NETWORK_EMPTY);
2625 ChangeDelayLoadStatus(false);
2626 return;
2629 ChangeDelayLoadStatus(true);
2631 ChangeNetworkState(NETWORK_LOADING);
2632 DispatchAsyncEvent(u"loadstart"_ns);
2634 // Delay setting mIsRunningSeletResource until after UpdatePreloadAction
2635 // so that we don't lose our state change by bailing out of the preload
2636 // state update
2637 UpdatePreloadAction();
2638 mIsRunningSelectResource = true;
2640 // If we have a 'src' attribute, use that exclusively.
2641 nsAutoString src;
2642 if (mSrcAttrStream) {
2643 SetupSrcMediaStreamPlayback(mSrcAttrStream);
2644 } else if (GetAttr(nsGkAtoms::src, src)) {
2645 nsCOMPtr<nsIURI> uri;
2646 MediaResult rv = NewURIFromString(src, getter_AddRefs(uri));
2647 if (NS_SUCCEEDED(rv)) {
2648 LOG(LogLevel::Debug, ("%p Trying load from src=%s", this,
2649 NS_ConvertUTF16toUTF8(src).get()));
2650 NS_ASSERTION(
2651 !mIsLoadingFromSourceChildren,
2652 "Should think we're not loading from source children by default");
2654 RemoveMediaElementFromURITable();
2655 if (!mSrcMediaSource) {
2656 mLoadingSrc = uri;
2657 } else {
2658 mLoadingSrc = nullptr;
2660 mLoadingSrcTriggeringPrincipal = mSrcAttrTriggeringPrincipal;
2661 DDLOG(DDLogCategory::Property, "loading_src",
2662 nsCString(NS_ConvertUTF16toUTF8(src)));
2663 bool hadMediaSource = !!mMediaSource;
2664 mMediaSource = mSrcMediaSource;
2665 if (mMediaSource && !hadMediaSource) {
2666 OwnerDoc()->AddMediaElementWithMSE();
2668 DDLINKCHILD("mediasource", mMediaSource.get());
2669 UpdatePreloadAction();
2670 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2671 // preload:none media, suspend the load here before we make any
2672 // network requests.
2673 SuspendLoad();
2674 return;
2677 rv = LoadResource();
2678 if (NS_SUCCEEDED(rv)) {
2679 return;
2681 } else {
2682 AutoTArray<nsString, 1> params = {src};
2683 ReportLoadError("MediaLoadInvalidURI", params);
2684 rv = MediaResult(rv.Code(), "MediaLoadInvalidURI");
2686 // The media element has neither a src attribute nor a source element child:
2687 // set the networkState to NETWORK_EMPTY, and abort these steps; the
2688 // synchronous section ends.
2689 GetMainThreadSerialEventTarget()->Dispatch(NewRunnableMethod<nsCString>(
2690 "HTMLMediaElement::NoSupportedMediaSourceError", this,
2691 &HTMLMediaElement::NoSupportedMediaSourceError, rv.Description()));
2692 } else {
2693 // Otherwise, the source elements will be used.
2694 mIsLoadingFromSourceChildren = true;
2695 LoadFromSourceChildren();
2699 void HTMLMediaElement::NotifyLoadError(const nsACString& aErrorDetails) {
2700 if (!mIsLoadingFromSourceChildren) {
2701 LOG(LogLevel::Debug, ("NotifyLoadError(), no supported media error"));
2702 NoSupportedMediaSourceError(aErrorDetails);
2703 } else if (mSourceLoadCandidate) {
2704 DispatchAsyncSourceError(mSourceLoadCandidate);
2705 QueueLoadFromSourceTask();
2706 } else {
2707 NS_WARNING("Should know the source we were loading from!");
2711 void HTMLMediaElement::NotifyMediaTrackAdded(dom::MediaTrack* aTrack) {
2712 // The set of tracks changed.
2713 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2716 void HTMLMediaElement::NotifyMediaTrackRemoved(dom::MediaTrack* aTrack) {
2717 // The set of tracks changed.
2718 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2721 void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) {
2722 MOZ_ASSERT(aTrack);
2723 if (!aTrack) {
2724 return;
2726 #ifdef DEBUG
2727 nsString id;
2728 aTrack->GetId(id);
2730 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s enabled", this,
2731 aTrack->AsAudioTrack() ? "Audio" : "Video",
2732 NS_ConvertUTF16toUTF8(id).get()));
2733 #endif
2735 MOZ_ASSERT((aTrack->AsAudioTrack() && aTrack->AsAudioTrack()->Enabled()) ||
2736 (aTrack->AsVideoTrack() && aTrack->AsVideoTrack()->Selected()));
2738 if (aTrack->AsAudioTrack()) {
2739 SetMutedInternal(mMuted & ~MUTED_BY_AUDIO_TRACK);
2740 } else if (aTrack->AsVideoTrack()) {
2741 if (!IsVideo()) {
2742 MOZ_ASSERT(false);
2743 return;
2745 mDisableVideo = false;
2746 } else {
2747 MOZ_ASSERT(false, "Unknown track type");
2750 if (mSrcStream) {
2751 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2752 if (mMediaStreamRenderer) {
2753 mMediaStreamRenderer->AddTrack(t->GetAudioStreamTrack());
2755 } else if (VideoTrack* t = aTrack->AsVideoTrack()) {
2756 MOZ_ASSERT(!mSelectedVideoStreamTrack);
2758 mSelectedVideoStreamTrack = t->GetVideoStreamTrack();
2759 mSelectedVideoStreamTrack->AddPrincipalChangeObserver(this);
2760 if (mMediaStreamRenderer) {
2761 mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
2763 if (mSecondaryMediaStreamRenderer) {
2764 mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
2766 if (mMediaInfo.HasVideo()) {
2767 mMediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
2769 nsContentUtils::CombineResourcePrincipals(
2770 &mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal());
2774 // The set of enabled/selected tracks changed.
2775 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2778 void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) {
2779 MOZ_ASSERT(aTrack);
2780 if (!aTrack) {
2781 return;
2784 nsString id;
2785 aTrack->GetId(id);
2787 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", this,
2788 aTrack->AsAudioTrack() ? "Audio" : "Video",
2789 NS_ConvertUTF16toUTF8(id).get()));
2791 MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) &&
2792 (!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected()));
2794 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2795 if (mSrcStream) {
2796 if (mMediaStreamRenderer) {
2797 mMediaStreamRenderer->RemoveTrack(t->GetAudioStreamTrack());
2800 // If we don't have any live tracks, we don't need to mute MediaElement.
2801 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
2802 if (AudioTracks()->Length() > 0) {
2803 bool shouldMute = true;
2804 for (uint32_t i = 0; i < AudioTracks()->Length(); ++i) {
2805 if ((*AudioTracks())[i]->Enabled()) {
2806 shouldMute = false;
2807 break;
2811 if (shouldMute) {
2812 SetMutedInternal(mMuted | MUTED_BY_AUDIO_TRACK);
2815 } else if (aTrack->AsVideoTrack()) {
2816 if (mSrcStream) {
2817 MOZ_DIAGNOSTIC_ASSERT(mSelectedVideoStreamTrack ==
2818 aTrack->AsVideoTrack()->GetVideoStreamTrack());
2819 if (mMediaStreamRenderer) {
2820 mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
2822 if (mSecondaryMediaStreamRenderer) {
2823 mSecondaryMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
2825 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
2826 mSelectedVideoStreamTrack = nullptr;
2830 // The set of enabled/selected tracks changed.
2831 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2834 void HTMLMediaElement::DealWithFailedElement(nsIContent* aSourceElement) {
2835 if (mShuttingDown) {
2836 return;
2839 DispatchAsyncSourceError(aSourceElement);
2840 GetMainThreadSerialEventTarget()->Dispatch(
2841 NewRunnableMethod("HTMLMediaElement::QueueLoadFromSourceTask", this,
2842 &HTMLMediaElement::QueueLoadFromSourceTask));
2845 void HTMLMediaElement::LoadFromSourceChildren() {
2846 NS_ASSERTION(mDelayingLoadEvent,
2847 "Should delay load event (if in document) during load");
2848 NS_ASSERTION(mIsLoadingFromSourceChildren,
2849 "Must remember we're loading from source children");
2851 AddMutationObserverUnlessExists(this);
2853 RemoveMediaTracks();
2855 while (true) {
2856 HTMLSourceElement* child = GetNextSource();
2857 if (!child) {
2858 // Exhausted candidates, wait for more candidates to be appended to
2859 // the media element.
2860 mLoadWaitStatus = WAITING_FOR_SOURCE;
2861 ChangeNetworkState(NETWORK_NO_SOURCE);
2862 ChangeDelayLoadStatus(false);
2863 ReportLoadError("MediaLoadExhaustedCandidates");
2864 return;
2867 // Must have src attribute.
2868 nsAutoString src;
2869 if (!child->GetAttr(nsGkAtoms::src, src)) {
2870 ReportLoadError("MediaLoadSourceMissingSrc");
2871 DealWithFailedElement(child);
2872 return;
2875 // If we have a type attribute, it must be a supported type.
2876 nsAutoString type;
2877 if (child->GetAttr(nsGkAtoms::type, type) && !type.IsEmpty()) {
2878 DecoderDoctorDiagnostics diagnostics;
2879 CanPlayStatus canPlay = GetCanPlay(type, &diagnostics);
2880 diagnostics.StoreFormatDiagnostics(OwnerDoc(), type,
2881 canPlay != CANPLAY_NO, __func__);
2882 if (canPlay == CANPLAY_NO) {
2883 // Check that at least one other source child exists and report that
2884 // we will try to load that one next.
2885 nsIContent* nextChild = mSourcePointer->GetNextSibling();
2886 AutoTArray<nsString, 2> params = {type, src};
2888 while (nextChild) {
2889 if (nextChild && nextChild->IsHTMLElement(nsGkAtoms::source)) {
2890 ReportLoadError("MediaLoadUnsupportedTypeAttributeLoadingNextChild",
2891 params);
2892 break;
2895 nextChild = nextChild->GetNextSibling();
2898 if (!nextChild) {
2899 ReportLoadError("MediaLoadUnsupportedTypeAttribute", params);
2902 DealWithFailedElement(child);
2903 return;
2906 nsAutoString media;
2907 child->GetAttr(nsGkAtoms::media, media);
2908 HTMLSourceElement* childSrc = HTMLSourceElement::FromNode(child);
2909 MOZ_ASSERT(childSrc, "Expect child to be HTMLSourceElement");
2910 if (childSrc && !childSrc->MatchesCurrentMedia()) {
2911 AutoTArray<nsString, 2> params = {media, src};
2912 ReportLoadError("MediaLoadSourceMediaNotMatched", params);
2913 DealWithFailedElement(child);
2914 LOG(LogLevel::Debug,
2915 ("%p Media did not match from <source>=%s type=%s media=%s", this,
2916 NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get(),
2917 NS_ConvertUTF16toUTF8(media).get()));
2918 return;
2920 LOG(LogLevel::Debug,
2921 ("%p Trying load from <source>=%s type=%s media=%s", this,
2922 NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get(),
2923 NS_ConvertUTF16toUTF8(media).get()));
2925 nsCOMPtr<nsIURI> uri;
2926 NewURIFromString(src, getter_AddRefs(uri));
2927 if (!uri) {
2928 AutoTArray<nsString, 1> params = {src};
2929 ReportLoadError("MediaLoadInvalidURI", params);
2930 DealWithFailedElement(child);
2931 return;
2934 RemoveMediaElementFromURITable();
2935 mLoadingSrc = uri;
2936 mLoadingSrcTriggeringPrincipal = child->GetSrcTriggeringPrincipal();
2937 DDLOG(DDLogCategory::Property, "loading_src",
2938 nsCString(NS_ConvertUTF16toUTF8(src)));
2939 bool hadMediaSource = !!mMediaSource;
2940 mMediaSource = child->GetSrcMediaSource();
2941 if (mMediaSource && !hadMediaSource) {
2942 OwnerDoc()->AddMediaElementWithMSE();
2944 DDLINKCHILD("mediasource", mMediaSource.get());
2945 NS_ASSERTION(mNetworkState == NETWORK_LOADING,
2946 "Network state should be loading");
2948 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2949 // preload:none media, suspend the load here before we make any
2950 // network requests.
2951 SuspendLoad();
2952 return;
2955 if (NS_SUCCEEDED(LoadResource())) {
2956 return;
2959 // If we fail to load, loop back and try loading the next resource.
2960 DispatchAsyncSourceError(child);
2962 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
2965 void HTMLMediaElement::SuspendLoad() {
2966 mSuspendedForPreloadNone = true;
2967 ChangeNetworkState(NETWORK_IDLE);
2968 ChangeDelayLoadStatus(false);
2971 void HTMLMediaElement::ResumeLoad(PreloadAction aAction) {
2972 NS_ASSERTION(mSuspendedForPreloadNone,
2973 "Must be halted for preload:none to resume from preload:none "
2974 "suspended load.");
2975 mSuspendedForPreloadNone = false;
2976 mPreloadAction = aAction;
2977 ChangeDelayLoadStatus(true);
2978 ChangeNetworkState(NETWORK_LOADING);
2979 if (!mIsLoadingFromSourceChildren) {
2980 // We were loading from the element's src attribute.
2981 MediaResult rv = LoadResource();
2982 if (NS_FAILED(rv)) {
2983 NoSupportedMediaSourceError(rv.Description());
2985 } else {
2986 // We were loading from a child <source> element. Try to resume the
2987 // load of that child, and if that fails, try the next child.
2988 if (NS_FAILED(LoadResource())) {
2989 LoadFromSourceChildren();
2994 bool HTMLMediaElement::AllowedToPlay() const {
2995 return media::AutoplayPolicy::IsAllowedToPlay(*this);
2998 uint32_t HTMLMediaElement::GetPreloadDefault() const {
2999 if (mMediaSource) {
3000 return HTMLMediaElement::PRELOAD_ATTR_METADATA;
3002 if (OnCellularConnection()) {
3003 return Preferences::GetInt("media.preload.default.cellular",
3004 HTMLMediaElement::PRELOAD_ATTR_NONE);
3006 return Preferences::GetInt("media.preload.default",
3007 HTMLMediaElement::PRELOAD_ATTR_METADATA);
3010 uint32_t HTMLMediaElement::GetPreloadDefaultAuto() const {
3011 if (OnCellularConnection()) {
3012 return Preferences::GetInt("media.preload.auto.cellular",
3013 HTMLMediaElement::PRELOAD_ATTR_METADATA);
3015 return Preferences::GetInt("media.preload.auto",
3016 HTMLMediaElement::PRELOAD_ENOUGH);
3019 void HTMLMediaElement::UpdatePreloadAction() {
3020 PreloadAction nextAction = PRELOAD_UNDEFINED;
3021 // If autoplay is set, or we're playing, we should always preload data,
3022 // as we'll need it to play.
3023 if ((AllowedToPlay() && HasAttr(nsGkAtoms::autoplay)) || !mPaused) {
3024 nextAction = HTMLMediaElement::PRELOAD_ENOUGH;
3025 } else {
3026 // Find the appropriate preload action by looking at the attribute.
3027 const nsAttrValue* val =
3028 mAttrs.GetAttr(nsGkAtoms::preload, kNameSpaceID_None);
3029 // MSE doesn't work if preload is none, so it ignores the pref when src is
3030 // from MSE.
3031 uint32_t preloadDefault = GetPreloadDefault();
3032 uint32_t preloadAuto = GetPreloadDefaultAuto();
3033 if (!val) {
3034 // Attribute is not set. Use the preload action specified by the
3035 // media.preload.default pref, or just preload metadata if not present.
3036 nextAction = static_cast<PreloadAction>(preloadDefault);
3037 } else if (val->Type() == nsAttrValue::eEnum) {
3038 PreloadAttrValue attr =
3039 static_cast<PreloadAttrValue>(val->GetEnumValue());
3040 if (attr == HTMLMediaElement::PRELOAD_ATTR_EMPTY ||
3041 attr == HTMLMediaElement::PRELOAD_ATTR_AUTO) {
3042 nextAction = static_cast<PreloadAction>(preloadAuto);
3043 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_METADATA) {
3044 nextAction = HTMLMediaElement::PRELOAD_METADATA;
3045 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_NONE) {
3046 nextAction = HTMLMediaElement::PRELOAD_NONE;
3048 } else {
3049 // Use the suggested "missing value default" of "metadata", or the value
3050 // specified by the media.preload.default, if present.
3051 nextAction = static_cast<PreloadAction>(preloadDefault);
3055 if (nextAction == HTMLMediaElement::PRELOAD_NONE && mIsDoingExplicitLoad) {
3056 nextAction = HTMLMediaElement::PRELOAD_METADATA;
3059 mPreloadAction = nextAction;
3061 if (nextAction == HTMLMediaElement::PRELOAD_ENOUGH) {
3062 if (mSuspendedForPreloadNone) {
3063 // Our load was previouly suspended due to the media having preload
3064 // value "none". The preload value has changed to preload:auto, so
3065 // resume the load.
3066 ResumeLoad(PRELOAD_ENOUGH);
3067 } else {
3068 // Preload as much of the video as we can, i.e. don't suspend after
3069 // the first frame.
3070 StopSuspendingAfterFirstFrame();
3073 } else if (nextAction == HTMLMediaElement::PRELOAD_METADATA) {
3074 // Ensure that the video can be suspended after first frame.
3075 mAllowSuspendAfterFirstFrame = true;
3076 if (mSuspendedForPreloadNone) {
3077 // Our load was previouly suspended due to the media having preload
3078 // value "none". The preload value has changed to preload:metadata, so
3079 // resume the load. We'll pause the load again after we've read the
3080 // metadata.
3081 ResumeLoad(PRELOAD_METADATA);
3086 MediaResult HTMLMediaElement::LoadResource() {
3087 NS_ASSERTION(mDelayingLoadEvent,
3088 "Should delay load event (if in document) during load");
3090 if (mChannelLoader) {
3091 mChannelLoader->Cancel();
3092 mChannelLoader = nullptr;
3095 // Set the media element's CORS mode only when loading a resource
3096 mCORSMode = AttrValueToCORSMode(GetParsedAttr(nsGkAtoms::crossorigin));
3098 HTMLMediaElement* other = LookupMediaElementURITable(mLoadingSrc);
3099 if (other && other->mDecoder) {
3100 // Clone it.
3101 // TODO: remove the cast by storing ChannelMediaDecoder in the URI table.
3102 nsresult rv = InitializeDecoderAsClone(
3103 static_cast<ChannelMediaDecoder*>(other->mDecoder.get()));
3104 if (NS_SUCCEEDED(rv)) return rv;
3107 if (mMediaSource) {
3108 MediaDecoderInit decoderInit(
3109 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
3110 ClampPlaybackRate(mPlaybackRate),
3111 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
3112 HasAttr(nsGkAtoms::loop),
3113 MediaContainerType(MEDIAMIMETYPE("application/x.mediasource")));
3115 RefPtr<MediaSourceDecoder> decoder = new MediaSourceDecoder(decoderInit);
3116 if (!mMediaSource->Attach(decoder)) {
3117 // TODO: Handle failure: run "If the media data cannot be fetched at
3118 // all, due to network errors, causing the user agent to give up
3119 // trying to fetch the resource" section of resource fetch algorithm.
3120 decoder->Shutdown();
3121 return MediaResult(NS_ERROR_FAILURE, "Failed to attach MediaSource");
3123 ChangeDelayLoadStatus(false);
3124 nsresult rv = decoder->Load(mMediaSource->GetPrincipal());
3125 if (NS_FAILED(rv)) {
3126 decoder->Shutdown();
3127 LOG(LogLevel::Debug,
3128 ("%p Failed to load for decoder %p", this, decoder.get()));
3129 return MediaResult(rv, "Fail to load decoder");
3131 rv = FinishDecoderSetup(decoder);
3132 return MediaResult(rv, "Failed to set up decoder");
3135 AssertReadyStateIsNothing();
3137 RefPtr<ChannelLoader> loader = new ChannelLoader;
3138 nsresult rv = loader->Load(this);
3139 if (NS_SUCCEEDED(rv)) {
3140 mChannelLoader = std::move(loader);
3142 return MediaResult(rv, "Failed to load channel");
3145 nsresult HTMLMediaElement::LoadWithChannel(nsIChannel* aChannel,
3146 nsIStreamListener** aListener) {
3147 NS_ENSURE_ARG_POINTER(aChannel);
3148 NS_ENSURE_ARG_POINTER(aListener);
3150 *aListener = nullptr;
3152 // Make sure we don't reenter during synchronous abort events.
3153 if (mIsRunningLoadMethod) return NS_OK;
3154 mIsRunningLoadMethod = true;
3155 AbortExistingLoads();
3156 mIsRunningLoadMethod = false;
3158 mLoadingSrcTriggeringPrincipal = nullptr;
3159 nsresult rv = aChannel->GetOriginalURI(getter_AddRefs(mLoadingSrc));
3160 NS_ENSURE_SUCCESS(rv, rv);
3162 ChangeDelayLoadStatus(true);
3163 rv = InitializeDecoderForChannel(aChannel, aListener);
3164 if (NS_FAILED(rv)) {
3165 ChangeDelayLoadStatus(false);
3166 return rv;
3169 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
3170 DispatchAsyncEvent(u"loadstart"_ns);
3172 return NS_OK;
3175 bool HTMLMediaElement::Seeking() const {
3176 return mDecoder && mDecoder->IsSeeking();
3179 double HTMLMediaElement::CurrentTime() const {
3180 if (mMediaStreamRenderer) {
3181 return ToMicrosecondResolution(mMediaStreamRenderer->CurrentTime());
3184 if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) {
3185 return std::clamp(mDecoder->GetCurrentTime(), 0.0, mDecoder->GetDuration());
3188 return mDefaultPlaybackStartPosition;
3191 void HTMLMediaElement::FastSeek(double aTime, ErrorResult& aRv) {
3192 LOG(LogLevel::Debug, ("%p FastSeek(%f) called by JS", this, aTime));
3193 Seek(aTime, SeekTarget::PrevSyncPoint, IgnoreErrors());
3196 already_AddRefed<Promise> HTMLMediaElement::SeekToNextFrame(ErrorResult& aRv) {
3197 /* This will cause JIT code to be kept around longer, to help performance
3198 * when using SeekToNextFrame to iterate through every frame of a video.
3200 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
3202 if (win) {
3203 if (JSObject* obj = win->AsGlobal()->GetGlobalJSObject()) {
3204 js::NotifyAnimationActivity(obj);
3208 Seek(CurrentTime(), SeekTarget::NextFrame, aRv);
3209 if (aRv.Failed()) {
3210 return nullptr;
3213 mSeekDOMPromise = CreateDOMPromise(aRv);
3214 if (NS_WARN_IF(aRv.Failed())) {
3215 return nullptr;
3218 return do_AddRef(mSeekDOMPromise);
3221 void HTMLMediaElement::SetCurrentTime(double aCurrentTime, ErrorResult& aRv) {
3222 LOG(LogLevel::Debug,
3223 ("%p SetCurrentTime(%lf) called by JS", this, aCurrentTime));
3224 Seek(aCurrentTime, SeekTarget::Accurate, IgnoreErrors());
3228 * Check if aValue is inside a range of aRanges, and if so returns true
3229 * and puts the range index in aIntervalIndex. If aValue is not
3230 * inside a range, returns false, and aIntervalIndex
3231 * is set to the index of the range which starts immediately after aValue
3232 * (and can be aRanges.Length() if aValue is after the last range).
3234 static bool IsInRanges(TimeRanges& aRanges, double aValue,
3235 uint32_t& aIntervalIndex) {
3236 uint32_t length = aRanges.Length();
3238 for (uint32_t i = 0; i < length; i++) {
3239 double start = aRanges.Start(i);
3240 if (start > aValue) {
3241 aIntervalIndex = i;
3242 return false;
3244 double end = aRanges.End(i);
3245 if (aValue <= end) {
3246 aIntervalIndex = i;
3247 return true;
3250 aIntervalIndex = length;
3251 return false;
3254 void HTMLMediaElement::Seek(double aTime, SeekTarget::Type aSeekType,
3255 ErrorResult& aRv) {
3256 // Note: Seek is called both by synchronous code that expects errors thrown in
3257 // aRv, as well as asynchronous code that expects a promise. Make sure all
3258 // synchronous errors are returned using aRv, not promise rejections.
3260 // aTime should be non-NaN.
3261 MOZ_ASSERT(!std::isnan(aTime));
3263 // Seeking step1, Set the media element's show poster flag to false.
3264 // https://html.spec.whatwg.org/multipage/media.html#dom-media-seek
3265 mShowPoster = false;
3267 // Detect if user has interacted with element by seeking so that
3268 // play will not be blocked when initiated by a script.
3269 if (UserActivation::IsHandlingUserInput()) {
3270 mIsBlessed = true;
3273 StopSuspendingAfterFirstFrame();
3275 if (mSrcAttrStream) {
3276 // do nothing since media streams have an empty Seekable range.
3277 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3278 return;
3281 if (mPlayed && mCurrentPlayRangeStart != -1.0) {
3282 double rangeEndTime = CurrentTime();
3283 LOG(LogLevel::Debug, ("%p Adding \'played\' a range : [%f, %f]", this,
3284 mCurrentPlayRangeStart, rangeEndTime));
3285 // Multiple seek without playing, or seek while playing.
3286 if (mCurrentPlayRangeStart != rangeEndTime) {
3287 // Don't round the left of the interval: it comes from script and needs
3288 // to be exact.
3289 mPlayed->Add(mCurrentPlayRangeStart, rangeEndTime);
3291 // Reset the current played range start time. We'll re-set it once
3292 // the seek completes.
3293 mCurrentPlayRangeStart = -1.0;
3296 if (mReadyState == HAVE_NOTHING) {
3297 mDefaultPlaybackStartPosition = aTime;
3298 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3299 return;
3302 if (!mDecoder) {
3303 // mDecoder must always be set in order to reach this point.
3304 NS_ASSERTION(mDecoder, "SetCurrentTime failed: no decoder");
3305 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3306 return;
3309 // Clamp the seek target to inside the seekable ranges.
3310 media::TimeRanges seekableRanges = mDecoder->GetSeekableTimeRanges();
3311 if (seekableRanges.IsInvalid()) {
3312 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3313 return;
3315 RefPtr<TimeRanges> seekable =
3316 new TimeRanges(ToSupports(OwnerDoc()), seekableRanges);
3317 uint32_t length = seekable->Length();
3318 if (length == 0) {
3319 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3320 return;
3323 // If the position we want to seek to is not in a seekable range, we seek
3324 // to the closest position in the seekable ranges instead. If two positions
3325 // are equally close, we seek to the closest position from the currentTime.
3326 // See seeking spec, point 7 :
3327 // http://www.whatwg.org/specs/web-apps/current-work/multipage/the-video-element.html#seeking
3328 uint32_t range = 0;
3329 bool isInRange = IsInRanges(*seekable, aTime, range);
3330 if (!isInRange) {
3331 if (range == 0) {
3332 // aTime is before the first range in |seekable|, the closest point we can
3333 // seek to is the start of the first range.
3334 aTime = seekable->Start(0);
3335 } else if (range == length) {
3336 // Seek target is after the end last range in seekable data.
3337 // Clamp the seek target to the end of the last seekable range.
3338 aTime = seekable->End(length - 1);
3339 } else {
3340 double leftBound = seekable->End(range - 1);
3341 double rightBound = seekable->Start(range);
3342 double distanceLeft = Abs(leftBound - aTime);
3343 double distanceRight = Abs(rightBound - aTime);
3344 if (distanceLeft == distanceRight) {
3345 double currentTime = CurrentTime();
3346 distanceLeft = Abs(leftBound - currentTime);
3347 distanceRight = Abs(rightBound - currentTime);
3349 aTime = (distanceLeft < distanceRight) ? leftBound : rightBound;
3353 // TODO: The spec requires us to update the current time to reflect the
3354 // actual seek target before beginning the synchronous section, but
3355 // that requires changing all MediaDecoderReaders to support telling
3356 // us the fastSeek target, and it's currently not possible to get
3357 // this information as we don't yet control the demuxer for all
3358 // MediaDecoderReaders.
3360 mPlayingBeforeSeek = IsPotentiallyPlaying();
3362 // The media backend is responsible for dispatching the timeupdate
3363 // event if it changes the playback position as a result of the seek.
3364 LOG(LogLevel::Debug, ("%p SetCurrentTime(%f) starting seek", this, aTime));
3365 mDecoder->Seek(aTime, aSeekType);
3367 // We changed whether we're seeking so we need to AddRemoveSelfReference.
3368 AddRemoveSelfReference();
3371 double HTMLMediaElement::Duration() const {
3372 if (mSrcStream) {
3373 if (mSrcStreamPlaybackEnded) {
3374 return CurrentTime();
3376 return std::numeric_limits<double>::infinity();
3379 if (mDecoder) {
3380 return mDecoder->GetDuration();
3383 return std::numeric_limits<double>::quiet_NaN();
3386 already_AddRefed<TimeRanges> HTMLMediaElement::Seekable() const {
3387 media::TimeRanges seekable =
3388 mDecoder ? mDecoder->GetSeekableTimeRanges() : media::TimeRanges();
3389 RefPtr<TimeRanges> ranges = new TimeRanges(
3390 ToSupports(OwnerDoc()), seekable.ToMicrosecondResolution());
3391 return ranges.forget();
3394 already_AddRefed<TimeRanges> HTMLMediaElement::Played() {
3395 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()));
3397 uint32_t timeRangeCount = 0;
3398 if (mPlayed) {
3399 timeRangeCount = mPlayed->Length();
3401 for (uint32_t i = 0; i < timeRangeCount; i++) {
3402 double begin = mPlayed->Start(i);
3403 double end = mPlayed->End(i);
3404 ranges->Add(begin, end);
3407 if (mCurrentPlayRangeStart != -1.0) {
3408 double now = CurrentTime();
3409 if (mCurrentPlayRangeStart != now) {
3410 // Don't round the left of the interval: it comes from script and needs
3411 // to be exact.
3412 ranges->Add(mCurrentPlayRangeStart, now);
3416 ranges->Normalize();
3417 return ranges.forget();
3420 void HTMLMediaElement::Pause(ErrorResult& aRv) {
3421 LOG(LogLevel::Debug, ("%p Pause() called by JS", this));
3422 if (mNetworkState == NETWORK_EMPTY) {
3423 LOG(LogLevel::Debug, ("Loading due to Pause()"));
3424 DoLoad();
3426 PauseInternal();
3429 void HTMLMediaElement::PauseInternal() {
3430 if (mDecoder && mNetworkState != NETWORK_EMPTY) {
3431 mDecoder->Pause();
3433 bool oldPaused = mPaused;
3434 mPaused = true;
3435 // Step 1,
3436 // https://html.spec.whatwg.org/multipage/media.html#internal-pause-steps
3437 mCanAutoplayFlag = false;
3438 // We changed mPaused and mCanAutoplayFlag which can affect
3439 // AddRemoveSelfReference
3440 AddRemoveSelfReference();
3441 UpdateSrcMediaStreamPlaying();
3442 if (mAudioChannelWrapper) {
3443 mAudioChannelWrapper->NotifyPlayStateChanged();
3446 // We don't need to resume media which is paused explicitly by user.
3447 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
3449 if (!oldPaused) {
3450 FireTimeUpdate(TimeupdateType::eMandatory);
3451 DispatchAsyncEvent(u"pause"_ns);
3452 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_ABORT_ERR);
3456 void HTMLMediaElement::SetVolume(double aVolume, ErrorResult& aRv) {
3457 LOG(LogLevel::Debug, ("%p SetVolume(%f) called by JS", this, aVolume));
3459 if (aVolume < 0.0 || aVolume > 1.0) {
3460 aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR);
3461 return;
3464 if (aVolume == mVolume) return;
3466 mVolume = aVolume;
3468 // Here we want just to update the volume.
3469 SetVolumeInternal();
3471 DispatchAsyncEvent(u"volumechange"_ns);
3473 // We allow inaudible autoplay. But changing our volume may make this
3474 // media audible. So pause if we are no longer supposed to be autoplaying.
3475 PauseIfShouldNotBePlaying();
3478 void HTMLMediaElement::MozGetMetadata(JSContext* aCx,
3479 JS::MutableHandle<JSObject*> aResult,
3480 ErrorResult& aRv) {
3481 if (mReadyState < HAVE_METADATA) {
3482 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3483 return;
3486 JS::Rooted<JSObject*> tags(aCx, JS_NewPlainObject(aCx));
3487 if (!tags) {
3488 aRv.Throw(NS_ERROR_FAILURE);
3489 return;
3491 if (mTags) {
3492 for (const auto& entry : *mTags) {
3493 nsString wideValue;
3494 CopyUTF8toUTF16(entry.GetData(), wideValue);
3495 JS::Rooted<JSString*> string(aCx,
3496 JS_NewUCStringCopyZ(aCx, wideValue.Data()));
3497 if (!string || !JS_DefineProperty(aCx, tags, entry.GetKey().Data(),
3498 string, JSPROP_ENUMERATE)) {
3499 NS_WARNING("couldn't create metadata object!");
3500 aRv.Throw(NS_ERROR_FAILURE);
3501 return;
3506 aResult.set(tags);
3509 void HTMLMediaElement::SetMutedInternal(uint32_t aMuted) {
3510 uint32_t oldMuted = mMuted;
3511 mMuted = aMuted;
3513 if (!!aMuted == !!oldMuted) {
3514 return;
3517 SetVolumeInternal();
3520 void HTMLMediaElement::PauseIfShouldNotBePlaying() {
3521 if (GetPaused()) {
3522 return;
3524 if (!AllowedToPlay()) {
3525 AUTOPLAY_LOG("pause because not allowed to play, element=%p", this);
3526 ErrorResult rv;
3527 Pause(rv);
3531 void HTMLMediaElement::SetVolumeInternal() {
3532 float effectiveVolume = ComputedVolume();
3534 if (mDecoder) {
3535 mDecoder->SetVolume(effectiveVolume);
3536 } else if (mMediaStreamRenderer) {
3537 mMediaStreamRenderer->SetAudioOutputVolume(effectiveVolume);
3540 NotifyAudioPlaybackChanged(
3541 AudioChannelService::AudibleChangedReasons::eVolumeChanged);
3544 void HTMLMediaElement::SetMuted(bool aMuted) {
3545 LOG(LogLevel::Debug, ("%p SetMuted(%d) called by JS", this, aMuted));
3546 if (aMuted == Muted()) {
3547 return;
3550 if (aMuted) {
3551 SetMutedInternal(mMuted | MUTED_BY_CONTENT);
3552 } else {
3553 SetMutedInternal(mMuted & ~MUTED_BY_CONTENT);
3556 DispatchAsyncEvent(u"volumechange"_ns);
3558 // We allow inaudible autoplay. But changing our mute status may make this
3559 // media audible. So pause if we are no longer supposed to be autoplaying.
3560 PauseIfShouldNotBePlaying();
3563 void HTMLMediaElement::GetAllEnabledMediaTracks(
3564 nsTArray<RefPtr<MediaTrack>>& aTracks) {
3565 if (AudioTrackList* tracks = AudioTracks()) {
3566 for (size_t i = 0; i < tracks->Length(); ++i) {
3567 AudioTrack* track = (*tracks)[i];
3568 if (track->Enabled()) {
3569 aTracks.AppendElement(track);
3573 if (IsVideo()) {
3574 if (VideoTrackList* tracks = VideoTracks()) {
3575 for (size_t i = 0; i < tracks->Length(); ++i) {
3576 VideoTrack* track = (*tracks)[i];
3577 if (track->Selected()) {
3578 aTracks.AppendElement(track);
3585 void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) {
3586 for (const auto& entry : mOutputTrackSources.Values()) {
3587 entry->SetEnabled(aEnabled);
3591 HTMLMediaElement::OutputMuteState HTMLMediaElement::OutputTracksMuted() {
3592 return mPaused || mReadyState <= HAVE_CURRENT_DATA ? OutputMuteState::Muted
3593 : OutputMuteState::Unmuted;
3596 void HTMLMediaElement::UpdateOutputTracksMuting() {
3597 for (const auto& entry : mOutputTrackSources.Values()) {
3598 entry->SetMutedByElement(OutputTracksMuted());
3602 void HTMLMediaElement::AddOutputTrackSourceToOutputStream(
3603 MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
3604 AddTrackMode aMode) {
3605 if (aOutputStream.mStream == mSrcStream) {
3606 // Cycle detected. This can happen since tracks are added async.
3607 // We avoid forwarding it to the output here or we'd get into an infloop.
3608 LOG(LogLevel::Warning,
3609 ("NOT adding output track source %p to output stream "
3610 "%p -- cycle detected",
3611 aSource, aOutputStream.mStream.get()));
3612 return;
3615 LOG(LogLevel::Debug, ("Adding output track source %p to output stream %p",
3616 aSource, aOutputStream.mStream.get()));
3618 RefPtr<MediaStreamTrack> domTrack;
3619 if (aSource->Track()->mType == MediaSegment::AUDIO) {
3620 domTrack = new AudioStreamTrack(
3621 aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
3622 MediaStreamTrackState::Live, aSource->Muted());
3623 } else {
3624 domTrack = new VideoStreamTrack(
3625 aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
3626 MediaStreamTrackState::Live, aSource->Muted());
3629 aOutputStream.mLiveTracks.AppendElement(domTrack);
3631 switch (aMode) {
3632 case AddTrackMode::ASYNC:
3633 GetMainThreadSerialEventTarget()->Dispatch(
3634 NewRunnableMethod<StoreRefPtrPassByPtr<MediaStreamTrack>>(
3635 "DOMMediaStream::AddTrackInternal", aOutputStream.mStream,
3636 &DOMMediaStream::AddTrackInternal, domTrack));
3637 break;
3638 case AddTrackMode::SYNC:
3639 aOutputStream.mStream->AddTrackInternal(domTrack);
3640 break;
3641 default:
3642 MOZ_CRASH("Unexpected mode");
3645 LOG(LogLevel::Debug,
3646 ("Created capture %s track %p",
3647 domTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get()));
3650 void HTMLMediaElement::UpdateOutputTrackSources() {
3651 // This updates the track sources in mOutputTrackSources so they're in sync
3652 // with the tracks being currently played, and state saying whether we should
3653 // be capturing tracks. This method is long so here is a breakdown:
3654 // - Figure out the tracks that should be captured
3655 // - Diff those against currently captured tracks (mOutputTrackSources), into
3656 // tracks-to-add, and tracks-to-remove
3657 // - Remove the tracks in tracks-to-remove and dispatch "removetrack" and
3658 // "ended" events for them
3659 // - If playback has ended, or there is no longer a media provider object,
3660 // remove any OutputMediaStreams that have the finish-when-ended flag set
3661 // - Create track sources for, and add to OutputMediaStreams, the tracks in
3662 // tracks-to-add
3664 const bool shouldHaveTrackSources = mTracksCaptured.Ref() &&
3665 !IsPlaybackEnded() &&
3666 mReadyState >= HAVE_METADATA;
3668 // Add track sources for all enabled/selected MediaTracks.
3669 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3670 if (!window) {
3671 return;
3674 if (mDecoder) {
3675 if (!mTracksCaptured.Ref()) {
3676 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::None);
3677 } else if (!AudioTracks() || !VideoTracks() || !shouldHaveTrackSources) {
3678 // We've been unlinked, or tracks are not yet known.
3679 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Halt);
3680 } else {
3681 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Capture,
3682 mTracksCaptured.Ref().get());
3686 // Start with all MediaTracks
3687 AutoTArray<RefPtr<MediaTrack>, 4> mediaTracksToAdd;
3688 if (shouldHaveTrackSources) {
3689 GetAllEnabledMediaTracks(mediaTracksToAdd);
3692 // ...and all MediaElementTrackSources.
3693 auto trackSourcesToRemove =
3694 ToTArray<AutoTArray<nsString, 4>>(mOutputTrackSources.Keys());
3696 // Then work out the differences.
3697 mediaTracksToAdd.RemoveLastElements(
3698 mediaTracksToAdd.end() -
3699 std::remove_if(mediaTracksToAdd.begin(), mediaTracksToAdd.end(),
3700 [this, &trackSourcesToRemove](const auto& track) {
3701 const bool remove =
3702 mOutputTrackSources.GetWeak(track->GetId());
3703 if (remove) {
3704 trackSourcesToRemove.RemoveElement(track->GetId());
3706 return remove;
3707 }));
3709 // First remove stale track sources.
3710 for (const auto& id : trackSourcesToRemove) {
3711 RefPtr<MediaElementTrackSource> source = mOutputTrackSources.GetWeak(id);
3713 LOG(LogLevel::Debug, ("Removing output track source %p for track %s",
3714 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3716 if (mDecoder) {
3717 mDecoder->RemoveOutputTrack(source->Track());
3720 // The source of this track just ended. Force-notify that it ended.
3721 // If we bounce it to the MediaTrackGraph it might not be picked up,
3722 // for instance if the MediaInputPort was destroyed in the same
3723 // iteration as it was added.
3724 GetMainThreadSerialEventTarget()->Dispatch(
3725 NewRunnableMethod("MediaElementTrackSource::OverrideEnded", source,
3726 &MediaElementTrackSource::OverrideEnded));
3728 // Remove the track from the MediaStream after it ended.
3729 for (OutputMediaStream& ms : mOutputStreams) {
3730 if (source->Track()->mType == MediaSegment::VIDEO &&
3731 ms.mCapturingAudioOnly) {
3732 continue;
3734 DebugOnly<size_t> length = ms.mLiveTracks.Length();
3735 ms.mLiveTracks.RemoveElementsBy(
3736 [&](const RefPtr<MediaStreamTrack>& aTrack) {
3737 if (&aTrack->GetSource() != source) {
3738 return false;
3740 GetMainThreadSerialEventTarget()->Dispatch(
3741 NewRunnableMethod<RefPtr<MediaStreamTrack>>(
3742 "DOMMediaStream::RemoveTrackInternal", ms.mStream,
3743 &DOMMediaStream::RemoveTrackInternal, aTrack));
3744 return true;
3746 MOZ_ASSERT(ms.mLiveTracks.Length() == length - 1);
3749 mOutputTrackSources.Remove(id);
3752 // Then update finish-when-ended output streams as needed.
3753 for (size_t i = mOutputStreams.Length(); i-- > 0;) {
3754 if (!mOutputStreams[i].mFinishWhenEnded) {
3755 continue;
3758 if (!mOutputStreams[i].mFinishWhenEndedLoadingSrc &&
3759 !mOutputStreams[i].mFinishWhenEndedAttrStream &&
3760 !mOutputStreams[i].mFinishWhenEndedMediaSource) {
3761 // This finish-when-ended stream has not seen any source loaded yet.
3762 // Update the loading src if it's time.
3763 if (!IsPlaybackEnded()) {
3764 if (mLoadingSrc) {
3765 mOutputStreams[i].mFinishWhenEndedLoadingSrc = mLoadingSrc;
3766 } else if (mSrcAttrStream) {
3767 mOutputStreams[i].mFinishWhenEndedAttrStream = mSrcAttrStream;
3768 } else if (mSrcMediaSource) {
3769 mOutputStreams[i].mFinishWhenEndedMediaSource = mSrcMediaSource;
3772 continue;
3775 // Discard finish-when-ended output streams with a loading src set as
3776 // needed.
3777 if (!IsPlaybackEnded() &&
3778 mLoadingSrc == mOutputStreams[i].mFinishWhenEndedLoadingSrc) {
3779 continue;
3781 if (!IsPlaybackEnded() &&
3782 mSrcAttrStream == mOutputStreams[i].mFinishWhenEndedAttrStream) {
3783 continue;
3785 if (!IsPlaybackEnded() &&
3786 mSrcMediaSource == mOutputStreams[i].mFinishWhenEndedMediaSource) {
3787 continue;
3789 LOG(LogLevel::Debug,
3790 ("Playback ended or source changed. Discarding stream %p",
3791 mOutputStreams[i].mStream.get()));
3792 mOutputStreams.RemoveElementAt(i);
3793 if (mOutputStreams.IsEmpty()) {
3794 mTracksCaptured = nullptr;
3795 // mTracksCaptured is one of the Watchables triggering this method.
3796 // Unsetting it here means we'll run through this method again very soon.
3797 return;
3801 // Finally add new MediaTracks.
3802 for (const auto& mediaTrack : mediaTracksToAdd) {
3803 nsAutoString id;
3804 mediaTrack->GetId(id);
3806 MediaSegment::Type type;
3807 if (mediaTrack->AsAudioTrack()) {
3808 type = MediaSegment::AUDIO;
3809 } else if (mediaTrack->AsVideoTrack()) {
3810 type = MediaSegment::VIDEO;
3811 } else {
3812 MOZ_CRASH("Unknown track type");
3815 RefPtr<ProcessedMediaTrack> track;
3816 RefPtr<MediaElementTrackSource> source;
3817 if (mDecoder) {
3818 track = mTracksCaptured.Ref()->mTrack->Graph()->CreateForwardedInputTrack(
3819 type);
3820 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
3821 if (!principal || IsCORSSameOrigin()) {
3822 principal = NodePrincipal();
3824 source = MakeAndAddRef<MediaElementTrackSource>(
3825 track, principal, OutputTracksMuted(),
3826 type == MediaSegment::VIDEO
3827 ? HTMLVideoElement::FromNode(this)->HasAlpha()
3828 : false);
3829 mDecoder->AddOutputTrack(track);
3830 } else if (mSrcStream) {
3831 MediaStreamTrack* inputTrack;
3832 if (AudioTrack* t = mediaTrack->AsAudioTrack()) {
3833 inputTrack = t->GetAudioStreamTrack();
3834 } else if (VideoTrack* t = mediaTrack->AsVideoTrack()) {
3835 inputTrack = t->GetVideoStreamTrack();
3836 } else {
3837 MOZ_CRASH("Unknown track type");
3839 MOZ_ASSERT(inputTrack);
3840 if (!inputTrack) {
3841 NS_ERROR("Input track not found in source stream");
3842 return;
3844 MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended());
3846 track = inputTrack->Graph()->CreateForwardedInputTrack(type);
3847 RefPtr<MediaInputPort> port = inputTrack->ForwardTrackContentsTo(track);
3848 source = MakeAndAddRef<MediaElementTrackSource>(
3849 inputTrack, &inputTrack->GetSource(), track, port,
3850 OutputTracksMuted());
3852 // Track is muted initially, so we don't leak data if it's added while
3853 // paused and an MTG iteration passes before the mute comes into effect.
3854 source->SetEnabled(mSrcStreamIsPlaying);
3855 } else {
3856 MOZ_CRASH("Unknown source");
3859 LOG(LogLevel::Debug, ("Adding output track source %p for track %s",
3860 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3862 track->QueueSetAutoend(false);
3863 MOZ_DIAGNOSTIC_ASSERT(!mOutputTrackSources.Contains(id));
3864 mOutputTrackSources.InsertOrUpdate(id, RefPtr{source});
3866 // Add the new track source to any existing output streams
3867 for (OutputMediaStream& ms : mOutputStreams) {
3868 if (source->Track()->mType == MediaSegment::VIDEO &&
3869 ms.mCapturingAudioOnly) {
3870 // If the output stream is for audio only we ignore video sources.
3871 continue;
3873 AddOutputTrackSourceToOutputStream(source, ms);
3878 bool HTMLMediaElement::CanBeCaptured(StreamCaptureType aCaptureType) {
3879 // Don't bother capturing when the document has gone away
3880 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3881 if (!window) {
3882 return false;
3885 // Prevent capturing restricted video
3886 if (aCaptureType == StreamCaptureType::CAPTURE_ALL_TRACKS &&
3887 ContainsRestrictedContent()) {
3888 return false;
3890 return true;
3893 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureStreamInternal(
3894 StreamCaptureBehavior aFinishBehavior, StreamCaptureType aStreamCaptureType,
3895 MediaTrackGraph* aGraph) {
3896 MOZ_ASSERT(CanBeCaptured(aStreamCaptureType));
3898 LogVisibility(CallerAPI::CAPTURE_STREAM);
3899 MarkAsTainted();
3901 if (mTracksCaptured.Ref()) {
3902 // Already have an output stream. Check whether the graph rate matches if
3903 // specified.
3904 if (aGraph && aGraph != mTracksCaptured.Ref()->mTrack->Graph()) {
3905 return nullptr;
3907 } else {
3908 // This is the first output stream, or there are no tracks. If the former,
3909 // start capturing all tracks. If the latter, they will be added later.
3910 MediaTrackGraph* graph = aGraph;
3911 if (!graph) {
3912 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3913 if (!window) {
3914 return nullptr;
3917 MediaTrackGraph::GraphDriverType graphDriverType =
3918 HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
3919 : MediaTrackGraph::SYSTEM_THREAD_DRIVER;
3920 graph = MediaTrackGraph::GetInstance(
3921 graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
3922 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
3924 mTracksCaptured = MakeRefPtr<SharedDummyTrack>(
3925 graph->CreateSourceTrack(MediaSegment::AUDIO));
3926 UpdateOutputTrackSources();
3929 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3930 OutputMediaStream* out = mOutputStreams.EmplaceBack(
3931 MakeRefPtr<DOMMediaStream>(window),
3932 aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO,
3933 aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED);
3935 if (aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED &&
3936 !mOutputTrackSources.IsEmpty()) {
3937 // This output stream won't receive any more tracks when playback of the
3938 // current src of this media element ends, or when the src of this media
3939 // element changes. If we're currently playing something (i.e., if there are
3940 // tracks currently captured), set the current src on the output stream so
3941 // this can be tracked. If we're not playing anything,
3942 // UpdateOutputTrackSources will set the current src when it becomes
3943 // available later.
3944 if (mLoadingSrc) {
3945 out->mFinishWhenEndedLoadingSrc = mLoadingSrc;
3947 if (mSrcAttrStream) {
3948 out->mFinishWhenEndedAttrStream = mSrcAttrStream;
3950 if (mSrcMediaSource) {
3951 out->mFinishWhenEndedMediaSource = mSrcMediaSource;
3953 MOZ_ASSERT(out->mFinishWhenEndedLoadingSrc ||
3954 out->mFinishWhenEndedAttrStream ||
3955 out->mFinishWhenEndedMediaSource);
3958 if (aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO) {
3959 if (mSrcStream) {
3960 // We don't support applying volume and mute to the captured stream, when
3961 // capturing a MediaStream.
3962 ReportToConsole(nsIScriptError::errorFlag,
3963 "MediaElementAudioCaptureOfMediaStreamError");
3966 // mAudioCaptured tells the user that the audio played by this media element
3967 // is being routed to the captureStreams *instead* of being played to
3968 // speakers.
3969 mAudioCaptured = true;
3972 for (const RefPtr<MediaElementTrackSource>& source :
3973 mOutputTrackSources.Values()) {
3974 if (source->Track()->mType == MediaSegment::VIDEO) {
3975 // Only add video tracks if we're a video element and the output stream
3976 // wants video.
3977 if (!IsVideo()) {
3978 continue;
3980 if (out->mCapturingAudioOnly) {
3981 continue;
3984 AddOutputTrackSourceToOutputStream(source, *out, AddTrackMode::SYNC);
3987 return do_AddRef(out->mStream);
3990 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureAudio(
3991 ErrorResult& aRv, MediaTrackGraph* aGraph) {
3992 MOZ_RELEASE_ASSERT(aGraph);
3994 if (!CanBeCaptured(StreamCaptureType::CAPTURE_AUDIO)) {
3995 aRv.Throw(NS_ERROR_FAILURE);
3996 return nullptr;
3999 RefPtr<DOMMediaStream> stream =
4000 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
4001 StreamCaptureType::CAPTURE_AUDIO, aGraph);
4002 if (!stream) {
4003 aRv.Throw(NS_ERROR_FAILURE);
4004 return nullptr;
4007 return stream.forget();
4010 RefPtr<GenericNonExclusivePromise> HTMLMediaElement::GetAllowedToPlayPromise() {
4011 MOZ_ASSERT(NS_IsMainThread());
4012 MOZ_ASSERT(!mOutputStreams.IsEmpty(),
4013 "This method should only be called during stream capturing!");
4014 if (AllowedToPlay()) {
4015 AUTOPLAY_LOG("MediaElement %p has allowed to play, resolve promise", this);
4016 return GenericNonExclusivePromise::CreateAndResolve(true, __func__);
4018 AUTOPLAY_LOG("create allow-to-play promise for MediaElement %p", this);
4019 return mAllowedToPlayPromise.Ensure(__func__);
4022 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStream(
4023 ErrorResult& aRv) {
4024 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
4025 aRv.Throw(NS_ERROR_FAILURE);
4026 return nullptr;
4029 RefPtr<DOMMediaStream> stream =
4030 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
4031 StreamCaptureType::CAPTURE_ALL_TRACKS, nullptr);
4032 if (!stream) {
4033 aRv.Throw(NS_ERROR_FAILURE);
4034 return nullptr;
4037 return stream.forget();
4040 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStreamUntilEnded(
4041 ErrorResult& aRv) {
4042 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
4043 aRv.Throw(NS_ERROR_FAILURE);
4044 return nullptr;
4047 RefPtr<DOMMediaStream> stream =
4048 CaptureStreamInternal(StreamCaptureBehavior::FINISH_WHEN_ENDED,
4049 StreamCaptureType::CAPTURE_ALL_TRACKS, nullptr);
4050 if (!stream) {
4051 aRv.Throw(NS_ERROR_FAILURE);
4052 return nullptr;
4055 return stream.forget();
4058 class MediaElementSetForURI : public nsURIHashKey {
4059 public:
4060 explicit MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {}
4061 MediaElementSetForURI(MediaElementSetForURI&& aOther) noexcept
4062 : nsURIHashKey(std::move(aOther)),
4063 mElements(std::move(aOther.mElements)) {}
4064 nsTArray<HTMLMediaElement*> mElements;
4067 using MediaElementURITable = nsTHashtable<MediaElementSetForURI>;
4068 // Elements in this table must have non-null mDecoder and mLoadingSrc, and those
4069 // can't change while the element is in the table. The table is keyed by
4070 // the element's mLoadingSrc. Each entry has a list of all elements with the
4071 // same mLoadingSrc.
4072 static MediaElementURITable* gElementTable;
4074 #ifdef DEBUG
4075 static bool URISafeEquals(nsIURI* a1, nsIURI* a2) {
4076 if (!a1 || !a2) {
4077 // Consider two empty URIs *not* equal!
4078 return false;
4080 bool equal = false;
4081 nsresult rv = a1->Equals(a2, &equal);
4082 return NS_SUCCEEDED(rv) && equal;
4084 // Returns the number of times aElement appears in the media element table
4085 // for aURI. If this returns other than 0 or 1, there's a bug somewhere!
4086 static unsigned MediaElementTableCount(HTMLMediaElement* aElement,
4087 nsIURI* aURI) {
4088 if (!gElementTable || !aElement) {
4089 return 0;
4091 uint32_t uriCount = 0;
4092 uint32_t otherCount = 0;
4093 for (const auto& entry : *gElementTable) {
4094 uint32_t count = 0;
4095 for (const auto& elem : entry.mElements) {
4096 if (elem == aElement) {
4097 count++;
4100 if (URISafeEquals(aURI, entry.GetKey())) {
4101 uriCount = count;
4102 } else {
4103 otherCount += count;
4106 NS_ASSERTION(otherCount == 0, "Should not have entries for unknown URIs");
4107 return uriCount;
4109 #endif
4111 void HTMLMediaElement::AddMediaElementToURITable() {
4112 NS_ASSERTION(mDecoder, "Call this only with decoder Load called");
4113 NS_ASSERTION(
4114 MediaElementTableCount(this, mLoadingSrc) == 0,
4115 "Should not have entry for element in element table before addition");
4116 if (!gElementTable) {
4117 gElementTable = new MediaElementURITable();
4119 MediaElementSetForURI* entry = gElementTable->PutEntry(mLoadingSrc);
4120 entry->mElements.AppendElement(this);
4121 NS_ASSERTION(
4122 MediaElementTableCount(this, mLoadingSrc) == 1,
4123 "Should have a single entry for element in element table after addition");
4126 void HTMLMediaElement::RemoveMediaElementFromURITable() {
4127 if (!mDecoder || !mLoadingSrc || !gElementTable) {
4128 return;
4130 MediaElementSetForURI* entry = gElementTable->GetEntry(mLoadingSrc);
4131 if (!entry) {
4132 return;
4134 entry->mElements.RemoveElement(this);
4135 if (entry->mElements.IsEmpty()) {
4136 gElementTable->RemoveEntry(entry);
4137 if (gElementTable->Count() == 0) {
4138 delete gElementTable;
4139 gElementTable = nullptr;
4142 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
4143 "After remove, should no longer have an entry in element table");
4146 HTMLMediaElement* HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) {
4147 if (!gElementTable) {
4148 return nullptr;
4150 MediaElementSetForURI* entry = gElementTable->GetEntry(aURI);
4151 if (!entry) {
4152 return nullptr;
4154 for (uint32_t i = 0; i < entry->mElements.Length(); ++i) {
4155 HTMLMediaElement* elem = entry->mElements[i];
4156 bool equal;
4157 // Look for elements that have the same principal and CORS mode.
4158 // Ditto for anything else that could cause us to send different headers.
4159 if (NS_SUCCEEDED(elem->NodePrincipal()->Equals(NodePrincipal(), &equal)) &&
4160 equal && elem->mCORSMode == mCORSMode) {
4161 // See SetupDecoder() below. We only add a element to the table when
4162 // mDecoder is a ChannelMediaDecoder.
4163 auto* decoder = static_cast<ChannelMediaDecoder*>(elem->mDecoder.get());
4164 NS_ASSERTION(decoder, "Decoder gone");
4165 if (decoder->CanClone()) {
4166 return elem;
4170 return nullptr;
4173 class HTMLMediaElement::ShutdownObserver : public nsIObserver {
4174 enum class Phase : int8_t { Init, Subscribed, Unsubscribed };
4176 public:
4177 NS_DECL_ISUPPORTS
4179 NS_IMETHOD Observe(nsISupports*, const char* aTopic,
4180 const char16_t*) override {
4181 if (mPhase != Phase::Subscribed) {
4182 // Bail out if we are not subscribed for this might be called even after
4183 // |nsContentUtils::UnregisterShutdownObserver(this)|.
4184 return NS_OK;
4186 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4187 if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
4188 mWeak->NotifyShutdownEvent();
4190 return NS_OK;
4192 void Subscribe(HTMLMediaElement* aPtr) {
4193 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Init);
4194 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4195 mWeak = aPtr;
4196 nsContentUtils::RegisterShutdownObserver(this);
4197 mPhase = Phase::Subscribed;
4199 void Unsubscribe() {
4200 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Subscribed);
4201 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4202 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4203 "ReleaseMediaElement should have been called first");
4204 mWeak = nullptr;
4205 nsContentUtils::UnregisterShutdownObserver(this);
4206 mPhase = Phase::Unsubscribed;
4208 void AddRefMediaElement() {
4209 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4210 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed, "Should only ever AddRef once");
4211 mWeak->AddRef();
4212 mAddRefed = true;
4214 void ReleaseMediaElement() {
4215 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4216 MOZ_DIAGNOSTIC_ASSERT(mAddRefed, "Should only release after AddRef");
4217 mWeak->Release();
4218 mAddRefed = false;
4221 private:
4222 virtual ~ShutdownObserver() {
4223 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Unsubscribed);
4224 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4225 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4226 "ReleaseMediaElement should have been called first");
4228 // Guaranteed to be valid by HTMLMediaElement.
4229 HTMLMediaElement* mWeak = nullptr;
4230 Phase mPhase = Phase::Init;
4231 bool mAddRefed = false;
4234 NS_IMPL_ISUPPORTS(HTMLMediaElement::ShutdownObserver, nsIObserver)
4236 class HTMLMediaElement::TitleChangeObserver final : public nsIObserver {
4237 public:
4238 NS_DECL_ISUPPORTS
4240 explicit TitleChangeObserver(HTMLMediaElement* aElement)
4241 : mElement(aElement) {
4242 MOZ_ASSERT(NS_IsMainThread());
4243 MOZ_ASSERT(aElement);
4246 NS_IMETHOD Observe(nsISupports*, const char* aTopic,
4247 const char16_t*) override {
4248 if (mElement) {
4249 mElement->UpdateStreamName();
4252 return NS_OK;
4255 void Subscribe() {
4256 nsCOMPtr<nsIObserverService> observerService =
4257 mozilla::services::GetObserverService();
4258 if (observerService) {
4259 observerService->AddObserver(this, "document-title-changed", false);
4263 void Unsubscribe() {
4264 nsCOMPtr<nsIObserverService> observerService =
4265 mozilla::services::GetObserverService();
4266 if (observerService) {
4267 observerService->RemoveObserver(this, "document-title-changed");
4271 private:
4272 ~TitleChangeObserver() = default;
4274 WeakPtr<HTMLMediaElement> mElement;
4277 NS_IMPL_ISUPPORTS(HTMLMediaElement::TitleChangeObserver, nsIObserver)
4279 HTMLMediaElement::HTMLMediaElement(
4280 already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo)
4281 : nsGenericHTMLElement(std::move(aNodeInfo)),
4282 mWatchManager(this, AbstractThread::MainThread()),
4283 mShutdownObserver(new ShutdownObserver),
4284 mTitleChangeObserver(new TitleChangeObserver(this)),
4285 mEventBlocker(new EventBlocker(this)),
4286 mPlayed(new TimeRanges(ToSupports(OwnerDoc()))),
4287 mTracksCaptured(nullptr, "HTMLMediaElement::mTracksCaptured"),
4288 mErrorSink(new ErrorSink(this)),
4289 mAudioChannelWrapper(new AudioChannelAgentCallback(this)),
4290 mSink(std::pair(nsString(), RefPtr<AudioDeviceInfo>())),
4291 mShowPoster(IsVideo()),
4292 mMediaControlKeyListener(new MediaControlKeyListener(this)) {
4293 MOZ_ASSERT(GetMainThreadSerialEventTarget());
4294 // Please don't add anything to this constructor or the initialization
4295 // list that can cause AddRef to be called. This prevents subclasses
4296 // from overriding AddRef in a way that works with our refcount
4297 // logging mechanisms. Put these things inside of the ::Init method
4298 // instead.
4301 void HTMLMediaElement::Init() {
4302 MOZ_ASSERT(mRefCnt == 0 && !mRefCnt.IsPurple(),
4303 "HTMLMediaElement::Init called when AddRef has been called "
4304 "at least once already, probably in the constructor. Please "
4305 "see the documentation in the HTMLMediaElement constructor.");
4306 MOZ_ASSERT(!mRefCnt.IsPurple());
4308 mAudioTrackList = new AudioTrackList(OwnerDoc()->GetParentObject(), this);
4309 mVideoTrackList = new VideoTrackList(OwnerDoc()->GetParentObject(), this);
4311 DecoderDoctorLogger::LogConstruction(this);
4313 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock);
4314 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateOutputTracksMuting);
4315 mWatchManager.Watch(
4316 mPaused, &HTMLMediaElement::NotifyMediaControlPlaybackStateChanged);
4317 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTracksMuting);
4319 mWatchManager.Watch(mTracksCaptured,
4320 &HTMLMediaElement::UpdateOutputTrackSources);
4321 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTrackSources);
4323 mWatchManager.Watch(mDownloadSuspendedByCache,
4324 &HTMLMediaElement::UpdateReadyStateInternal);
4325 mWatchManager.Watch(mFirstFrameLoaded,
4326 &HTMLMediaElement::UpdateReadyStateInternal);
4327 mWatchManager.Watch(mSrcStreamPlaybackEnded,
4328 &HTMLMediaElement::UpdateReadyStateInternal);
4330 ErrorResult rv;
4332 double defaultVolume = Preferences::GetFloat("media.default_volume", 1.0);
4333 SetVolume(defaultVolume, rv);
4335 RegisterActivityObserver();
4336 NotifyOwnerDocumentActivityChanged();
4338 // We initialize the MediaShutdownManager as the HTMLMediaElement is always
4339 // constructed on the main thread, and not during stable state.
4340 // (MediaShutdownManager make use of nsIAsyncShutdownClient which is written
4341 // in JS)
4342 MediaShutdownManager::InitStatics();
4344 #if defined(MOZ_WIDGET_ANDROID)
4345 GVAutoplayPermissionRequestor::AskForPermissionIfNeeded(
4346 OwnerDoc()->GetInnerWindow());
4347 #endif
4349 OwnerDoc()->SetDocTreeHadMedia();
4350 mShutdownObserver->Subscribe(this);
4351 mInitialized = true;
4354 HTMLMediaElement::~HTMLMediaElement() {
4355 MOZ_ASSERT(mInitialized,
4356 "HTMLMediaElement must be initialized before it is destroyed.");
4357 NS_ASSERTION(
4358 !mHasSelfReference,
4359 "How can we be destroyed if we're still holding a self reference?");
4361 mWatchManager.Shutdown();
4363 mShutdownObserver->Unsubscribe();
4365 mTitleChangeObserver->Unsubscribe();
4367 if (mVideoFrameContainer) {
4368 mVideoFrameContainer->ForgetElement();
4370 UnregisterActivityObserver();
4372 mSetCDMRequest.DisconnectIfExists();
4373 mAllowedToPlayPromise.RejectIfExists(NS_ERROR_FAILURE, __func__);
4375 if (mDecoder) {
4376 ShutdownDecoder();
4378 if (mProgressTimer) {
4379 StopProgress();
4381 if (mSrcStream) {
4382 EndSrcMediaStreamPlayback();
4385 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
4386 "Destroyed media element should no longer be in element table");
4388 if (mChannelLoader) {
4389 mChannelLoader->Cancel();
4392 if (mAudioChannelWrapper) {
4393 mAudioChannelWrapper->Shutdown();
4394 mAudioChannelWrapper = nullptr;
4397 if (mResumeDelayedPlaybackAgent) {
4398 mResumePlaybackRequest.DisconnectIfExists();
4399 mResumeDelayedPlaybackAgent = nullptr;
4402 mMediaControlKeyListener->StopIfNeeded();
4403 mMediaControlKeyListener = nullptr;
4405 WakeLockRelease();
4407 DecoderDoctorLogger::LogDestruction(this);
4410 void HTMLMediaElement::StopSuspendingAfterFirstFrame() {
4411 mAllowSuspendAfterFirstFrame = false;
4412 if (!mSuspendedAfterFirstFrame) return;
4413 mSuspendedAfterFirstFrame = false;
4414 if (mDecoder) {
4415 mDecoder->Resume();
4419 void HTMLMediaElement::SetPlayedOrSeeked(bool aValue) {
4420 if (aValue == mHasPlayedOrSeeked) {
4421 return;
4424 mHasPlayedOrSeeked = aValue;
4426 // Force a reflow so that the poster frame hides or shows immediately.
4427 nsIFrame* frame = GetPrimaryFrame();
4428 if (!frame) {
4429 return;
4431 frame->PresShell()->FrameNeedsReflow(frame, IntrinsicDirty::FrameAndAncestors,
4432 NS_FRAME_IS_DIRTY);
4435 void HTMLMediaElement::NotifyXPCOMShutdown() { ShutdownDecoder(); }
4437 already_AddRefed<Promise> HTMLMediaElement::Play(ErrorResult& aRv) {
4438 LOG(LogLevel::Debug,
4439 ("%p Play() called by JS readyState=%d", this, mReadyState.Ref()));
4441 // 4.8.12.8
4442 // When the play() method on a media element is invoked, the user agent must
4443 // run the following steps.
4445 RefPtr<PlayPromise> promise = CreatePlayPromise(aRv);
4446 if (NS_WARN_IF(aRv.Failed())) {
4447 return nullptr;
4450 // 4.8.12.8 - Step 1:
4451 // If the media element is not allowed to play, return a promise rejected
4452 // with a "NotAllowedError" DOMException and abort these steps.
4453 // NOTE: we may require requesting permission from the user, so we do the
4454 // "not allowed" check below.
4456 // 4.8.12.8 - Step 2:
4457 // If the media element's error attribute is not null and its code
4458 // attribute has the value MEDIA_ERR_SRC_NOT_SUPPORTED, return a promise
4459 // rejected with a "NotSupportedError" DOMException and abort these steps.
4460 if (GetError() && GetError()->Code() == MEDIA_ERR_SRC_NOT_SUPPORTED) {
4461 LOG(LogLevel::Debug,
4462 ("%p Play() promise rejected because source not supported.", this));
4463 promise->MaybeReject(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
4464 return promise.forget();
4467 // 4.8.12.8 - Step 3:
4468 // Let promise be a new promise and append promise to the list of pending
4469 // play promises.
4470 // Note: Promise appended to list of pending promises as needed below.
4472 if (ShouldBeSuspendedByInactiveDocShell()) {
4473 LOG(LogLevel::Debug, ("%p no allow to play by the docShell for now", this));
4474 mPendingPlayPromises.AppendElement(promise);
4475 return promise.forget();
4478 // We may delay starting playback of a media resource for an unvisited tab
4479 // until it's going to foreground or being resumed by the play tab icon.
4480 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
4481 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
4482 LOG(LogLevel::Debug, ("%p delay Play() call", this));
4483 MaybeDoLoad();
4484 // When play is delayed, save a reference to the promise, and return it.
4485 // The promise will be resolved when we resume play by either the tab is
4486 // brought to the foreground, or the audio tab indicator is clicked.
4487 mPendingPlayPromises.AppendElement(promise);
4488 return promise.forget();
4491 const bool handlingUserInput = UserActivation::IsHandlingUserInput();
4492 mPendingPlayPromises.AppendElement(promise);
4494 if (AllowedToPlay()) {
4495 AUTOPLAY_LOG("allow MediaElement %p to play", this);
4496 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
4497 PlayInternal(handlingUserInput);
4498 UpdateCustomPolicyAfterPlayed();
4499 } else {
4500 AUTOPLAY_LOG("reject MediaElement %p to play", this);
4501 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR);
4503 return promise.forget();
4506 void HTMLMediaElement::DispatchEventsWhenPlayWasNotAllowed() {
4507 if (StaticPrefs::media_autoplay_block_event_enabled()) {
4508 DispatchAsyncEvent(u"blocked"_ns);
4510 DispatchBlockEventForVideoControl();
4511 if (!mHasEverBeenBlockedForAutoplay) {
4512 MaybeNotifyAutoplayBlocked();
4513 ReportToConsole(nsIScriptError::warningFlag, "BlockAutoplayError");
4514 mHasEverBeenBlockedForAutoplay = true;
4518 void HTMLMediaElement::MaybeNotifyAutoplayBlocked() {
4519 // This event is used to notify front-end side that we've blocked autoplay,
4520 // so front-end side should show blocking icon as well.
4521 RefPtr<AsyncEventDispatcher> asyncDispatcher =
4522 new AsyncEventDispatcher(OwnerDoc(), u"GloballyAutoplayBlocked"_ns,
4523 CanBubble::eYes, ChromeOnlyDispatch::eYes);
4524 asyncDispatcher->PostDOMEvent();
4527 void HTMLMediaElement::DispatchBlockEventForVideoControl() {
4528 #if defined(MOZ_WIDGET_ANDROID)
4529 nsVideoFrame* videoFrame = do_QueryFrame(GetPrimaryFrame());
4530 if (!videoFrame || !videoFrame->GetVideoControls()) {
4531 return;
4534 RefPtr<AsyncEventDispatcher> asyncDispatcher = new AsyncEventDispatcher(
4535 videoFrame->GetVideoControls(), u"MozNoControlsBlockedVideo"_ns,
4536 CanBubble::eYes);
4537 asyncDispatcher->PostDOMEvent();
4538 #endif
4541 void HTMLMediaElement::PlayInternal(bool aHandlingUserInput) {
4542 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE) {
4543 // The media load algorithm will be initiated by a user interaction.
4544 // We want to boost the channel priority for better responsiveness.
4545 // Note this must be done before UpdatePreloadAction() which will
4546 // update |mPreloadAction|.
4547 mUseUrgentStartForChannel = true;
4550 StopSuspendingAfterFirstFrame();
4551 SetPlayedOrSeeked(true);
4553 // 4.8.12.8 - Step 4:
4554 // If the media element's networkState attribute has the value NETWORK_EMPTY,
4555 // invoke the media element's resource selection algorithm.
4556 MaybeDoLoad();
4557 if (mSuspendedForPreloadNone) {
4558 ResumeLoad(PRELOAD_ENOUGH);
4561 // 4.8.12.8 - Step 5:
4562 // If the playback has ended and the direction of playback is forwards,
4563 // seek to the earliest possible position of the media resource.
4565 // Even if we just did Load() or ResumeLoad(), we could already have a decoder
4566 // here if we managed to clone an existing decoder.
4567 if (mDecoder) {
4568 if (mDecoder->IsEnded()) {
4569 SetCurrentTime(0);
4571 if (!mSuspendedByInactiveDocOrDocshell) {
4572 mDecoder->Play();
4576 if (mCurrentPlayRangeStart == -1.0) {
4577 mCurrentPlayRangeStart = CurrentTime();
4580 const bool oldPaused = mPaused;
4581 mPaused = false;
4582 // Step 5,
4583 // https://html.spec.whatwg.org/multipage/media.html#internal-play-steps
4584 mCanAutoplayFlag = false;
4586 // We changed mPaused and mCanAutoplayFlag which can affect
4587 // AddRemoveSelfReference and our preload status.
4588 AddRemoveSelfReference();
4589 UpdatePreloadAction();
4590 UpdateSrcMediaStreamPlaying();
4591 StartMediaControlKeyListenerIfNeeded();
4593 // Once play() has been called in a user generated event handler,
4594 // it is allowed to autoplay. Note: we can reach here when not in
4595 // a user generated event handler if our readyState has not yet
4596 // reached HAVE_METADATA.
4597 mIsBlessed |= aHandlingUserInput;
4599 // TODO: If the playback has ended, then the user agent must set
4600 // seek to the effective start.
4602 // 4.8.12.8 - Step 6:
4603 // If the media element's paused attribute is true, run the following steps:
4604 if (oldPaused) {
4605 // 6.1. Change the value of paused to false. (Already done.)
4606 // This step is uplifted because the "block-media-playback" feature needs
4607 // the mPaused to be false before UpdateAudioChannelPlayingState() being
4608 // called.
4610 // 6.2. If the show poster flag is true, set the element's show poster flag
4611 // to false and run the time marches on steps.
4612 if (mShowPoster) {
4613 mShowPoster = false;
4614 if (mTextTrackManager) {
4615 mTextTrackManager->TimeMarchesOn();
4619 // 6.3. Queue a task to fire a simple event named play at the element.
4620 DispatchAsyncEvent(u"play"_ns);
4622 // 6.4. If the media element's readyState attribute has the value
4623 // HAVE_NOTHING, HAVE_METADATA, or HAVE_CURRENT_DATA, queue a task to
4624 // fire a simple event named waiting at the element.
4625 // Otherwise, the media element's readyState attribute has the value
4626 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA: notify about playing for the
4627 // element.
4628 switch (mReadyState) {
4629 case HAVE_NOTHING:
4630 DispatchAsyncEvent(u"waiting"_ns);
4631 break;
4632 case HAVE_METADATA:
4633 case HAVE_CURRENT_DATA:
4634 DispatchAsyncEvent(u"waiting"_ns);
4635 break;
4636 case HAVE_FUTURE_DATA:
4637 case HAVE_ENOUGH_DATA:
4638 NotifyAboutPlaying();
4639 break;
4641 } else if (mReadyState >= HAVE_FUTURE_DATA) {
4642 // 7. Otherwise, if the media element's readyState attribute has the value
4643 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA, take pending play promises and
4644 // queue a task to resolve pending play promises with the result.
4645 AsyncResolvePendingPlayPromises();
4648 // 8. Set the media element's autoplaying flag to false. (Already done.)
4650 // 9. Return promise.
4651 // (Done in caller.)
4654 void HTMLMediaElement::MaybeDoLoad() {
4655 if (mNetworkState == NETWORK_EMPTY) {
4656 DoLoad();
4660 void HTMLMediaElement::UpdateWakeLock() {
4661 MOZ_ASSERT(NS_IsMainThread());
4662 // Ensure we have a wake lock if we're playing audibly. This ensures the
4663 // device doesn't sleep while playing.
4664 bool playing = !mPaused;
4665 bool isAudible = Volume() > 0.0 && !mMuted && mIsAudioTrackAudible;
4666 // WakeLock when playing audible media.
4667 if (playing && isAudible) {
4668 CreateAudioWakeLockIfNeeded();
4669 } else {
4670 ReleaseAudioWakeLockIfExists();
4674 void HTMLMediaElement::CreateAudioWakeLockIfNeeded() {
4675 if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) {
4676 return;
4678 if (!mWakeLock) {
4679 RefPtr<power::PowerManagerService> pmService =
4680 power::PowerManagerService::GetInstance();
4681 NS_ENSURE_TRUE_VOID(pmService);
4683 ErrorResult rv;
4684 mWakeLock = pmService->NewWakeLock(u"audio-playing"_ns,
4685 OwnerDoc()->GetInnerWindow(), rv);
4689 void HTMLMediaElement::ReleaseAudioWakeLockIfExists() {
4690 if (mWakeLock) {
4691 ErrorResult rv;
4692 mWakeLock->Unlock(rv);
4693 rv.SuppressException();
4694 mWakeLock = nullptr;
4698 void HTMLMediaElement::WakeLockRelease() { ReleaseAudioWakeLockIfExists(); }
4700 void HTMLMediaElement::GetEventTargetParent(EventChainPreVisitor& aVisitor) {
4701 if (!this->Controls() || !aVisitor.mEvent->mFlags.mIsTrusted) {
4702 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4703 return;
4706 // We will need to trap pointer, touch, and mouse events within the media
4707 // element, allowing media control exclusive consumption on these events,
4708 // and preventing the content from handling them.
4709 switch (aVisitor.mEvent->mMessage) {
4710 case ePointerDown:
4711 case ePointerUp:
4712 case eTouchEnd:
4713 // Always prevent touchmove captured in video element from being handled by
4714 // content, since we always do that for touchstart.
4715 case eTouchMove:
4716 case eTouchStart:
4717 case eMouseClick:
4718 case eMouseDoubleClick:
4719 case eMouseDown:
4720 case eMouseUp:
4721 aVisitor.mCanHandle = false;
4722 return;
4724 // The *move events however are only comsumed when the range input is being
4725 // dragged.
4726 case ePointerMove:
4727 case eMouseMove: {
4728 nsINode* node =
4729 nsINode::FromEventTargetOrNull(aVisitor.mEvent->mOriginalTarget);
4730 if (MOZ_UNLIKELY(!node)) {
4731 return;
4733 HTMLInputElement* el = nullptr;
4734 if (node->ChromeOnlyAccess()) {
4735 if (node->IsHTMLElement(nsGkAtoms::input)) {
4736 // The node is a <input type="range">
4737 el = static_cast<HTMLInputElement*>(node);
4738 } else if (node->GetParentNode() &&
4739 node->GetParentNode()->IsHTMLElement(nsGkAtoms::input)) {
4740 // The node is a child of <input type="range">
4741 el = static_cast<HTMLInputElement*>(node->GetParentNode());
4744 if (el && el->IsDraggingRange()) {
4745 aVisitor.mCanHandle = false;
4746 return;
4748 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4749 return;
4751 default:
4752 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4753 return;
4757 bool HTMLMediaElement::ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
4758 const nsAString& aValue,
4759 nsIPrincipal* aMaybeScriptedPrincipal,
4760 nsAttrValue& aResult) {
4761 // Mappings from 'preload' attribute strings to an enumeration.
4762 static const nsAttrValue::EnumTable kPreloadTable[] = {
4763 {"", HTMLMediaElement::PRELOAD_ATTR_EMPTY},
4764 {"none", HTMLMediaElement::PRELOAD_ATTR_NONE},
4765 {"metadata", HTMLMediaElement::PRELOAD_ATTR_METADATA},
4766 {"auto", HTMLMediaElement::PRELOAD_ATTR_AUTO},
4767 {nullptr, 0}};
4769 if (aNamespaceID == kNameSpaceID_None) {
4770 if (aAttribute == nsGkAtoms::crossorigin) {
4771 ParseCORSValue(aValue, aResult);
4772 return true;
4774 if (aAttribute == nsGkAtoms::preload) {
4775 return aResult.ParseEnumValue(aValue, kPreloadTable, false);
4779 return nsGenericHTMLElement::ParseAttribute(aNamespaceID, aAttribute, aValue,
4780 aMaybeScriptedPrincipal, aResult);
4783 void HTMLMediaElement::DoneCreatingElement() {
4784 if (HasAttr(nsGkAtoms::muted)) {
4785 mMuted |= MUTED_BY_CONTENT;
4789 bool HTMLMediaElement::IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
4790 int32_t* aTabIndex) {
4791 if (nsGenericHTMLElement::IsHTMLFocusable(aWithMouse, aIsFocusable,
4792 aTabIndex)) {
4793 return true;
4796 *aIsFocusable = true;
4797 return false;
4800 int32_t HTMLMediaElement::TabIndexDefault() { return 0; }
4802 void HTMLMediaElement::AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
4803 const nsAttrValue* aValue,
4804 const nsAttrValue* aOldValue,
4805 nsIPrincipal* aMaybeScriptedPrincipal,
4806 bool aNotify) {
4807 if (aNameSpaceID == kNameSpaceID_None) {
4808 if (aName == nsGkAtoms::src) {
4809 mSrcMediaSource = nullptr;
4810 mSrcAttrTriggeringPrincipal = nsContentUtils::GetAttrTriggeringPrincipal(
4811 this, aValue ? aValue->GetStringValue() : EmptyString(),
4812 aMaybeScriptedPrincipal);
4813 if (aValue) {
4814 nsString srcStr = aValue->GetStringValue();
4815 nsCOMPtr<nsIURI> uri;
4816 NewURIFromString(srcStr, getter_AddRefs(uri));
4817 if (uri && IsMediaSourceURI(uri)) {
4818 nsresult rv = NS_GetSourceForMediaSourceURI(
4819 uri, getter_AddRefs(mSrcMediaSource));
4820 if (NS_FAILED(rv)) {
4821 nsAutoString spec;
4822 GetCurrentSrc(spec);
4823 AutoTArray<nsString, 1> params = {spec};
4824 ReportLoadError("MediaLoadInvalidURI", params);
4828 } else if (aName == nsGkAtoms::autoplay) {
4829 if (aNotify) {
4830 if (aValue) {
4831 StopSuspendingAfterFirstFrame();
4832 CheckAutoplayDataReady();
4834 // This attribute can affect AddRemoveSelfReference
4835 AddRemoveSelfReference();
4836 UpdatePreloadAction();
4838 } else if (aName == nsGkAtoms::preload) {
4839 UpdatePreloadAction();
4840 } else if (aName == nsGkAtoms::loop) {
4841 if (mDecoder) {
4842 mDecoder->SetLooping(!!aValue);
4844 } else if (aName == nsGkAtoms::controls && IsInComposedDoc()) {
4845 NotifyUAWidgetSetupOrChange();
4849 // Since AfterMaybeChangeAttr may call DoLoad, make sure that it is called
4850 // *after* any possible changes to mSrcMediaSource.
4851 if (aValue) {
4852 AfterMaybeChangeAttr(aNameSpaceID, aName, aNotify);
4855 return nsGenericHTMLElement::AfterSetAttr(
4856 aNameSpaceID, aName, aValue, aOldValue, aMaybeScriptedPrincipal, aNotify);
4859 void HTMLMediaElement::OnAttrSetButNotChanged(int32_t aNamespaceID,
4860 nsAtom* aName,
4861 const nsAttrValueOrString& aValue,
4862 bool aNotify) {
4863 AfterMaybeChangeAttr(aNamespaceID, aName, aNotify);
4865 return nsGenericHTMLElement::OnAttrSetButNotChanged(aNamespaceID, aName,
4866 aValue, aNotify);
4869 void HTMLMediaElement::AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName,
4870 bool aNotify) {
4871 if (aNamespaceID == kNameSpaceID_None) {
4872 if (aName == nsGkAtoms::src) {
4873 DoLoad();
4878 nsresult HTMLMediaElement::BindToTree(BindContext& aContext, nsINode& aParent) {
4879 nsresult rv = nsGenericHTMLElement::BindToTree(aContext, aParent);
4881 if (IsInComposedDoc()) {
4882 // Construct Shadow Root so web content can be hidden in the DOM.
4883 AttachAndSetUAShadowRoot();
4885 // The preload action depends on the value of the autoplay attribute.
4886 // It's value may have changed, so update it.
4887 UpdatePreloadAction();
4890 NotifyDecoderActivityChanges();
4891 mMediaControlKeyListener->UpdateOwnerBrowsingContextIfNeeded();
4892 return rv;
4895 void HTMLMediaElement::UnbindFromTree(UnbindContext& aContext) {
4896 mVisibilityState = Visibility::Untracked;
4898 if (IsInComposedDoc()) {
4899 NotifyUAWidgetTeardown();
4902 nsGenericHTMLElement::UnbindFromTree(aContext);
4904 MOZ_ASSERT(IsActuallyInvisible());
4905 NotifyDecoderActivityChanges();
4907 // https://html.spec.whatwg.org/#playing-the-media-resource:remove-an-element-from-a-document
4909 // Dispatch a task to run once we're in a stable state which ensures we're
4910 // paused if we're no longer in a document. Note that we need to dispatch this
4911 // even if there are other tasks in flight for this because these can be
4912 // cancelled if there's a new load.
4914 // FIXME(emilio): Per that spec section, we should only do this if we used to
4915 // be connected, though other browsers match our current behavior...
4917 // Also, https://github.com/whatwg/html/issues/4928
4918 nsCOMPtr<nsIRunnable> task =
4919 NS_NewRunnableFunction("dom::HTMLMediaElement::UnbindFromTree",
4920 [self = RefPtr<HTMLMediaElement>(this)]() {
4921 if (!self->IsInComposedDoc()) {
4922 self->PauseInternal();
4923 self->mMediaControlKeyListener->StopIfNeeded();
4926 RunInStableState(task);
4929 /* static */
4930 CanPlayStatus HTMLMediaElement::GetCanPlay(
4931 const nsAString& aType, DecoderDoctorDiagnostics* aDiagnostics) {
4932 Maybe<MediaContainerType> containerType = MakeMediaContainerType(aType);
4933 if (!containerType) {
4934 return CANPLAY_NO;
4936 CanPlayStatus status =
4937 DecoderTraits::CanHandleContainerType(*containerType, aDiagnostics);
4938 if (status == CANPLAY_YES &&
4939 (*containerType).ExtendedType().Codecs().IsEmpty()) {
4940 // Per spec: 'Generally, a user agent should never return "probably" for a
4941 // type that allows the `codecs` parameter if that parameter is not
4942 // present.' As all our currently-supported types allow for `codecs`, we can
4943 // do this check here.
4944 // TODO: Instead, missing `codecs` should be checked in each decoder's
4945 // `IsSupportedType` call from `CanHandleCodecsType()`.
4946 // See bug 1399023.
4947 return CANPLAY_MAYBE;
4949 return status;
4952 void HTMLMediaElement::CanPlayType(const nsAString& aType, nsAString& aResult) {
4953 DecoderDoctorDiagnostics diagnostics;
4954 CanPlayStatus canPlay = GetCanPlay(aType, &diagnostics);
4955 diagnostics.StoreFormatDiagnostics(OwnerDoc(), aType, canPlay != CANPLAY_NO,
4956 __func__);
4957 switch (canPlay) {
4958 case CANPLAY_NO:
4959 aResult.Truncate();
4960 break;
4961 case CANPLAY_YES:
4962 aResult.AssignLiteral("probably");
4963 break;
4964 case CANPLAY_MAYBE:
4965 aResult.AssignLiteral("maybe");
4966 break;
4967 default:
4968 MOZ_ASSERT_UNREACHABLE("Unexpected case.");
4969 break;
4972 LOG(LogLevel::Debug,
4973 ("%p CanPlayType(%s) = \"%s\"", this, NS_ConvertUTF16toUTF8(aType).get(),
4974 NS_ConvertUTF16toUTF8(aResult).get()));
4977 void HTMLMediaElement::AssertReadyStateIsNothing() {
4978 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
4979 if (mReadyState != HAVE_NOTHING) {
4980 char buf[1024];
4981 SprintfLiteral(buf,
4982 "readyState=%d networkState=%d mLoadWaitStatus=%d "
4983 "mSourceLoadCandidate=%d "
4984 "mIsLoadingFromSourceChildren=%d mPreloadAction=%d "
4985 "mSuspendedForPreloadNone=%d error=%d",
4986 int(mReadyState), int(mNetworkState), int(mLoadWaitStatus),
4987 !!mSourceLoadCandidate, mIsLoadingFromSourceChildren,
4988 int(mPreloadAction), mSuspendedForPreloadNone,
4989 GetError() ? GetError()->Code() : 0);
4990 MOZ_CRASH_UNSAFE_PRINTF("ReadyState should be HAVE_NOTHING! %s", buf);
4992 #endif
4995 nsresult HTMLMediaElement::InitializeDecoderAsClone(
4996 ChannelMediaDecoder* aOriginal) {
4997 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
4998 NS_ASSERTION(mDecoder == nullptr, "Shouldn't have a decoder");
4999 AssertReadyStateIsNothing();
5001 MediaDecoderInit decoderInit(
5002 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
5003 ClampPlaybackRate(mPlaybackRate),
5004 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
5005 HasAttr(nsGkAtoms::loop), aOriginal->ContainerType());
5007 RefPtr<ChannelMediaDecoder> decoder = aOriginal->Clone(decoderInit);
5008 if (!decoder) return NS_ERROR_FAILURE;
5010 LOG(LogLevel::Debug,
5011 ("%p Cloned decoder %p from %p", this, decoder.get(), aOriginal));
5013 return FinishDecoderSetup(decoder);
5016 template <typename DecoderType, typename... LoadArgs>
5017 nsresult HTMLMediaElement::SetupDecoder(DecoderType* aDecoder,
5018 LoadArgs&&... aArgs) {
5019 LOG(LogLevel::Debug, ("%p Created decoder %p for type %s", this, aDecoder,
5020 aDecoder->ContainerType().OriginalString().Data()));
5022 nsresult rv = aDecoder->Load(std::forward<LoadArgs>(aArgs)...);
5023 if (NS_FAILED(rv)) {
5024 aDecoder->Shutdown();
5025 LOG(LogLevel::Debug, ("%p Failed to load for decoder %p", this, aDecoder));
5026 return rv;
5029 rv = FinishDecoderSetup(aDecoder);
5030 // Only ChannelMediaDecoder supports resource cloning.
5031 if (std::is_same_v<DecoderType, ChannelMediaDecoder> && NS_SUCCEEDED(rv)) {
5032 AddMediaElementToURITable();
5033 NS_ASSERTION(
5034 MediaElementTableCount(this, mLoadingSrc) == 1,
5035 "Media element should have single table entry if decode initialized");
5038 return rv;
5041 nsresult HTMLMediaElement::InitializeDecoderForChannel(
5042 nsIChannel* aChannel, nsIStreamListener** aListener) {
5043 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
5044 AssertReadyStateIsNothing();
5046 DecoderDoctorDiagnostics diagnostics;
5048 nsAutoCString mimeType;
5049 aChannel->GetContentType(mimeType);
5050 NS_ASSERTION(!mimeType.IsEmpty(), "We should have the Content-Type.");
5051 NS_ConvertUTF8toUTF16 mimeUTF16(mimeType);
5053 RefPtr<HTMLMediaElement> self = this;
5054 auto reportCanPlay = [&, self](bool aCanPlay) {
5055 diagnostics.StoreFormatDiagnostics(self->OwnerDoc(), mimeUTF16, aCanPlay,
5056 __func__);
5057 if (!aCanPlay) {
5058 nsAutoString src;
5059 self->GetCurrentSrc(src);
5060 AutoTArray<nsString, 2> params = {mimeUTF16, src};
5061 self->ReportLoadError("MediaLoadUnsupportedMimeType", params);
5065 auto onExit = MakeScopeExit([self] {
5066 if (self->mChannelLoader) {
5067 self->mChannelLoader->Done();
5068 self->mChannelLoader = nullptr;
5072 Maybe<MediaContainerType> containerType = MakeMediaContainerType(mimeType);
5073 if (!containerType) {
5074 reportCanPlay(false);
5075 return NS_ERROR_FAILURE;
5078 MediaDecoderInit decoderInit(
5079 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
5080 ClampPlaybackRate(mPlaybackRate),
5081 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
5082 HasAttr(nsGkAtoms::loop), *containerType);
5084 #ifdef MOZ_ANDROID_HLS_SUPPORT
5085 if (HLSDecoder::IsSupportedType(*containerType)) {
5086 RefPtr<HLSDecoder> decoder = HLSDecoder::Create(decoderInit);
5087 if (!decoder) {
5088 reportCanPlay(false);
5089 return NS_ERROR_OUT_OF_MEMORY;
5091 reportCanPlay(true);
5092 return SetupDecoder(decoder.get(), aChannel);
5094 #endif
5096 RefPtr<ChannelMediaDecoder> decoder =
5097 ChannelMediaDecoder::Create(decoderInit, &diagnostics);
5098 if (!decoder) {
5099 reportCanPlay(false);
5100 return NS_ERROR_FAILURE;
5103 reportCanPlay(true);
5104 bool isPrivateBrowsing = NodePrincipal()->GetPrivateBrowsingId() > 0;
5105 return SetupDecoder(decoder.get(), aChannel, isPrivateBrowsing, aListener);
5108 nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) {
5109 ChangeNetworkState(NETWORK_LOADING);
5111 // Set mDecoder now so if methods like GetCurrentSrc get called between
5112 // here and Load(), they work.
5113 SetDecoder(aDecoder);
5115 // Notify the decoder of the initial activity status.
5116 NotifyDecoderActivityChanges();
5118 // Update decoder principal before we start decoding, since it
5119 // can affect how we feed data to MediaStreams
5120 NotifyDecoderPrincipalChanged();
5122 // Set sink device if we have one. Otherwise the default is used.
5123 if (mSink.second) {
5124 mDecoder->SetSink(mSink.second);
5127 if (mMediaKeys) {
5128 if (mMediaKeys->GetCDMProxy()) {
5129 mDecoder->SetCDMProxy(mMediaKeys->GetCDMProxy());
5130 } else {
5131 // CDM must have crashed.
5132 ShutdownDecoder();
5133 return NS_ERROR_FAILURE;
5137 if (mChannelLoader) {
5138 mChannelLoader->Done();
5139 mChannelLoader = nullptr;
5142 // We may want to suspend the new stream now.
5143 // This will also do an AddRemoveSelfReference.
5144 NotifyOwnerDocumentActivityChanged();
5146 if (!mDecoder) {
5147 // NotifyOwnerDocumentActivityChanged may shutdown the decoder if the
5148 // owning document is inactive and we're in the EME case. We could try and
5149 // handle this, but at the time of writing it's a pretty niche case, so just
5150 // bail.
5151 return NS_ERROR_FAILURE;
5154 if (mSuspendedByInactiveDocOrDocshell) {
5155 mDecoder->Suspend();
5158 if (!mPaused) {
5159 SetPlayedOrSeeked(true);
5160 if (!mSuspendedByInactiveDocOrDocshell) {
5161 mDecoder->Play();
5165 MaybeBeginCloningVisually();
5167 return NS_OK;
5170 void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
5171 if (!mSrcStream) {
5172 return;
5175 bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused &&
5176 !mSuspendedByInactiveDocOrDocshell;
5177 if (shouldPlay == mSrcStreamIsPlaying) {
5178 return;
5180 mSrcStreamIsPlaying = shouldPlay;
5182 LOG(LogLevel::Debug,
5183 ("MediaElement %p %s playback of DOMMediaStream %p", this,
5184 shouldPlay ? "Setting up" : "Removing", mSrcStream.get()));
5186 if (shouldPlay) {
5187 mSrcStreamPlaybackEnded = false;
5188 mSrcStreamReportPlaybackEnded = false;
5190 if (mMediaStreamRenderer) {
5191 mMediaStreamRenderer->Start();
5193 if (mSecondaryMediaStreamRenderer) {
5194 mSecondaryMediaStreamRenderer->Start();
5197 SetCapturedOutputStreamsEnabled(true); // Unmute
5198 // If the input is a media stream, we don't check its data and always regard
5199 // it as audible when it's playing.
5200 SetAudibleState(true);
5201 } else {
5202 if (mMediaStreamRenderer) {
5203 mMediaStreamRenderer->Stop();
5205 if (mSecondaryMediaStreamRenderer) {
5206 mSecondaryMediaStreamRenderer->Stop();
5208 SetCapturedOutputStreamsEnabled(false); // Mute
5212 void HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying() {
5213 if (!mMediaStreamRenderer) {
5214 // Notifications are async, the renderer could have been cleared.
5215 return;
5218 mMediaStreamRenderer->SetProgressingCurrentTime(IsPotentiallyPlaying());
5221 void HTMLMediaElement::UpdateSrcStreamTime() {
5222 MOZ_ASSERT(NS_IsMainThread());
5224 if (mSrcStreamPlaybackEnded) {
5225 // We do a separate FireTimeUpdate() when this is set.
5226 return;
5229 FireTimeUpdate(TimeupdateType::ePeriodic);
5232 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) {
5233 NS_ASSERTION(!mSrcStream, "Should have been ended already");
5235 mLoadingSrc = nullptr;
5236 mSrcStream = aStream;
5238 VideoFrameContainer* container = GetVideoFrameContainer();
5239 RefPtr<FirstFrameVideoOutput> firstFrameOutput =
5240 container ? MakeAndAddRef<FirstFrameVideoOutput>(container,
5241 AbstractMainThread())
5242 : nullptr;
5243 mMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
5244 AbstractMainThread(), container, firstFrameOutput, this);
5245 mWatchManager.Watch(mPaused,
5246 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5247 mWatchManager.Watch(mReadyState,
5248 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5249 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5250 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5251 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5252 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5253 mWatchManager.Watch(mMediaStreamRenderer->CurrentGraphTime(),
5254 &HTMLMediaElement::UpdateSrcStreamTime);
5255 SetVolumeInternal();
5256 if (mSink.second) {
5257 mMediaStreamRenderer->SetAudioOutputDevice(mSink.second);
5260 UpdateSrcMediaStreamPlaying();
5261 UpdateSrcStreamPotentiallyPlaying();
5262 mSrcStreamVideoPrincipal = NodePrincipal();
5264 // If we pause this media element, track changes in the underlying stream
5265 // will continue to fire events at this element and alter its track list.
5266 // That's simpler than delaying the events, but probably confusing...
5267 nsTArray<RefPtr<MediaStreamTrack>> tracks;
5268 mSrcStream->GetTracks(tracks);
5269 for (const RefPtr<MediaStreamTrack>& track : tracks) {
5270 NotifyMediaStreamTrackAdded(track);
5273 mMediaStreamTrackListener = MakeUnique<MediaStreamTrackListener>(this);
5274 mSrcStream->RegisterTrackListener(mMediaStreamTrackListener.get());
5276 ChangeNetworkState(NETWORK_IDLE);
5277 ChangeDelayLoadStatus(false);
5279 // FirstFrameLoaded() will be called when the stream has tracks.
5282 void HTMLMediaElement::EndSrcMediaStreamPlayback() {
5283 MOZ_ASSERT(mSrcStream);
5285 UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
5287 if (mSelectedVideoStreamTrack) {
5288 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
5290 mSelectedVideoStreamTrack = nullptr;
5292 MOZ_ASSERT_IF(mSecondaryMediaStreamRenderer,
5293 !mMediaStreamRenderer == !mSecondaryMediaStreamRenderer);
5294 if (mMediaStreamRenderer) {
5295 mWatchManager.Unwatch(mPaused,
5296 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5297 mWatchManager.Unwatch(mReadyState,
5298 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5299 mWatchManager.Unwatch(mSrcStreamPlaybackEnded,
5300 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5301 mWatchManager.Unwatch(
5302 mSrcStreamPlaybackEnded,
5303 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5304 mWatchManager.Unwatch(mMediaStreamRenderer->CurrentGraphTime(),
5305 &HTMLMediaElement::UpdateSrcStreamTime);
5306 mMediaStreamRenderer->Shutdown();
5307 mMediaStreamRenderer = nullptr;
5309 if (mSecondaryMediaStreamRenderer) {
5310 mSecondaryMediaStreamRenderer->Shutdown();
5311 mSecondaryMediaStreamRenderer = nullptr;
5314 mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get());
5315 mMediaStreamTrackListener = nullptr;
5316 mSrcStreamPlaybackEnded = false;
5317 mSrcStreamReportPlaybackEnded = false;
5318 mSrcStreamVideoPrincipal = nullptr;
5320 mSrcStream = nullptr;
5323 static already_AddRefed<AudioTrack> CreateAudioTrack(
5324 AudioStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5325 nsAutoString id;
5326 nsAutoString label;
5327 aStreamTrack->GetId(id);
5328 aStreamTrack->GetLabel(label, CallerType::System);
5330 return MediaTrackList::CreateAudioTrack(aOwnerGlobal, id, u"main"_ns, label,
5331 u""_ns, true, aStreamTrack);
5334 static already_AddRefed<VideoTrack> CreateVideoTrack(
5335 VideoStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5336 nsAutoString id;
5337 nsAutoString label;
5338 aStreamTrack->GetId(id);
5339 aStreamTrack->GetLabel(label, CallerType::System);
5341 return MediaTrackList::CreateVideoTrack(aOwnerGlobal, id, u"main"_ns, label,
5342 u""_ns, aStreamTrack);
5345 void HTMLMediaElement::NotifyMediaStreamTrackAdded(
5346 const RefPtr<MediaStreamTrack>& aTrack) {
5347 MOZ_ASSERT(aTrack);
5349 if (aTrack->Ended()) {
5350 return;
5353 #ifdef DEBUG
5354 nsAutoString id;
5355 aTrack->GetId(id);
5357 LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", this,
5358 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5359 NS_ConvertUTF16toUTF8(id).get()));
5360 #endif
5362 if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) {
5363 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
5364 RefPtr<AudioTrack> audioTrack =
5365 CreateAudioTrack(t, AudioTracks()->GetOwnerGlobal());
5366 AudioTracks()->AddTrack(audioTrack);
5367 } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
5368 // TODO: Fix this per the spec on bug 1273443.
5369 if (!IsVideo()) {
5370 return;
5372 MOZ_DIAGNOSTIC_ASSERT(VideoTracks(), "Element can't have been unlinked");
5373 RefPtr<VideoTrack> videoTrack =
5374 CreateVideoTrack(t, VideoTracks()->GetOwnerGlobal());
5375 VideoTracks()->AddTrack(videoTrack);
5376 // New MediaStreamTrack added, set the new added video track as selected
5377 // video track when there is no selected track.
5378 if (VideoTracks()->SelectedIndex() == -1) {
5379 MOZ_ASSERT(!mSelectedVideoStreamTrack);
5380 videoTrack->SetEnabledInternal(true, dom::MediaTrack::FIRE_NO_EVENTS);
5384 // The set of enabled AudioTracks and selected video track might have changed.
5385 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5386 AbstractThread::DispatchDirectTask(
5387 NewRunnableMethod("HTMLMediaElement::FirstFrameLoaded", this,
5388 &HTMLMediaElement::FirstFrameLoaded));
5391 void HTMLMediaElement::NotifyMediaStreamTrackRemoved(
5392 const RefPtr<MediaStreamTrack>& aTrack) {
5393 MOZ_ASSERT(aTrack);
5395 nsAutoString id;
5396 aTrack->GetId(id);
5398 LOG(LogLevel::Debug, ("%p, Removing %sTrack with id %s", this,
5399 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5400 NS_ConvertUTF16toUTF8(id).get()));
5402 MOZ_DIAGNOSTIC_ASSERT(AudioTracks() && VideoTracks(),
5403 "Element can't have been unlinked");
5404 if (dom::MediaTrack* t = AudioTracks()->GetTrackById(id)) {
5405 AudioTracks()->RemoveTrack(t);
5406 } else if (dom::MediaTrack* t = VideoTracks()->GetTrackById(id)) {
5407 VideoTracks()->RemoveTrack(t);
5408 } else {
5409 NS_ASSERTION(aTrack->AsVideoStreamTrack() && !IsVideo(),
5410 "MediaStreamTrack ended but did not exist in track lists. "
5411 "This is only allowed if a video element ends and we are an "
5412 "audio element.");
5413 return;
5417 void HTMLMediaElement::ProcessMediaFragmentURI() {
5418 if (!mLoadingSrc) {
5419 mFragmentStart = mFragmentEnd = -1.0;
5420 return;
5422 nsMediaFragmentURIParser parser(mLoadingSrc);
5424 if (mDecoder && parser.HasEndTime()) {
5425 mFragmentEnd = parser.GetEndTime();
5428 if (parser.HasStartTime()) {
5429 SetCurrentTime(parser.GetStartTime());
5430 mFragmentStart = parser.GetStartTime();
5434 void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo,
5435 UniquePtr<const MetadataTags> aTags) {
5436 MOZ_ASSERT(NS_IsMainThread());
5438 if (mDecoder) {
5439 ConstructMediaTracks(aInfo);
5442 SetMediaInfo(*aInfo);
5444 mIsEncrypted =
5445 aInfo->IsEncrypted() || mPendingEncryptedInitData.IsEncrypted();
5446 mTags = std::move(aTags);
5447 mLoadedDataFired = false;
5448 ChangeReadyState(HAVE_METADATA);
5450 // Add output tracks synchronously now to be sure they're available in
5451 // "loadedmetadata" event handlers.
5452 UpdateOutputTrackSources();
5454 DispatchAsyncEvent(u"durationchange"_ns);
5455 if (IsVideo() && HasVideo()) {
5456 DispatchAsyncEvent(u"resize"_ns);
5457 Invalidate(ImageSizeChanged::No, Some(mMediaInfo.mVideo.mDisplay),
5458 ForceInvalidate::No);
5460 NS_ASSERTION(!HasVideo() || (mMediaInfo.mVideo.mDisplay.width > 0 &&
5461 mMediaInfo.mVideo.mDisplay.height > 0),
5462 "Video resolution must be known on 'loadedmetadata'");
5463 DispatchAsyncEvent(u"loadedmetadata"_ns);
5465 if (mDecoder && mDecoder->IsTransportSeekable() &&
5466 mDecoder->IsMediaSeekable()) {
5467 ProcessMediaFragmentURI();
5468 mDecoder->SetFragmentEndTime(mFragmentEnd);
5470 if (mIsEncrypted) {
5471 // We only support playback of encrypted content via MSE by default.
5472 if (!mMediaSource && Preferences::GetBool("media.eme.mse-only", true)) {
5473 DecodeError(
5474 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
5475 "Encrypted content not supported outside of MSE"));
5476 return;
5479 // Dispatch a distinct 'encrypted' event for each initData we have.
5480 for (const auto& initData : mPendingEncryptedInitData.mInitDatas) {
5481 DispatchEncrypted(initData.mInitData, initData.mType);
5483 mPendingEncryptedInitData.Reset();
5486 if (IsVideo() && aInfo->HasVideo()) {
5487 // We are a video element playing video so update the screen wakelock
5488 NotifyOwnerDocumentActivityChanged();
5491 if (mDefaultPlaybackStartPosition != 0.0) {
5492 SetCurrentTime(mDefaultPlaybackStartPosition);
5493 mDefaultPlaybackStartPosition = 0.0;
5496 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5499 void HTMLMediaElement::FirstFrameLoaded() {
5500 LOG(LogLevel::Debug,
5501 ("%p, FirstFrameLoaded() mFirstFrameLoaded=%d mWaitingForKey=%d", this,
5502 mFirstFrameLoaded.Ref(), mWaitingForKey));
5504 NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended");
5506 if (!mFirstFrameLoaded) {
5507 mFirstFrameLoaded = true;
5510 ChangeDelayLoadStatus(false);
5512 if (mDecoder && mAllowSuspendAfterFirstFrame && mPaused &&
5513 !HasAttr(nsGkAtoms::autoplay) &&
5514 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA) {
5515 mSuspendedAfterFirstFrame = true;
5516 mDecoder->Suspend();
5520 void HTMLMediaElement::NetworkError(const MediaResult& aError) {
5521 if (mReadyState == HAVE_NOTHING) {
5522 NoSupportedMediaSourceError(aError.Description());
5523 } else {
5524 Error(MEDIA_ERR_NETWORK);
5528 void HTMLMediaElement::DecodeError(const MediaResult& aError) {
5529 nsAutoString src;
5530 GetCurrentSrc(src);
5531 AutoTArray<nsString, 1> params = {src};
5532 ReportLoadError("MediaLoadDecodeError", params);
5534 DecoderDoctorDiagnostics diagnostics;
5535 diagnostics.StoreDecodeError(OwnerDoc(), aError, src, __func__);
5537 if (mIsLoadingFromSourceChildren) {
5538 mErrorSink->ResetError();
5539 if (mSourceLoadCandidate) {
5540 DispatchAsyncSourceError(mSourceLoadCandidate);
5541 QueueLoadFromSourceTask();
5542 } else {
5543 NS_WARNING("Should know the source we were loading from!");
5545 } else if (mReadyState == HAVE_NOTHING) {
5546 NoSupportedMediaSourceError(aError.Description());
5547 } else if (IsCORSSameOrigin()) {
5548 Error(MEDIA_ERR_DECODE, aError.Description());
5549 } else {
5550 Error(MEDIA_ERR_DECODE, "Failed to decode media"_ns);
5554 void HTMLMediaElement::DecodeWarning(const MediaResult& aError) {
5555 nsAutoString src;
5556 GetCurrentSrc(src);
5557 DecoderDoctorDiagnostics diagnostics;
5558 diagnostics.StoreDecodeWarning(OwnerDoc(), aError, src, __func__);
5561 bool HTMLMediaElement::HasError() const { return GetError(); }
5563 void HTMLMediaElement::LoadAborted() { Error(MEDIA_ERR_ABORTED); }
5565 void HTMLMediaElement::Error(uint16_t aErrorCode,
5566 const nsACString& aErrorDetails) {
5567 mErrorSink->SetError(aErrorCode, aErrorDetails);
5568 ChangeDelayLoadStatus(false);
5569 UpdateAudioChannelPlayingState();
5572 void HTMLMediaElement::PlaybackEnded() {
5573 // We changed state which can affect AddRemoveSelfReference
5574 AddRemoveSelfReference();
5576 NS_ASSERTION(!mDecoder || mDecoder->IsEnded(),
5577 "Decoder fired ended, but not in ended state");
5579 // IsPlaybackEnded() became true.
5580 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5582 if (mSrcStream) {
5583 LOG(LogLevel::Debug,
5584 ("%p, got duration by reaching the end of the resource", this));
5585 mSrcStreamPlaybackEnded = true;
5586 DispatchAsyncEvent(u"durationchange"_ns);
5587 } else {
5588 // mediacapture-main:
5589 // Setting the loop attribute has no effect since a MediaStream has no
5590 // defined end and therefore cannot be looped.
5591 if (HasAttr(nsGkAtoms::loop)) {
5592 SetCurrentTime(0);
5593 return;
5597 FireTimeUpdate(TimeupdateType::eMandatory);
5599 if (!mPaused) {
5600 Pause();
5603 if (mSrcStream) {
5604 // A MediaStream that goes from inactive to active shall be eligible for
5605 // autoplay again according to the mediacapture-main spec.
5606 mCanAutoplayFlag = true;
5609 if (StaticPrefs::media_mediacontrol_stopcontrol_aftermediaends()) {
5610 mMediaControlKeyListener->StopIfNeeded();
5612 DispatchAsyncEvent(u"ended"_ns);
5615 void HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded() {
5616 mSrcStreamReportPlaybackEnded = mSrcStreamPlaybackEnded;
5619 void HTMLMediaElement::SeekStarted() { DispatchAsyncEvent(u"seeking"_ns); }
5621 void HTMLMediaElement::SeekCompleted() {
5622 mPlayingBeforeSeek = false;
5623 SetPlayedOrSeeked(true);
5624 if (mTextTrackManager) {
5625 mTextTrackManager->DidSeek();
5627 // https://html.spec.whatwg.org/multipage/media.html#seeking:dom-media-seek
5628 // (Step 16)
5629 // TODO (bug 1688131): run these steps in a stable state.
5630 FireTimeUpdate(TimeupdateType::eMandatory);
5631 DispatchAsyncEvent(u"seeked"_ns);
5632 // We changed whether we're seeking so we need to AddRemoveSelfReference
5633 AddRemoveSelfReference();
5634 if (mCurrentPlayRangeStart == -1.0) {
5635 mCurrentPlayRangeStart = CurrentTime();
5638 if (mSeekDOMPromise) {
5639 AbstractMainThread()->Dispatch(NS_NewRunnableFunction(
5640 __func__, [promise = std::move(mSeekDOMPromise)] {
5641 promise->MaybeResolveWithUndefined();
5642 }));
5644 MOZ_ASSERT(!mSeekDOMPromise);
5647 void HTMLMediaElement::SeekAborted() {
5648 if (mSeekDOMPromise) {
5649 AbstractMainThread()->Dispatch(NS_NewRunnableFunction(
5650 __func__, [promise = std::move(mSeekDOMPromise)] {
5651 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
5652 }));
5654 MOZ_ASSERT(!mSeekDOMPromise);
5657 void HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache) {
5658 LOG(LogLevel::Debug,
5659 ("%p, mDownloadSuspendedByCache=%d", this, aSuspendedByCache));
5660 mDownloadSuspendedByCache = aSuspendedByCache;
5663 void HTMLMediaElement::DownloadSuspended() {
5664 if (mNetworkState == NETWORK_LOADING) {
5665 DispatchAsyncEvent(u"progress"_ns);
5667 ChangeNetworkState(NETWORK_IDLE);
5670 void HTMLMediaElement::DownloadResumed() {
5671 ChangeNetworkState(NETWORK_LOADING);
5674 void HTMLMediaElement::CheckProgress(bool aHaveNewProgress) {
5675 MOZ_ASSERT(NS_IsMainThread());
5676 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5678 TimeStamp now = TimeStamp::NowLoRes();
5680 if (aHaveNewProgress) {
5681 mDataTime = now;
5684 // If this is the first progress, or PROGRESS_MS has passed since the last
5685 // progress event fired and more data has arrived since then, fire a
5686 // progress event.
5687 NS_ASSERTION(
5688 (mProgressTime.IsNull() && !aHaveNewProgress) || !mDataTime.IsNull(),
5689 "null TimeStamp mDataTime should not be used in comparison");
5690 if (mProgressTime.IsNull()
5691 ? aHaveNewProgress
5692 : (now - mProgressTime >=
5693 TimeDuration::FromMilliseconds(PROGRESS_MS) &&
5694 mDataTime > mProgressTime)) {
5695 DispatchAsyncEvent(u"progress"_ns);
5696 // Resolution() ensures that future data will have now > mProgressTime,
5697 // and so will trigger another event. mDataTime is not reset because it
5698 // is still required to detect stalled; it is similarly offset by
5699 // resolution to indicate the new data has not yet arrived.
5700 mProgressTime = now - TimeDuration::Resolution();
5701 if (mDataTime > mProgressTime) {
5702 mDataTime = mProgressTime;
5704 if (!mProgressTimer) {
5705 NS_ASSERTION(aHaveNewProgress,
5706 "timer dispatched when there was no timer");
5707 // Were stalled. Restart timer.
5708 StartProgressTimer();
5709 if (!mLoadedDataFired) {
5710 ChangeDelayLoadStatus(true);
5713 // Download statistics may have been updated, force a recheck of the
5714 // readyState.
5715 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5718 if (now - mDataTime >= TimeDuration::FromMilliseconds(STALL_MS)) {
5719 if (!mMediaSource) {
5720 DispatchAsyncEvent(u"stalled"_ns);
5721 } else {
5722 ChangeDelayLoadStatus(false);
5725 NS_ASSERTION(mProgressTimer, "detected stalled without timer");
5726 // Stop timer events, which prevents repeated stalled events until there
5727 // is more progress.
5728 StopProgress();
5731 AddRemoveSelfReference();
5734 /* static */
5735 void HTMLMediaElement::ProgressTimerCallback(nsITimer* aTimer, void* aClosure) {
5736 auto* decoder = static_cast<HTMLMediaElement*>(aClosure);
5737 decoder->CheckProgress(false);
5740 void HTMLMediaElement::StartProgressTimer() {
5741 MOZ_ASSERT(NS_IsMainThread());
5742 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5743 NS_ASSERTION(!mProgressTimer, "Already started progress timer.");
5745 NS_NewTimerWithFuncCallback(
5746 getter_AddRefs(mProgressTimer), ProgressTimerCallback, this, PROGRESS_MS,
5747 nsITimer::TYPE_REPEATING_SLACK, "HTMLMediaElement::ProgressTimerCallback",
5748 GetMainThreadSerialEventTarget());
5751 void HTMLMediaElement::StartProgress() {
5752 // Record the time now for detecting stalled.
5753 mDataTime = TimeStamp::NowLoRes();
5754 // Reset mProgressTime so that mDataTime is not indicating bytes received
5755 // after the last progress event.
5756 mProgressTime = TimeStamp();
5757 StartProgressTimer();
5760 void HTMLMediaElement::StopProgress() {
5761 MOZ_ASSERT(NS_IsMainThread());
5762 if (!mProgressTimer) {
5763 return;
5766 mProgressTimer->Cancel();
5767 mProgressTimer = nullptr;
5770 void HTMLMediaElement::DownloadProgressed() {
5771 if (mNetworkState != NETWORK_LOADING) {
5772 return;
5774 CheckProgress(true);
5777 bool HTMLMediaElement::ShouldCheckAllowOrigin() {
5778 return mCORSMode != CORS_NONE;
5781 bool HTMLMediaElement::IsCORSSameOrigin() {
5782 bool subsumes;
5783 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
5784 return (NS_SUCCEEDED(NodePrincipal()->Subsumes(principal, &subsumes)) &&
5785 subsumes) ||
5786 ShouldCheckAllowOrigin();
5789 void HTMLMediaElement::UpdateReadyStateInternal() {
5790 if (!mDecoder && !mSrcStream) {
5791 // Not initialized - bail out.
5792 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5793 "Not initialized",
5794 this));
5795 return;
5798 if (mDecoder && mReadyState < HAVE_METADATA) {
5799 // aNextFrame might have a next frame because the decoder can advance
5800 // on its own thread before MetadataLoaded gets a chance to run.
5801 // The arrival of more data can't change us out of this readyState.
5802 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5803 "Decoder ready state < HAVE_METADATA",
5804 this));
5805 return;
5808 if (mDecoder) {
5809 // IsPlaybackEnded() might have become false.
5810 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5813 if (mSrcStream && mReadyState < HAVE_METADATA) {
5814 bool hasAudioTracks = AudioTracks() && !AudioTracks()->IsEmpty();
5815 bool hasVideoTracks = VideoTracks() && !VideoTracks()->IsEmpty();
5816 if (!hasAudioTracks && !hasVideoTracks) {
5817 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5818 "Stream with no tracks",
5819 this));
5820 // Give it one last chance to remove the self reference if needed.
5821 AddRemoveSelfReference();
5822 return;
5825 if (IsVideo() && hasVideoTracks && !HasVideo()) {
5826 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5827 "Stream waiting for video",
5828 this));
5829 return;
5832 LOG(LogLevel::Debug,
5833 ("MediaElement %p UpdateReadyStateInternal() Stream has "
5834 "metadata; audioTracks=%d, videoTracks=%d, "
5835 "hasVideoFrame=%d",
5836 this, AudioTracks()->Length(), VideoTracks()->Length(), HasVideo()));
5838 // We are playing a stream that has video and a video frame is now set.
5839 // This means we have all metadata needed to change ready state.
5840 MediaInfo mediaInfo = mMediaInfo;
5841 if (hasAudioTracks) {
5842 mediaInfo.EnableAudio();
5844 if (hasVideoTracks) {
5845 mediaInfo.EnableVideo();
5846 if (mSelectedVideoStreamTrack) {
5847 mediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
5850 MetadataLoaded(&mediaInfo, nullptr);
5853 if (mMediaSource) {
5854 // readyState has changed, assuming it's following the pending mediasource
5855 // operations. Notify the Mediasource that the operations have completed.
5856 mMediaSource->CompletePendingTransactions();
5859 enum NextFrameStatus nextFrameStatus = NextFrameStatus();
5860 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
5861 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE && mDecoder &&
5862 !mDecoder->IsEnded()) {
5863 nextFrameStatus = mDecoder->NextFrameBufferedStatus();
5865 } else if (mWaitingForKey == WAITING_FOR_KEY) {
5866 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE ||
5867 nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5868 // http://w3c.github.io/encrypted-media/#wait-for-key
5869 // Continuing 7.3.4 Queue a "waitingforkey" Event
5870 // 4. Queue a task to fire a simple event named waitingforkey
5871 // at the media element.
5872 // 5. Set the readyState of media element to HAVE_METADATA.
5873 // NOTE: We'll change to HAVE_CURRENT_DATA or HAVE_METADATA
5874 // depending on whether we've loaded the first frame or not
5875 // below.
5876 // 6. Suspend playback.
5877 // Note: Playback will already be stalled, as the next frame is
5878 // unavailable.
5879 mWaitingForKey = WAITING_FOR_KEY_DISPATCHED;
5880 DispatchAsyncEvent(u"waitingforkey"_ns);
5882 } else {
5883 MOZ_ASSERT(mWaitingForKey == WAITING_FOR_KEY_DISPATCHED);
5884 if (nextFrameStatus == NEXT_FRAME_AVAILABLE) {
5885 // We have new frames after dispatching "waitingforkey".
5886 // This means we've got the key and can reset mWaitingForKey now.
5887 mWaitingForKey = NOT_WAITING_FOR_KEY;
5891 if (nextFrameStatus == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING) {
5892 LOG(LogLevel::Debug,
5893 ("MediaElement %p UpdateReadyStateInternal() "
5894 "NEXT_FRAME_UNAVAILABLE_SEEKING; Forcing HAVE_METADATA",
5895 this));
5896 ChangeReadyState(HAVE_METADATA);
5897 return;
5900 if (IsVideo() && VideoTracks() && !VideoTracks()->IsEmpty() &&
5901 !IsPlaybackEnded() && GetImageContainer() &&
5902 !GetImageContainer()->HasCurrentImage()) {
5903 // Don't advance if we are playing video, but don't have a video frame.
5904 // Also, if video became available after advancing to HAVE_CURRENT_DATA
5905 // while we are still playing, we need to revert to HAVE_METADATA until
5906 // a video frame is available.
5907 LOG(LogLevel::Debug,
5908 ("MediaElement %p UpdateReadyStateInternal() "
5909 "Playing video but no video frame; Forcing HAVE_METADATA",
5910 this));
5911 ChangeReadyState(HAVE_METADATA);
5912 return;
5915 if (!mFirstFrameLoaded) {
5916 // We haven't yet loaded the first frame, making us unable to determine
5917 // if we have enough valid data at the present stage.
5918 return;
5921 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5922 // Force HAVE_CURRENT_DATA when buffering.
5923 ChangeReadyState(HAVE_CURRENT_DATA);
5924 return;
5927 // TextTracks must be loaded for the HAVE_ENOUGH_DATA and
5928 // HAVE_FUTURE_DATA.
5929 // So force HAVE_CURRENT_DATA if text tracks not loaded.
5930 if (mTextTrackManager && !mTextTrackManager->IsLoaded()) {
5931 ChangeReadyState(HAVE_CURRENT_DATA);
5932 return;
5935 if (mDownloadSuspendedByCache && mDecoder && !mDecoder->IsEnded()) {
5936 // The decoder has signaled that the download has been suspended by the
5937 // media cache. So move readyState into HAVE_ENOUGH_DATA, in case there's
5938 // script waiting for a "canplaythrough" event; without this forced
5939 // transition, we will never fire the "canplaythrough" event if the
5940 // media cache is too small, and scripts are bound to fail. Don't force
5941 // this transition if the decoder is in ended state; the readyState
5942 // should remain at HAVE_CURRENT_DATA in this case.
5943 // Note that this state transition includes the case where we finished
5944 // downloaded the whole data stream.
5945 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5946 "Decoder download suspended by cache",
5947 this));
5948 ChangeReadyState(HAVE_ENOUGH_DATA);
5949 return;
5952 if (nextFrameStatus != MediaDecoderOwner::NEXT_FRAME_AVAILABLE) {
5953 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5954 "Next frame not available",
5955 this));
5956 ChangeReadyState(HAVE_CURRENT_DATA);
5957 return;
5960 if (mSrcStream) {
5961 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5962 "Stream HAVE_ENOUGH_DATA",
5963 this));
5964 ChangeReadyState(HAVE_ENOUGH_DATA);
5965 return;
5968 // Now see if we should set HAVE_ENOUGH_DATA.
5969 // If it's something we don't know the size of, then we can't
5970 // make a real estimate, so we go straight to HAVE_ENOUGH_DATA once
5971 // we've downloaded enough data that our download rate is considered
5972 // reliable. We have to move to HAVE_ENOUGH_DATA at some point or
5973 // autoplay elements for live streams will never play. Otherwise we
5974 // move to HAVE_ENOUGH_DATA if we can play through the entire media
5975 // without stopping to buffer.
5976 if (mDecoder->CanPlayThrough()) {
5977 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5978 "Decoder can play through",
5979 this));
5980 ChangeReadyState(HAVE_ENOUGH_DATA);
5981 return;
5983 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5984 "Default; Decoder has future data",
5985 this));
5986 ChangeReadyState(HAVE_FUTURE_DATA);
5989 static const char* const gReadyStateToString[] = {
5990 "HAVE_NOTHING", "HAVE_METADATA", "HAVE_CURRENT_DATA", "HAVE_FUTURE_DATA",
5991 "HAVE_ENOUGH_DATA"};
5993 void HTMLMediaElement::ChangeReadyState(nsMediaReadyState aState) {
5994 if (mReadyState == aState) {
5995 return;
5998 nsMediaReadyState oldState = mReadyState;
5999 mReadyState = aState;
6000 LOG(LogLevel::Debug,
6001 ("%p Ready state changed to %s", this, gReadyStateToString[aState]));
6003 DDLOG(DDLogCategory::Property, "ready_state", gReadyStateToString[aState]);
6005 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
6006 // The user agent must synchronously unset cues' active flag whenever the
6007 // media element's readyState is changed back to HAVE_NOTHING.
6008 if (mReadyState == HAVE_NOTHING && mTextTrackManager) {
6009 mTextTrackManager->NotifyReset();
6012 if (mNetworkState == NETWORK_EMPTY) {
6013 return;
6016 UpdateAudioChannelPlayingState();
6018 // Handle raising of "waiting" event during seek (see 4.8.10.9)
6019 // or
6020 // 4.8.12.7 Ready states:
6021 // "If the previous ready state was HAVE_FUTURE_DATA or more, and the new
6022 // ready state is HAVE_CURRENT_DATA or less
6023 // If the media element was potentially playing before its readyState
6024 // attribute changed to a value lower than HAVE_FUTURE_DATA, and the element
6025 // has not ended playback, and playback has not stopped due to errors,
6026 // paused for user interaction, or paused for in-band content, the user agent
6027 // must queue a task to fire a simple event named timeupdate at the element,
6028 // and queue a task to fire a simple event named waiting at the element."
6029 if (mPlayingBeforeSeek && mReadyState < HAVE_FUTURE_DATA) {
6030 DispatchAsyncEvent(u"waiting"_ns);
6031 } else if (oldState >= HAVE_FUTURE_DATA && mReadyState < HAVE_FUTURE_DATA &&
6032 !Paused() && !Ended() && !mErrorSink->mError) {
6033 FireTimeUpdate(TimeupdateType::eMandatory);
6034 DispatchAsyncEvent(u"waiting"_ns);
6037 if (oldState < HAVE_CURRENT_DATA && mReadyState >= HAVE_CURRENT_DATA &&
6038 !mLoadedDataFired) {
6039 DispatchAsyncEvent(u"loadeddata"_ns);
6040 mLoadedDataFired = true;
6043 if (oldState < HAVE_FUTURE_DATA && mReadyState >= HAVE_FUTURE_DATA) {
6044 DispatchAsyncEvent(u"canplay"_ns);
6045 if (!mPaused) {
6046 if (mDecoder && !mSuspendedByInactiveDocOrDocshell) {
6047 MOZ_ASSERT(AllowedToPlay());
6048 mDecoder->Play();
6050 NotifyAboutPlaying();
6054 CheckAutoplayDataReady();
6056 if (oldState < HAVE_ENOUGH_DATA && mReadyState >= HAVE_ENOUGH_DATA) {
6057 DispatchAsyncEvent(u"canplaythrough"_ns);
6061 static const char* const gNetworkStateToString[] = {"EMPTY", "IDLE", "LOADING",
6062 "NO_SOURCE"};
6064 void HTMLMediaElement::ChangeNetworkState(nsMediaNetworkState aState) {
6065 if (mNetworkState == aState) {
6066 return;
6069 nsMediaNetworkState oldState = mNetworkState;
6070 mNetworkState = aState;
6071 LOG(LogLevel::Debug,
6072 ("%p Network state changed to %s", this, gNetworkStateToString[aState]));
6073 DDLOG(DDLogCategory::Property, "network_state",
6074 gNetworkStateToString[aState]);
6076 if (oldState == NETWORK_LOADING) {
6077 // Stop progress notification when exiting NETWORK_LOADING.
6078 StopProgress();
6081 if (mNetworkState == NETWORK_LOADING) {
6082 // Start progress notification when entering NETWORK_LOADING.
6083 StartProgress();
6084 } else if (mNetworkState == NETWORK_IDLE && !mErrorSink->mError) {
6085 // Fire 'suspend' event when entering NETWORK_IDLE and no error presented.
6086 DispatchAsyncEvent(u"suspend"_ns);
6089 // According to the resource selection (step2, step9-18), dedicated media
6090 // source failure step (step4) and aborting existing load (step4), set show
6091 // poster flag to true. https://html.spec.whatwg.org/multipage/media.html
6092 if (mNetworkState == NETWORK_NO_SOURCE || mNetworkState == NETWORK_EMPTY) {
6093 mShowPoster = true;
6096 // Changing mNetworkState affects AddRemoveSelfReference().
6097 AddRemoveSelfReference();
6100 bool HTMLMediaElement::IsEligibleForAutoplay() {
6101 // We also activate autoplay when playing a media source since the data
6102 // download is controlled by the script and there is no way to evaluate
6103 // MediaDecoder::CanPlayThrough().
6105 if (!HasAttr(nsGkAtoms::autoplay)) {
6106 return false;
6109 if (!mCanAutoplayFlag) {
6110 return false;
6113 if (IsEditable()) {
6114 return false;
6117 if (!mPaused) {
6118 return false;
6121 if (mSuspendedByInactiveDocOrDocshell) {
6122 return false;
6125 // Static document is used for print preview and printing, should not be
6126 // autoplay
6127 if (OwnerDoc()->IsStaticDocument()) {
6128 return false;
6131 if (ShouldBeSuspendedByInactiveDocShell()) {
6132 LOG(LogLevel::Debug, ("%p prohibiting autoplay by the docShell", this));
6133 return false;
6136 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
6137 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
6138 LOG(LogLevel::Debug, ("%p delay playing from autoplay", this));
6139 return false;
6142 return mReadyState >= HAVE_ENOUGH_DATA;
6145 void HTMLMediaElement::CheckAutoplayDataReady() {
6146 if (!IsEligibleForAutoplay()) {
6147 return;
6149 if (!AllowedToPlay()) {
6150 DispatchEventsWhenPlayWasNotAllowed();
6151 return;
6153 RunAutoplay();
6156 void HTMLMediaElement::RunAutoplay() {
6157 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
6158 mPaused = false;
6159 // We changed mPaused which can affect AddRemoveSelfReference
6160 AddRemoveSelfReference();
6161 UpdateSrcMediaStreamPlaying();
6162 UpdateAudioChannelPlayingState();
6163 StartMediaControlKeyListenerIfNeeded();
6165 if (mDecoder) {
6166 SetPlayedOrSeeked(true);
6167 if (mCurrentPlayRangeStart == -1.0) {
6168 mCurrentPlayRangeStart = CurrentTime();
6170 MOZ_ASSERT(!mSuspendedByInactiveDocOrDocshell);
6171 mDecoder->Play();
6172 } else if (mSrcStream) {
6173 SetPlayedOrSeeked(true);
6176 // https://html.spec.whatwg.org/multipage/media.html#ready-states:show-poster-flag
6177 if (mShowPoster) {
6178 mShowPoster = false;
6179 if (mTextTrackManager) {
6180 mTextTrackManager->TimeMarchesOn();
6184 // For blocked media, the event would be pending until it is resumed.
6185 DispatchAsyncEvent(u"play"_ns);
6187 DispatchAsyncEvent(u"playing"_ns);
6190 bool HTMLMediaElement::IsActuallyInvisible() const {
6191 // That means an element is not connected. It probably hasn't connected to a
6192 // document tree, or connects to a disconnected DOM tree.
6193 if (!IsInComposedDoc()) {
6194 return true;
6197 // An element is not in user's view port, which means it's either existing in
6198 // somewhere in the page where user hasn't seen yet, or is being set
6199 // `display:none`.
6200 if (!IsInViewPort()) {
6201 return true;
6204 // Element being used in picture-in-picture mode would be always visible.
6205 if (IsBeingUsedInPictureInPictureMode()) {
6206 return false;
6209 // That check is the page is in the background.
6210 return OwnerDoc()->Hidden();
6213 bool HTMLMediaElement::IsInViewPort() const {
6214 return mVisibilityState == Visibility::ApproximatelyVisible;
6217 VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() {
6218 if (mShuttingDown) {
6219 return nullptr;
6222 if (mVideoFrameContainer) return mVideoFrameContainer;
6224 // Only video frames need an image container.
6225 if (!IsVideo()) {
6226 return nullptr;
6229 mVideoFrameContainer = new VideoFrameContainer(
6230 this, MakeAndAddRef<ImageContainer>(ImageContainer::ASYNCHRONOUS));
6232 return mVideoFrameContainer;
6235 void HTMLMediaElement::PrincipalChanged(MediaStreamTrack* aTrack) {
6236 if (aTrack != mSelectedVideoStreamTrack) {
6237 return;
6240 nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
6241 aTrack->GetPrincipal());
6243 LOG(LogLevel::Debug,
6244 ("HTMLMediaElement %p video track principal changed to %p (combined "
6245 "into %p). Waiting for it to reach VideoFrameContainer before setting.",
6246 this, aTrack->GetPrincipal(), mSrcStreamVideoPrincipal.get()));
6248 if (mVideoFrameContainer) {
6249 UpdateSrcStreamVideoPrincipal(
6250 mVideoFrameContainer->GetLastPrincipalHandle());
6254 void HTMLMediaElement::UpdateSrcStreamVideoPrincipal(
6255 const PrincipalHandle& aPrincipalHandle) {
6256 nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
6257 mSrcStream->GetVideoTracks(videoTracks);
6259 for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
6260 if (PrincipalHandleMatches(aPrincipalHandle, track->GetPrincipal()) &&
6261 !track->Ended()) {
6262 // When the PrincipalHandle for the VideoFrameContainer changes to that of
6263 // a live track in mSrcStream we know that a removed track was displayed
6264 // but is no longer so.
6265 LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
6266 "PrincipalHandle matches track %p. That's all we "
6267 "need.",
6268 this, track.get()));
6269 mSrcStreamVideoPrincipal = track->GetPrincipal();
6270 break;
6275 void HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(
6276 VideoFrameContainer* aContainer,
6277 const PrincipalHandle& aNewPrincipalHandle) {
6278 MOZ_ASSERT(NS_IsMainThread());
6280 if (!mSrcStream) {
6281 return;
6284 LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
6285 "VideoFrameContainer.",
6286 this));
6288 UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
6291 already_AddRefed<nsMediaEventRunner> HTMLMediaElement::GetEventRunner(
6292 const nsAString& aName, EventFlag aFlag) {
6293 RefPtr<nsMediaEventRunner> runner;
6294 if (aName.EqualsLiteral("playing")) {
6295 runner = new nsNotifyAboutPlayingRunner(this, TakePendingPlayPromises());
6296 } else if (aName.EqualsLiteral("timeupdate")) {
6297 runner = new nsTimeupdateRunner(this, aFlag == EventFlag::eMandatory);
6298 } else {
6299 runner = new nsAsyncEventRunner(aName, this);
6301 return runner.forget();
6304 nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName) {
6305 LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
6306 NS_ConvertUTF16toUTF8(aName).get()));
6308 if (mEventBlocker->ShouldBlockEventDelivery()) {
6309 RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
6310 mEventBlocker->PostponeEvent(runner);
6311 return NS_OK;
6314 return nsContentUtils::DispatchTrustedEvent(OwnerDoc(), this, aName,
6315 CanBubble::eNo, Cancelable::eNo);
6318 void HTMLMediaElement::DispatchAsyncEvent(const nsAString& aName) {
6319 RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
6320 DispatchAsyncEvent(std::move(runner));
6323 void HTMLMediaElement::DispatchAsyncEvent(RefPtr<nsMediaEventRunner> aRunner) {
6324 NS_ConvertUTF16toUTF8 eventName(aRunner->EventName());
6325 LOG_EVENT(LogLevel::Debug, ("%p Queuing event %s", this, eventName.get()));
6326 DDLOG(DDLogCategory::Event, "HTMLMediaElement", nsCString(eventName.get()));
6327 if (mEventBlocker->ShouldBlockEventDelivery()) {
6328 mEventBlocker->PostponeEvent(aRunner);
6329 return;
6331 GetMainThreadSerialEventTarget()->Dispatch(aRunner.forget());
6334 bool HTMLMediaElement::IsPotentiallyPlaying() const {
6335 // TODO:
6336 // playback has not stopped due to errors,
6337 // and the element has not paused for user interaction
6338 return !mPaused &&
6339 (mReadyState == HAVE_ENOUGH_DATA || mReadyState == HAVE_FUTURE_DATA) &&
6340 !IsPlaybackEnded();
6343 bool HTMLMediaElement::IsPlaybackEnded() const {
6344 // TODO:
6345 // the current playback position is equal to the effective end of the media
6346 // resource. See bug 449157.
6347 if (mDecoder) {
6348 return mReadyState >= HAVE_METADATA && mDecoder->IsEnded();
6350 if (mSrcStream) {
6351 return mReadyState >= HAVE_METADATA && mSrcStreamPlaybackEnded;
6353 return false;
6356 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentPrincipal() {
6357 if (mDecoder) {
6358 return mDecoder->GetCurrentPrincipal();
6360 if (mSrcStream) {
6361 nsTArray<RefPtr<MediaStreamTrack>> tracks;
6362 mSrcStream->GetTracks(tracks);
6363 nsCOMPtr<nsIPrincipal> principal = mSrcStream->GetPrincipal();
6364 return principal.forget();
6366 return nullptr;
6369 bool HTMLMediaElement::HadCrossOriginRedirects() {
6370 if (mDecoder) {
6371 return mDecoder->HadCrossOriginRedirects();
6373 return false;
6376 bool HTMLMediaElement::ShouldResistFingerprinting(RFPTarget aTarget) const {
6377 return OwnerDoc()->ShouldResistFingerprinting(aTarget);
6380 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentVideoPrincipal() {
6381 if (mDecoder) {
6382 return mDecoder->GetCurrentPrincipal();
6384 if (mSrcStream) {
6385 nsCOMPtr<nsIPrincipal> principal = mSrcStreamVideoPrincipal;
6386 return principal.forget();
6388 return nullptr;
6391 void HTMLMediaElement::NotifyDecoderPrincipalChanged() {
6392 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
6393 bool isSameOrigin = !principal || IsCORSSameOrigin();
6394 mDecoder->UpdateSameOriginStatus(isSameOrigin);
6396 if (isSameOrigin) {
6397 principal = NodePrincipal();
6399 for (const auto& entry : mOutputTrackSources.Values()) {
6400 entry->SetPrincipal(principal);
6402 mDecoder->SetOutputTracksPrincipal(principal);
6405 void HTMLMediaElement::Invalidate(ImageSizeChanged aImageSizeChanged,
6406 const Maybe<nsIntSize>& aNewIntrinsicSize,
6407 ForceInvalidate aForceInvalidate) {
6408 nsIFrame* frame = GetPrimaryFrame();
6409 if (aNewIntrinsicSize) {
6410 UpdateMediaSize(aNewIntrinsicSize.value());
6411 if (frame) {
6412 nsPresContext* presContext = frame->PresContext();
6413 PresShell* presShell = presContext->PresShell();
6414 presShell->FrameNeedsReflow(frame,
6415 IntrinsicDirty::FrameAncestorsAndDescendants,
6416 NS_FRAME_IS_DIRTY);
6420 RefPtr<ImageContainer> imageContainer = GetImageContainer();
6421 bool asyncInvalidate = imageContainer && imageContainer->IsAsync() &&
6422 aForceInvalidate == ForceInvalidate::No;
6423 if (frame) {
6424 if (aImageSizeChanged == ImageSizeChanged::Yes) {
6425 frame->InvalidateFrame();
6426 } else {
6427 frame->InvalidateLayer(DisplayItemType::TYPE_VIDEO, nullptr, nullptr,
6428 asyncInvalidate ? nsIFrame::UPDATE_IS_ASYNC : 0);
6432 SVGObserverUtils::InvalidateDirectRenderingObservers(this);
6435 void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize) {
6436 MOZ_ASSERT(NS_IsMainThread());
6438 if (IsVideo() && mReadyState != HAVE_NOTHING &&
6439 mMediaInfo.mVideo.mDisplay != aSize) {
6440 DispatchAsyncEvent(u"resize"_ns);
6443 mMediaInfo.mVideo.mDisplay = aSize;
6444 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
6447 void HTMLMediaElement::SuspendOrResumeElement(bool aSuspendElement) {
6448 LOG(LogLevel::Debug, ("%p SuspendOrResumeElement(suspend=%d) docHidden=%d",
6449 this, aSuspendElement, OwnerDoc()->Hidden()));
6451 if (aSuspendElement == mSuspendedByInactiveDocOrDocshell) {
6452 return;
6455 mSuspendedByInactiveDocOrDocshell = aSuspendElement;
6456 UpdateSrcMediaStreamPlaying();
6457 UpdateAudioChannelPlayingState();
6459 if (aSuspendElement) {
6460 if (mDecoder) {
6461 mDecoder->Pause();
6462 mDecoder->Suspend();
6463 mDecoder->SetDelaySeekMode(true);
6465 mEventBlocker->SetBlockEventDelivery(true);
6466 // We won't want to resume media element from the bfcache.
6467 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
6468 mMediaControlKeyListener->StopIfNeeded();
6469 } else {
6470 if (mDecoder) {
6471 mDecoder->Resume();
6472 if (!mPaused && !mDecoder->IsEnded()) {
6473 mDecoder->Play();
6475 mDecoder->SetDelaySeekMode(false);
6477 mEventBlocker->SetBlockEventDelivery(false);
6478 // If the media element has been blocked and isn't still allowed to play
6479 // when it comes back from the bfcache, we would notify front end to show
6480 // the blocking icon in order to inform user that the site is still being
6481 // blocked.
6482 if (mHasEverBeenBlockedForAutoplay && !AllowedToPlay()) {
6483 MaybeNotifyAutoplayBlocked();
6485 StartMediaControlKeyListenerIfNeeded();
6487 if (StaticPrefs::media_testing_only_events()) {
6488 auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
6489 this, u"MozMediaSuspendChanged"_ns, CanBubble::eYes,
6490 ChromeOnlyDispatch::eYes);
6491 dispatcher->PostDOMEvent();
6495 bool HTMLMediaElement::IsBeingDestroyed() {
6496 nsIDocShell* docShell = OwnerDoc()->GetDocShell();
6497 bool isBeingDestroyed = false;
6498 if (docShell) {
6499 docShell->IsBeingDestroyed(&isBeingDestroyed);
6501 return isBeingDestroyed;
6504 bool HTMLMediaElement::ShouldBeSuspendedByInactiveDocShell() const {
6505 BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
6506 return bc && !bc->IsActive() && bc->Top()->GetSuspendMediaWhenInactive();
6509 void HTMLMediaElement::NotifyOwnerDocumentActivityChanged() {
6510 if (mDecoder && !IsBeingDestroyed()) {
6511 NotifyDecoderActivityChanges();
6514 // We would suspend media when the document is inactive, or its docshell has
6515 // been set to hidden and explicitly wants to suspend media. In those cases,
6516 // the media would be not visible and we don't want them to continue playing.
6517 bool shouldSuspend =
6518 !OwnerDoc()->IsActive() || ShouldBeSuspendedByInactiveDocShell();
6519 SuspendOrResumeElement(shouldSuspend);
6521 // If the owning document has become inactive we should shutdown the CDM.
6522 if (!OwnerDoc()->IsCurrentActiveDocument() && mMediaKeys) {
6523 // We don't shutdown MediaKeys here because it also listens for document
6524 // activity and will take care of shutting down itself.
6525 DDUNLINKCHILD(mMediaKeys.get());
6526 mMediaKeys = nullptr;
6527 if (mDecoder) {
6528 ShutdownDecoder();
6532 AddRemoveSelfReference();
6535 void HTMLMediaElement::NotifyFullScreenChanged() {
6536 const bool isInFullScreen = IsInFullScreen();
6537 if (isInFullScreen) {
6538 StartMediaControlKeyListenerIfNeeded();
6539 if (!mMediaControlKeyListener->IsStarted()) {
6540 MEDIACONTROL_LOG("Failed to start the listener when entering fullscreen");
6543 // Updating controller fullscreen state no matter the listener starts or not.
6544 BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
6545 if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(bc)) {
6546 updater->NotifyMediaFullScreenState(bc->Id(), isInFullScreen);
6550 void HTMLMediaElement::AddRemoveSelfReference() {
6551 // XXX we could release earlier here in many situations if we examined
6552 // which event listeners are attached. Right now we assume there is a
6553 // potential listener for every event. We would also have to keep the
6554 // element alive if it was playing and producing audio output --- right now
6555 // that's covered by the !mPaused check.
6556 Document* ownerDoc = OwnerDoc();
6558 // See the comment at the top of this file for the explanation of this
6559 // boolean expression.
6560 bool needSelfReference =
6561 !mShuttingDown && ownerDoc->IsActive() &&
6562 (mDelayingLoadEvent || (!mPaused && !Ended()) ||
6563 (mDecoder && mDecoder->IsSeeking()) || IsEligibleForAutoplay() ||
6564 (mMediaSource ? mProgressTimer : mNetworkState == NETWORK_LOADING));
6566 if (needSelfReference != mHasSelfReference) {
6567 mHasSelfReference = needSelfReference;
6568 RefPtr<HTMLMediaElement> self = this;
6569 if (needSelfReference) {
6570 // The shutdown observer will hold a strong reference to us. This
6571 // will do to keep us alive. We need to know about shutdown so that
6572 // we can release our self-reference.
6573 GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
6574 "dom::HTMLMediaElement::AddSelfReference",
6575 [self]() { self->mShutdownObserver->AddRefMediaElement(); }));
6576 } else {
6577 // Dispatch Release asynchronously so that we don't destroy this object
6578 // inside a call stack of method calls on this object
6579 GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
6580 "dom::HTMLMediaElement::AddSelfReference",
6581 [self]() { self->mShutdownObserver->ReleaseMediaElement(); }));
6586 void HTMLMediaElement::NotifyShutdownEvent() {
6587 mShuttingDown = true;
6588 ResetState();
6589 AddRemoveSelfReference();
6592 void HTMLMediaElement::DispatchAsyncSourceError(nsIContent* aSourceElement) {
6593 LOG_EVENT(LogLevel::Debug, ("%p Queuing simple source error event", this));
6595 nsCOMPtr<nsIRunnable> event =
6596 new nsSourceErrorEventRunner(this, aSourceElement);
6597 GetMainThreadSerialEventTarget()->Dispatch(event.forget());
6600 void HTMLMediaElement::NotifyAddedSource() {
6601 // If a source element is inserted as a child of a media element
6602 // that has no src attribute and whose networkState has the value
6603 // NETWORK_EMPTY, the user agent must invoke the media element's
6604 // resource selection algorithm.
6605 if (!HasAttr(nsGkAtoms::src) && mNetworkState == NETWORK_EMPTY) {
6606 AssertReadyStateIsNothing();
6607 QueueSelectResourceTask();
6610 // A load was paused in the resource selection algorithm, waiting for
6611 // a new source child to be added, resume the resource selection algorithm.
6612 if (mLoadWaitStatus == WAITING_FOR_SOURCE) {
6613 // Rest the flag so we don't queue multiple LoadFromSourceTask() when
6614 // multiple <source> are attached in an event loop.
6615 mLoadWaitStatus = NOT_WAITING;
6616 QueueLoadFromSourceTask();
6620 HTMLSourceElement* HTMLMediaElement::GetNextSource() {
6621 mSourceLoadCandidate = nullptr;
6623 while (true) {
6624 if (mSourcePointer == nsINode::GetLastChild()) {
6625 return nullptr; // no more children
6628 if (!mSourcePointer) {
6629 mSourcePointer = nsINode::GetFirstChild();
6630 } else {
6631 mSourcePointer = mSourcePointer->GetNextSibling();
6633 nsIContent* child = mSourcePointer;
6635 // If child is a <source> element, it is the next candidate.
6636 if (auto* source = HTMLSourceElement::FromNodeOrNull(child)) {
6637 mSourceLoadCandidate = source;
6638 return source;
6641 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
6642 return nullptr;
6645 void HTMLMediaElement::ChangeDelayLoadStatus(bool aDelay) {
6646 if (mDelayingLoadEvent == aDelay) return;
6648 mDelayingLoadEvent = aDelay;
6650 LOG(LogLevel::Debug, ("%p ChangeDelayLoadStatus(%d) doc=0x%p", this, aDelay,
6651 mLoadBlockedDoc.get()));
6652 if (mDecoder) {
6653 mDecoder->SetLoadInBackground(!aDelay);
6655 if (aDelay) {
6656 mLoadBlockedDoc = OwnerDoc();
6657 mLoadBlockedDoc->BlockOnload();
6658 } else {
6659 // mLoadBlockedDoc might be null due to GC unlinking
6660 if (mLoadBlockedDoc) {
6661 mLoadBlockedDoc->UnblockOnload(false);
6662 mLoadBlockedDoc = nullptr;
6666 // We changed mDelayingLoadEvent which can affect AddRemoveSelfReference
6667 AddRemoveSelfReference();
6670 already_AddRefed<nsILoadGroup> HTMLMediaElement::GetDocumentLoadGroup() {
6671 if (!OwnerDoc()->IsActive()) {
6672 NS_WARNING("Load group requested for media element in inactive document.");
6674 return OwnerDoc()->GetDocumentLoadGroup();
6677 nsresult HTMLMediaElement::CopyInnerTo(Element* aDest) {
6678 nsresult rv = nsGenericHTMLElement::CopyInnerTo(aDest);
6679 NS_ENSURE_SUCCESS(rv, rv);
6680 if (aDest->OwnerDoc()->IsStaticDocument()) {
6681 HTMLMediaElement* dest = static_cast<HTMLMediaElement*>(aDest);
6682 dest->SetMediaInfo(mMediaInfo);
6684 return rv;
6687 already_AddRefed<TimeRanges> HTMLMediaElement::Buffered() const {
6688 media::TimeIntervals buffered =
6689 mDecoder ? mDecoder->GetBuffered() : media::TimeIntervals();
6690 RefPtr<TimeRanges> ranges = new TimeRanges(
6691 ToSupports(OwnerDoc()), buffered.ToMicrosecondResolution());
6692 return ranges.forget();
6695 void HTMLMediaElement::SetRequestHeaders(nsIHttpChannel* aChannel) {
6696 // Send Accept header for video and audio types only (Bug 489071)
6697 SetAcceptHeader(aChannel);
6699 // Apache doesn't send Content-Length when gzip transfer encoding is used,
6700 // which prevents us from estimating the video length (if explicit
6701 // Content-Duration and a length spec in the container are not present either)
6702 // and from seeking. So, disable the standard "Accept-Encoding: gzip,deflate"
6703 // that we usually send. See bug 614760.
6704 DebugOnly<nsresult> rv =
6705 aChannel->SetRequestHeader("Accept-Encoding"_ns, ""_ns, false);
6706 MOZ_ASSERT(NS_SUCCEEDED(rv));
6708 // Set the Referrer header
6710 // FIXME: Shouldn't this use the Element constructor? Though I guess it
6711 // doesn't matter as no HTMLMediaElement supports the referrerinfo attribute.
6712 auto referrerInfo = MakeRefPtr<ReferrerInfo>(*OwnerDoc());
6713 rv = aChannel->SetReferrerInfoWithoutClone(referrerInfo);
6714 MOZ_ASSERT(NS_SUCCEEDED(rv));
6717 const TimeStamp& HTMLMediaElement::LastTimeupdateDispatchTime() const {
6718 MOZ_ASSERT(NS_IsMainThread());
6719 return mLastTimeUpdateDispatchTime;
6722 void HTMLMediaElement::UpdateLastTimeupdateDispatchTime() {
6723 MOZ_ASSERT(NS_IsMainThread());
6724 mLastTimeUpdateDispatchTime = TimeStamp::Now();
6727 bool HTMLMediaElement::ShouldQueueTimeupdateAsyncTask(
6728 TimeupdateType aType) const {
6729 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
6730 // That means dispatching `timeupdate` is mandatorily required in the spec.
6731 if (aType == TimeupdateType::eMandatory) {
6732 return true;
6735 // The timeupdate only occurs when the current playback position changes.
6736 // https://html.spec.whatwg.org/multipage/media.html#event-media-timeupdate
6737 if (mLastCurrentTime == CurrentTime()) {
6738 return false;
6741 // Number of milliseconds between timeupdate events as defined by spec.
6742 if (!mQueueTimeUpdateRunnerTime.IsNull() &&
6743 TimeStamp::Now() - mQueueTimeUpdateRunnerTime <
6744 TimeDuration::FromMilliseconds(TIMEUPDATE_MS)) {
6745 return false;
6747 return true;
6750 void HTMLMediaElement::FireTimeUpdate(TimeupdateType aType) {
6751 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
6753 if (ShouldQueueTimeupdateAsyncTask(aType)) {
6754 RefPtr<nsMediaEventRunner> runner =
6755 GetEventRunner(u"timeupdate"_ns, aType == TimeupdateType::eMandatory
6756 ? EventFlag::eMandatory
6757 : EventFlag::eNone);
6758 DispatchAsyncEvent(std::move(runner));
6759 mQueueTimeUpdateRunnerTime = TimeStamp::Now();
6760 mLastCurrentTime = CurrentTime();
6762 if (mFragmentEnd >= 0.0 && CurrentTime() >= mFragmentEnd) {
6763 Pause();
6764 mFragmentEnd = -1.0;
6765 mFragmentStart = -1.0;
6766 mDecoder->SetFragmentEndTime(mFragmentEnd);
6769 // Update the cues displaying on the video.
6770 // Here mTextTrackManager can be null if the cycle collector has unlinked
6771 // us before our parent. In that case UnbindFromTree will call us
6772 // when our parent is unlinked.
6773 if (mTextTrackManager) {
6774 mTextTrackManager->TimeMarchesOn();
6778 MediaError* HTMLMediaElement::GetError() const { return mErrorSink->mError; }
6780 void HTMLMediaElement::GetCurrentSpec(nsCString& aString) {
6781 // If playing a regular URL, an ObjectURL of a Blob/File, return that.
6782 if (mLoadingSrc) {
6783 mLoadingSrc->GetSpec(aString);
6784 } else if (mSrcMediaSource) {
6785 // If playing an ObjectURL, and it's a MediaSource, return the value of the
6786 // `src` attribute.
6787 nsAutoString src;
6788 GetSrc(src);
6789 CopyUTF16toUTF8(src, aString);
6790 } else {
6791 // Playing e.g. a MediaStream via an object URL - return an empty string
6792 aString.Truncate();
6796 double HTMLMediaElement::MozFragmentEnd() {
6797 double duration = Duration();
6799 // If there is no end fragment, or the fragment end is greater than the
6800 // duration, return the duration.
6801 return (mFragmentEnd < 0.0 || mFragmentEnd > duration) ? duration
6802 : mFragmentEnd;
6805 void HTMLMediaElement::SetDefaultPlaybackRate(double aDefaultPlaybackRate,
6806 ErrorResult& aRv) {
6807 if (mSrcAttrStream) {
6808 return;
6811 if (aDefaultPlaybackRate < 0) {
6812 aRv.Throw(NS_ERROR_NOT_IMPLEMENTED);
6813 return;
6816 double defaultPlaybackRate = ClampPlaybackRate(aDefaultPlaybackRate);
6818 if (mDefaultPlaybackRate == defaultPlaybackRate) {
6819 return;
6822 mDefaultPlaybackRate = defaultPlaybackRate;
6823 DispatchAsyncEvent(u"ratechange"_ns);
6826 void HTMLMediaElement::SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv) {
6827 if (mSrcAttrStream) {
6828 return;
6831 // Changing the playback rate of a media that has more than two channels is
6832 // not supported.
6833 if (aPlaybackRate < 0) {
6834 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
6835 return;
6838 if (mPlaybackRate == aPlaybackRate) {
6839 return;
6842 mPlaybackRate = aPlaybackRate;
6843 // Playback rate threshold above which audio is muted.
6844 uint32_t threshold = StaticPrefs::media_audio_playbackrate_muting_threshold();
6845 if (mPlaybackRate != 0.0 &&
6846 (mPlaybackRate > threshold || mPlaybackRate < 1. / threshold)) {
6847 SetMutedInternal(mMuted | MUTED_BY_INVALID_PLAYBACK_RATE);
6848 } else {
6849 SetMutedInternal(mMuted & ~MUTED_BY_INVALID_PLAYBACK_RATE);
6852 if (mDecoder) {
6853 mDecoder->SetPlaybackRate(ClampPlaybackRate(mPlaybackRate));
6855 DispatchAsyncEvent(u"ratechange"_ns);
6858 void HTMLMediaElement::SetPreservesPitch(bool aPreservesPitch) {
6859 mPreservesPitch = aPreservesPitch;
6860 if (mDecoder) {
6861 mDecoder->SetPreservesPitch(mPreservesPitch);
6865 ImageContainer* HTMLMediaElement::GetImageContainer() {
6866 VideoFrameContainer* container = GetVideoFrameContainer();
6867 return container ? container->GetImageContainer() : nullptr;
6870 void HTMLMediaElement::UpdateAudioChannelPlayingState() {
6871 if (mAudioChannelWrapper) {
6872 mAudioChannelWrapper->UpdateAudioChannelPlayingState();
6876 static const char* VisibilityString(Visibility aVisibility) {
6877 switch (aVisibility) {
6878 case Visibility::Untracked: {
6879 return "Untracked";
6881 case Visibility::ApproximatelyNonVisible: {
6882 return "ApproximatelyNonVisible";
6884 case Visibility::ApproximatelyVisible: {
6885 return "ApproximatelyVisible";
6889 return "NAN";
6892 void HTMLMediaElement::OnVisibilityChange(Visibility aNewVisibility) {
6893 LOG(LogLevel::Debug,
6894 ("OnVisibilityChange(): %s\n", VisibilityString(aNewVisibility)));
6896 mVisibilityState = aNewVisibility;
6897 if (StaticPrefs::media_test_video_suspend()) {
6898 DispatchAsyncEvent(u"visibilitychanged"_ns);
6901 if (!mDecoder) {
6902 return;
6904 NotifyDecoderActivityChanges();
6907 MediaKeys* HTMLMediaElement::GetMediaKeys() const { return mMediaKeys; }
6909 bool HTMLMediaElement::ContainsRestrictedContent() const {
6910 return GetMediaKeys() != nullptr;
6913 void HTMLMediaElement::SetCDMProxyFailure(const MediaResult& aResult) {
6914 LOG(LogLevel::Debug, ("%s", __func__));
6915 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6917 ResetSetMediaKeysTempVariables();
6919 mSetMediaKeysDOMPromise->MaybeReject(aResult.Code(), aResult.Message());
6922 void HTMLMediaElement::RemoveMediaKeys() {
6923 LOG(LogLevel::Debug, ("%s", __func__));
6924 // 5.2.3 Stop using the CDM instance represented by the mediaKeys attribute
6925 // to decrypt media data and remove the association with the media element.
6926 if (mMediaKeys) {
6927 mMediaKeys->Unbind();
6929 mMediaKeys = nullptr;
6932 bool HTMLMediaElement::TryRemoveMediaKeysAssociation() {
6933 MOZ_ASSERT(mMediaKeys);
6934 LOG(LogLevel::Debug, ("%s", __func__));
6935 // 5.2.1 If the user agent or CDM do not support removing the association,
6936 // let this object's attaching media keys value be false and reject promise
6937 // with a new DOMException whose name is NotSupportedError.
6938 // 5.2.2 If the association cannot currently be removed, let this object's
6939 // attaching media keys value be false and reject promise with a new
6940 // DOMException whose name is InvalidStateError.
6941 if (mDecoder) {
6942 RefPtr<HTMLMediaElement> self = this;
6943 mDecoder->SetCDMProxy(nullptr)
6944 ->Then(
6945 AbstractMainThread(), __func__,
6946 [self]() {
6947 self->mSetCDMRequest.Complete();
6949 self->RemoveMediaKeys();
6950 if (self->AttachNewMediaKeys()) {
6951 // No incoming MediaKeys object or MediaDecoder is not
6952 // created yet.
6953 self->MakeAssociationWithCDMResolved();
6956 [self](const MediaResult& aResult) {
6957 self->mSetCDMRequest.Complete();
6958 // 5.2.4 If the preceding step failed, let this object's
6959 // attaching media keys value be false and reject promise with
6960 // a new DOMException whose name is the appropriate error name.
6961 self->SetCDMProxyFailure(aResult);
6963 ->Track(mSetCDMRequest);
6964 return false;
6967 RemoveMediaKeys();
6968 return true;
6971 bool HTMLMediaElement::DetachExistingMediaKeys() {
6972 LOG(LogLevel::Debug, ("%s", __func__));
6973 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6974 // 5.1 If mediaKeys is not null, CDM instance represented by mediaKeys is
6975 // already in use by another media element, and the user agent is unable
6976 // to use it with this element, let this object's attaching media keys
6977 // value be false and reject promise with a new DOMException whose name
6978 // is QuotaExceededError.
6979 if (mIncomingMediaKeys && mIncomingMediaKeys->IsBoundToMediaElement()) {
6980 SetCDMProxyFailure(MediaResult(
6981 NS_ERROR_DOM_MEDIA_KEY_QUOTA_EXCEEDED_ERR,
6982 "MediaKeys object is already bound to another HTMLMediaElement"));
6983 return false;
6986 // 5.2 If the mediaKeys attribute is not null, run the following steps:
6987 if (mMediaKeys) {
6988 return TryRemoveMediaKeysAssociation();
6990 return true;
6993 void HTMLMediaElement::MakeAssociationWithCDMResolved() {
6994 LOG(LogLevel::Debug, ("%s", __func__));
6995 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6997 // 5.4 Set the mediaKeys attribute to mediaKeys.
6998 mMediaKeys = mIncomingMediaKeys;
6999 #ifdef MOZ_WMF_CDM
7000 if (mMediaKeys && mMediaKeys->GetCDMProxy()) {
7001 mIsUsingWMFCDM = !!mMediaKeys->GetCDMProxy()->AsWMFCDMProxy();
7003 #endif
7004 // 5.5 Let this object's attaching media keys value be false.
7005 ResetSetMediaKeysTempVariables();
7006 // 5.6 Resolve promise.
7007 mSetMediaKeysDOMPromise->MaybeResolveWithUndefined();
7008 mSetMediaKeysDOMPromise = nullptr;
7011 bool HTMLMediaElement::TryMakeAssociationWithCDM(CDMProxy* aProxy) {
7012 LOG(LogLevel::Debug, ("%s", __func__));
7013 MOZ_ASSERT(aProxy);
7015 // 5.3.3 Queue a task to run the "Attempt to Resume Playback If Necessary"
7016 // algorithm on the media element.
7017 // Note: Setting the CDMProxy on the MediaDecoder will unblock playback.
7018 if (mDecoder) {
7019 // CDMProxy is set asynchronously in MediaFormatReader, once it's done,
7020 // HTMLMediaElement should resolve or reject the DOM promise.
7021 RefPtr<HTMLMediaElement> self = this;
7022 mDecoder->SetCDMProxy(aProxy)
7023 ->Then(
7024 AbstractMainThread(), __func__,
7025 [self]() {
7026 self->mSetCDMRequest.Complete();
7027 self->MakeAssociationWithCDMResolved();
7029 [self](const MediaResult& aResult) {
7030 self->mSetCDMRequest.Complete();
7031 self->SetCDMProxyFailure(aResult);
7033 ->Track(mSetCDMRequest);
7034 return false;
7036 return true;
7039 bool HTMLMediaElement::AttachNewMediaKeys() {
7040 LOG(LogLevel::Debug,
7041 ("%s incoming MediaKeys(%p)", __func__, mIncomingMediaKeys.get()));
7042 MOZ_ASSERT(mSetMediaKeysDOMPromise);
7044 // 5.3. If mediaKeys is not null, run the following steps:
7045 if (mIncomingMediaKeys) {
7046 auto* cdmProxy = mIncomingMediaKeys->GetCDMProxy();
7047 if (!cdmProxy) {
7048 SetCDMProxyFailure(MediaResult(
7049 NS_ERROR_DOM_INVALID_STATE_ERR,
7050 "CDM crashed before binding MediaKeys object to HTMLMediaElement"));
7051 return false;
7054 // 5.3.1 Associate the CDM instance represented by mediaKeys with the
7055 // media element for decrypting media data.
7056 if (NS_FAILED(mIncomingMediaKeys->Bind(this))) {
7057 // 5.3.2 If the preceding step failed, run the following steps:
7059 // 5.3.2.1 Set the mediaKeys attribute to null.
7060 mMediaKeys = nullptr;
7061 // 5.3.2.2 Let this object's attaching media keys value be false.
7062 // 5.3.2.3 Reject promise with a new DOMException whose name is
7063 // the appropriate error name.
7064 SetCDMProxyFailure(
7065 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
7066 "Failed to bind MediaKeys object to HTMLMediaElement"));
7067 return false;
7069 return TryMakeAssociationWithCDM(cdmProxy);
7071 return true;
7074 void HTMLMediaElement::ResetSetMediaKeysTempVariables() {
7075 mAttachingMediaKey = false;
7076 mIncomingMediaKeys = nullptr;
7079 already_AddRefed<Promise> HTMLMediaElement::SetMediaKeys(
7080 mozilla::dom::MediaKeys* aMediaKeys, ErrorResult& aRv) {
7081 LOG(LogLevel::Debug, ("%p SetMediaKeys(%p) mMediaKeys=%p mDecoder=%p", this,
7082 aMediaKeys, mMediaKeys.get(), mDecoder.get()));
7084 if (MozAudioCaptured()) {
7085 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
7086 return nullptr;
7089 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7090 if (!win) {
7091 aRv.Throw(NS_ERROR_UNEXPECTED);
7092 return nullptr;
7094 RefPtr<DetailedPromise> promise = DetailedPromise::Create(
7095 win->AsGlobal(), aRv, "HTMLMediaElement.setMediaKeys"_ns);
7096 if (aRv.Failed()) {
7097 return nullptr;
7100 // 1. If mediaKeys and the mediaKeys attribute are the same object,
7101 // return a resolved promise.
7102 if (mMediaKeys == aMediaKeys) {
7103 promise->MaybeResolveWithUndefined();
7104 return promise.forget();
7107 // 2. If this object's attaching media keys value is true, return a
7108 // promise rejected with a new DOMException whose name is InvalidStateError.
7109 if (mAttachingMediaKey) {
7110 promise->MaybeRejectWithInvalidStateError(
7111 "A MediaKeys object is in attaching operation.");
7112 return promise.forget();
7115 // 3. Let this object's attaching media keys value be true.
7116 mAttachingMediaKey = true;
7117 mIncomingMediaKeys = aMediaKeys;
7119 // 4. Let promise be a new promise.
7120 mSetMediaKeysDOMPromise = promise;
7122 // 5. Run the following steps in parallel:
7124 // 5.1 & 5.2 & 5.3
7125 if (!DetachExistingMediaKeys() || !AttachNewMediaKeys()) {
7126 return promise.forget();
7129 // 5.4, 5.5, 5.6
7130 MakeAssociationWithCDMResolved();
7132 // 6. Return promise.
7133 return promise.forget();
7136 EventHandlerNonNull* HTMLMediaElement::GetOnencrypted() {
7137 return EventTarget::GetEventHandler(nsGkAtoms::onencrypted);
7140 void HTMLMediaElement::SetOnencrypted(EventHandlerNonNull* aCallback) {
7141 EventTarget::SetEventHandler(nsGkAtoms::onencrypted, aCallback);
7144 EventHandlerNonNull* HTMLMediaElement::GetOnwaitingforkey() {
7145 return EventTarget::GetEventHandler(nsGkAtoms::onwaitingforkey);
7148 void HTMLMediaElement::SetOnwaitingforkey(EventHandlerNonNull* aCallback) {
7149 EventTarget::SetEventHandler(nsGkAtoms::onwaitingforkey, aCallback);
7152 void HTMLMediaElement::DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
7153 const nsAString& aInitDataType) {
7154 LOG(LogLevel::Debug, ("%p DispatchEncrypted initDataType='%s'", this,
7155 NS_ConvertUTF16toUTF8(aInitDataType).get()));
7157 if (mReadyState == HAVE_NOTHING) {
7158 // Ready state not HAVE_METADATA (yet), don't dispatch encrypted now.
7159 // Queueing for later dispatch in MetadataLoaded.
7160 mPendingEncryptedInitData.AddInitData(aInitDataType, aInitData);
7161 return;
7164 RefPtr<MediaEncryptedEvent> event;
7165 if (IsCORSSameOrigin()) {
7166 event = MediaEncryptedEvent::Constructor(this, aInitDataType, aInitData);
7167 } else {
7168 event = MediaEncryptedEvent::Constructor(this);
7171 RefPtr<AsyncEventDispatcher> asyncDispatcher =
7172 new AsyncEventDispatcher(this, event.forget());
7173 asyncDispatcher->PostDOMEvent();
7176 bool HTMLMediaElement::IsEventAttributeNameInternal(nsAtom* aName) {
7177 return aName == nsGkAtoms::onencrypted ||
7178 nsGenericHTMLElement::IsEventAttributeNameInternal(aName);
7181 void HTMLMediaElement::NotifyWaitingForKey() {
7182 LOG(LogLevel::Debug, ("%p, NotifyWaitingForKey()", this));
7184 // http://w3c.github.io/encrypted-media/#wait-for-key
7185 // 7.3.4 Queue a "waitingforkey" Event
7186 // 1. Let the media element be the specified HTMLMediaElement object.
7187 // 2. If the media element's waiting for key value is true, abort these steps.
7188 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
7189 // 3. Set the media element's waiting for key value to true.
7190 // Note: algorithm continues in UpdateReadyStateInternal() when all decoded
7191 // data enqueued in the MDSM is consumed.
7192 mWaitingForKey = WAITING_FOR_KEY;
7193 // mWaitingForKey changed outside of UpdateReadyStateInternal. This may
7194 // affect mReadyState.
7195 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
7199 AudioTrackList* HTMLMediaElement::AudioTracks() { return mAudioTrackList; }
7201 VideoTrackList* HTMLMediaElement::VideoTracks() { return mVideoTrackList; }
7203 TextTrackList* HTMLMediaElement::GetTextTracks() {
7204 return GetOrCreateTextTrackManager()->GetTextTracks();
7207 already_AddRefed<TextTrack> HTMLMediaElement::AddTextTrack(
7208 TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage) {
7209 return GetOrCreateTextTrackManager()->AddTextTrack(
7210 aKind, aLabel, aLanguage, TextTrackMode::Hidden,
7211 TextTrackReadyState::Loaded, TextTrackSource::AddTextTrack);
7214 void HTMLMediaElement::PopulatePendingTextTrackList() {
7215 if (mTextTrackManager) {
7216 mTextTrackManager->PopulatePendingList();
7220 TextTrackManager* HTMLMediaElement::GetOrCreateTextTrackManager() {
7221 if (!mTextTrackManager) {
7222 mTextTrackManager = new TextTrackManager(this);
7223 mTextTrackManager->AddListeners();
7225 return mTextTrackManager;
7228 MediaDecoderOwner::NextFrameStatus HTMLMediaElement::NextFrameStatus() {
7229 if (mDecoder) {
7230 return mDecoder->NextFrameStatus();
7232 if (mSrcStream) {
7233 AutoTArray<RefPtr<MediaTrack>, 4> tracks;
7234 GetAllEnabledMediaTracks(tracks);
7235 if (!tracks.IsEmpty() && !mSrcStreamPlaybackEnded) {
7236 return NEXT_FRAME_AVAILABLE;
7238 return NEXT_FRAME_UNAVAILABLE;
7240 return NEXT_FRAME_UNINITIALIZED;
7243 void HTMLMediaElement::SetDecoder(MediaDecoder* aDecoder) {
7244 MOZ_ASSERT(aDecoder); // Use ShutdownDecoder() to clear.
7245 if (mDecoder) {
7246 ShutdownDecoder();
7248 mDecoder = aDecoder;
7249 DDLINKCHILD("decoder", mDecoder.get());
7250 if (mDecoder && mForcedHidden) {
7251 mDecoder->SetForcedHidden(mForcedHidden);
7255 float HTMLMediaElement::ComputedVolume() const {
7256 return mMuted ? 0.0f
7257 : mAudioChannelWrapper ? mAudioChannelWrapper->GetEffectiveVolume()
7258 : static_cast<float>(mVolume);
7261 bool HTMLMediaElement::ComputedMuted() const {
7262 return (mMuted & MUTED_BY_AUDIO_CHANNEL);
7265 bool HTMLMediaElement::IsSuspendedByInactiveDocOrDocShell() const {
7266 return mSuspendedByInactiveDocOrDocshell;
7269 bool HTMLMediaElement::IsCurrentlyPlaying() const {
7270 // We have playable data, but we still need to check whether data is "real"
7271 // current data.
7272 return mReadyState >= HAVE_CURRENT_DATA && !IsPlaybackEnded();
7275 void HTMLMediaElement::SetAudibleState(bool aAudible) {
7276 if (mIsAudioTrackAudible != aAudible) {
7277 mIsAudioTrackAudible = aAudible;
7278 NotifyAudioPlaybackChanged(
7279 AudioChannelService::AudibleChangedReasons::eDataAudibleChanged);
7283 void HTMLMediaElement::NotifyAudioPlaybackChanged(
7284 AudibleChangedReasons aReason) {
7285 if (mAudioChannelWrapper) {
7286 mAudioChannelWrapper->NotifyAudioPlaybackChanged(aReason);
7288 // We would start the listener after media becomes audible.
7289 const bool isAudible = IsAudible();
7290 if (isAudible && !mMediaControlKeyListener->IsStarted()) {
7291 StartMediaControlKeyListenerIfNeeded();
7293 mMediaControlKeyListener->UpdateMediaAudibleState(isAudible);
7294 // only request wake lock for audible media.
7295 UpdateWakeLock();
7298 void HTMLMediaElement::SetMediaInfo(const MediaInfo& aInfo) {
7299 const bool oldHasAudio = mMediaInfo.HasAudio();
7300 mMediaInfo = aInfo;
7301 if ((aInfo.HasAudio() != oldHasAudio) && mResumeDelayedPlaybackAgent) {
7302 mResumeDelayedPlaybackAgent->UpdateAudibleState(this, IsAudible());
7304 nsILoadContext* loadContext = OwnerDoc()->GetLoadContext();
7305 if (HasAudio() && loadContext && !loadContext->UsePrivateBrowsing()) {
7306 mTitleChangeObserver->Subscribe();
7307 UpdateStreamName();
7308 } else {
7309 mTitleChangeObserver->Unsubscribe();
7311 if (mAudioChannelWrapper) {
7312 mAudioChannelWrapper->AudioCaptureTrackChangeIfNeeded();
7314 UpdateWakeLock();
7317 MediaInfo HTMLMediaElement::GetMediaInfo() const { return mMediaInfo; }
7319 FrameStatistics* HTMLMediaElement::GetFrameStatistics() const {
7320 return mDecoder ? &(mDecoder->GetFrameStatistics()) : nullptr;
7323 void HTMLMediaElement::DispatchAsyncTestingEvent(const nsAString& aName) {
7324 if (!StaticPrefs::media_testing_only_events()) {
7325 return;
7327 DispatchAsyncEvent(aName);
7330 void HTMLMediaElement::AudioCaptureTrackChange(bool aCapture) {
7331 // No need to capture a silent media element.
7332 if (!HasAudio()) {
7333 return;
7336 if (aCapture && !mStreamWindowCapturer) {
7337 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
7338 if (!window) {
7339 return;
7342 MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
7343 MediaTrackGraph::AUDIO_THREAD_DRIVER, window,
7344 MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
7345 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
7346 RefPtr<DOMMediaStream> stream =
7347 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
7348 StreamCaptureType::CAPTURE_AUDIO, mtg);
7349 mStreamWindowCapturer =
7350 MakeUnique<MediaStreamWindowCapturer>(stream, window->WindowID());
7351 } else if (!aCapture && mStreamWindowCapturer) {
7352 for (size_t i = 0; i < mOutputStreams.Length(); i++) {
7353 if (mOutputStreams[i].mStream == mStreamWindowCapturer->mStream) {
7354 // We own this MediaStream, it is not exposed to JS.
7355 AutoTArray<RefPtr<MediaStreamTrack>, 2> tracks;
7356 mStreamWindowCapturer->mStream->GetTracks(tracks);
7357 for (auto& track : tracks) {
7358 track->Stop();
7360 mOutputStreams.RemoveElementAt(i);
7361 break;
7364 mStreamWindowCapturer = nullptr;
7365 if (mOutputStreams.IsEmpty()) {
7366 mTracksCaptured = nullptr;
7371 void HTMLMediaElement::NotifyCueDisplayStatesChanged() {
7372 if (!mTextTrackManager) {
7373 return;
7376 mTextTrackManager->DispatchUpdateCueDisplay();
7379 void HTMLMediaElement::LogVisibility(CallerAPI aAPI) {
7380 const bool isVisible = mVisibilityState == Visibility::ApproximatelyVisible;
7382 LOG(LogLevel::Debug, ("%p visibility = %u, API: '%d' and 'All'", this,
7383 isVisible, static_cast<int>(aAPI)));
7385 if (!isVisible) {
7386 LOG(LogLevel::Debug, ("%p inTree = %u, API: '%d' and 'All'", this,
7387 IsInComposedDoc(), static_cast<int>(aAPI)));
7391 void HTMLMediaElement::UpdateCustomPolicyAfterPlayed() {
7392 if (mAudioChannelWrapper) {
7393 mAudioChannelWrapper->NotifyPlayStateChanged();
7397 AbstractThread* HTMLMediaElement::AbstractMainThread() const {
7398 return AbstractThread::MainThread();
7401 nsTArray<RefPtr<PlayPromise>> HTMLMediaElement::TakePendingPlayPromises() {
7402 return std::move(mPendingPlayPromises);
7405 void HTMLMediaElement::NotifyAboutPlaying() {
7406 // Stick to the DispatchAsyncEvent() call path for now because we want to
7407 // trigger some telemetry-related codes in the DispatchAsyncEvent() method.
7408 DispatchAsyncEvent(u"playing"_ns);
7411 already_AddRefed<PlayPromise> HTMLMediaElement::CreatePlayPromise(
7412 ErrorResult& aRv) const {
7413 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7415 if (!win) {
7416 aRv.Throw(NS_ERROR_UNEXPECTED);
7417 return nullptr;
7420 RefPtr<PlayPromise> promise = PlayPromise::Create(win->AsGlobal(), aRv);
7421 LOG(LogLevel::Debug, ("%p created PlayPromise %p", this, promise.get()));
7423 return promise.forget();
7426 already_AddRefed<Promise> HTMLMediaElement::CreateDOMPromise(
7427 ErrorResult& aRv) const {
7428 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7430 if (!win) {
7431 aRv.Throw(NS_ERROR_UNEXPECTED);
7432 return nullptr;
7435 return Promise::Create(win->AsGlobal(), aRv);
7438 void HTMLMediaElement::AsyncResolvePendingPlayPromises() {
7439 if (mShuttingDown) {
7440 return;
7443 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7444 this, TakePendingPlayPromises());
7446 GetMainThreadSerialEventTarget()->Dispatch(event.forget());
7449 void HTMLMediaElement::AsyncRejectPendingPlayPromises(nsresult aError) {
7450 if (!mPaused) {
7451 mPaused = true;
7452 DispatchAsyncEvent(u"pause"_ns);
7455 if (mShuttingDown) {
7456 return;
7459 if (aError == NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR) {
7460 DispatchEventsWhenPlayWasNotAllowed();
7463 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7464 this, TakePendingPlayPromises(), aError);
7466 GetMainThreadSerialEventTarget()->Dispatch(event.forget());
7469 void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) {
7470 MOZ_ASSERT(NS_IsMainThread(),
7471 "MediaKeys expects to be interacted with on main thread!");
7472 if (!mMediaKeys) {
7473 return;
7475 mMediaKeys->GetKeySystem(aInfo.mKeySystem);
7476 mMediaKeys->GetSessionsInfo(aInfo.mSessionsInfo);
7479 void HTMLMediaElement::NotifyDecoderActivityChanges() const {
7480 if (mDecoder) {
7481 mDecoder->NotifyOwnerActivityChanged(IsActuallyInvisible(),
7482 IsInComposedDoc());
7486 Document* HTMLMediaElement::GetDocument() const { return OwnerDoc(); }
7488 bool HTMLMediaElement::IsAudible() const {
7489 // No audio track.
7490 if (!HasAudio()) {
7491 return false;
7494 // Muted or the volume should not be ~0
7495 if (mMuted || (std::fabs(Volume()) <= 1e-7)) {
7496 return false;
7499 return mIsAudioTrackAudible;
7502 Maybe<nsAutoString> HTMLMediaElement::GetKeySystem() const {
7503 if (!mMediaKeys) {
7504 return Nothing();
7506 nsAutoString keySystem;
7507 mMediaKeys->GetKeySystem(keySystem);
7508 return Some(keySystem);
7511 void HTMLMediaElement::ConstructMediaTracks(const MediaInfo* aInfo) {
7512 if (!aInfo) {
7513 return;
7516 AudioTrackList* audioList = AudioTracks();
7517 if (audioList && aInfo->HasAudio()) {
7518 const TrackInfo& info = aInfo->mAudio;
7519 RefPtr<AudioTrack> track = MediaTrackList::CreateAudioTrack(
7520 audioList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7521 info.mLanguage, info.mEnabled);
7523 audioList->AddTrack(track);
7526 VideoTrackList* videoList = VideoTracks();
7527 if (videoList && aInfo->HasVideo()) {
7528 const TrackInfo& info = aInfo->mVideo;
7529 RefPtr<VideoTrack> track = MediaTrackList::CreateVideoTrack(
7530 videoList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7531 info.mLanguage);
7533 videoList->AddTrack(track);
7534 track->SetEnabledInternal(info.mEnabled, MediaTrack::FIRE_NO_EVENTS);
7538 void HTMLMediaElement::RemoveMediaTracks() {
7539 if (mAudioTrackList) {
7540 mAudioTrackList->RemoveTracks();
7542 if (mVideoTrackList) {
7543 mVideoTrackList->RemoveTracks();
7547 class MediaElementGMPCrashHelper : public GMPCrashHelper {
7548 public:
7549 explicit MediaElementGMPCrashHelper(HTMLMediaElement* aElement)
7550 : mElement(aElement) {
7551 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7553 already_AddRefed<nsPIDOMWindowInner> GetPluginCrashedEventTarget() override {
7554 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7555 if (!mElement) {
7556 return nullptr;
7558 return do_AddRef(mElement->OwnerDoc()->GetInnerWindow());
7561 private:
7562 WeakPtr<HTMLMediaElement> mElement;
7565 already_AddRefed<GMPCrashHelper> HTMLMediaElement::CreateGMPCrashHelper() {
7566 return MakeAndAddRef<MediaElementGMPCrashHelper>(this);
7569 void HTMLMediaElement::MarkAsTainted() {
7570 mHasSuspendTaint = true;
7572 if (mDecoder) {
7573 mDecoder->SetSuspendTaint(true);
7577 bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj) {
7578 return nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::debugger) ||
7579 nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::tabs);
7582 already_AddRefed<Promise> HTMLMediaElement::SetSinkId(const nsAString& aSinkId,
7583 ErrorResult& aRv) {
7584 LOG(LogLevel::Info,
7585 ("%p, setSinkId(%s)", this, NS_ConvertUTF16toUTF8(aSinkId).get()));
7586 nsCOMPtr<nsPIDOMWindowInner> win = OwnerDoc()->GetInnerWindow();
7587 if (!win) {
7588 aRv.Throw(NS_ERROR_UNEXPECTED);
7589 return nullptr;
7592 RefPtr<Promise> promise = Promise::Create(win->AsGlobal(), aRv);
7593 if (aRv.Failed()) {
7594 return nullptr;
7597 if (!FeaturePolicyUtils::IsFeatureAllowed(win->GetExtantDoc(),
7598 u"speaker-selection"_ns)) {
7599 promise->MaybeRejectWithNotAllowedError(
7600 "Document's Permissions Policy does not allow setSinkId()");
7603 if (mSink.first.Equals(aSinkId)) {
7604 promise->MaybeResolveWithUndefined();
7605 return promise.forget();
7608 RefPtr<MediaDevices> mediaDevices = win->Navigator()->GetMediaDevices(aRv);
7609 if (aRv.Failed()) {
7610 return nullptr;
7613 nsString sinkId(aSinkId);
7614 mediaDevices->GetSinkDevice(sinkId)
7615 ->Then(
7616 AbstractMainThread(), __func__,
7617 [self = RefPtr<HTMLMediaElement>(this),
7618 this](RefPtr<AudioDeviceInfo>&& aInfo) {
7619 // Sink found switch output device.
7620 MOZ_ASSERT(aInfo);
7621 if (mDecoder) {
7622 RefPtr<SinkInfoPromise> p = mDecoder->SetSink(aInfo)->Then(
7623 AbstractMainThread(), __func__,
7624 [aInfo](const GenericPromise::ResolveOrRejectValue& aValue) {
7625 if (aValue.IsResolve()) {
7626 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7628 return SinkInfoPromise::CreateAndReject(
7629 aValue.RejectValue(), __func__);
7631 return p;
7633 if (mSrcStream) {
7634 MOZ_ASSERT(mMediaStreamRenderer);
7635 RefPtr<SinkInfoPromise> p =
7636 mMediaStreamRenderer->SetAudioOutputDevice(aInfo)->Then(
7637 AbstractMainThread(), __func__,
7638 [aInfo](
7639 const GenericPromise::ResolveOrRejectValue& aValue) {
7640 if (aValue.IsResolve()) {
7641 return SinkInfoPromise::CreateAndResolve(aInfo,
7642 __func__);
7644 return SinkInfoPromise::CreateAndReject(
7645 aValue.RejectValue(), __func__);
7647 return p;
7649 // No media attached to the element save it for later.
7650 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7652 [](nsresult res) {
7653 // Promise is rejected, sink not found.
7654 return SinkInfoPromise::CreateAndReject(res, __func__);
7656 ->Then(AbstractMainThread(), __func__,
7657 [promise, self = RefPtr<HTMLMediaElement>(this), this,
7658 sinkId](const SinkInfoPromise::ResolveOrRejectValue& aValue) {
7659 if (aValue.IsResolve()) {
7660 LOG(LogLevel::Info, ("%p, set sinkid=%s", this,
7661 NS_ConvertUTF16toUTF8(sinkId).get()));
7662 mSink = std::pair(sinkId, aValue.ResolveValue());
7663 promise->MaybeResolveWithUndefined();
7664 } else {
7665 switch (aValue.RejectValue()) {
7666 case NS_ERROR_ABORT:
7667 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
7668 break;
7669 case NS_ERROR_NOT_AVAILABLE: {
7670 promise->MaybeRejectWithNotFoundError(
7671 "The object can not be found here.");
7672 break;
7674 default:
7675 MOZ_ASSERT_UNREACHABLE("Invalid error.");
7680 aRv = NS_OK;
7681 return promise.forget();
7684 void HTMLMediaElement::NotifyTextTrackModeChanged() {
7685 if (mPendingTextTrackChanged) {
7686 return;
7688 mPendingTextTrackChanged = true;
7689 AbstractMainThread()->Dispatch(
7690 NS_NewRunnableFunction("HTMLMediaElement::NotifyTextTrackModeChanged",
7691 [this, self = RefPtr<HTMLMediaElement>(this)]() {
7692 mPendingTextTrackChanged = false;
7693 if (!mTextTrackManager) {
7694 return;
7696 GetTextTracks()->CreateAndDispatchChangeEvent();
7697 // https://html.spec.whatwg.org/multipage/media.html#text-track-model:show-poster-flag
7698 if (!mShowPoster) {
7699 mTextTrackManager->TimeMarchesOn();
7701 }));
7704 void HTMLMediaElement::CreateResumeDelayedMediaPlaybackAgentIfNeeded() {
7705 if (mResumeDelayedPlaybackAgent) {
7706 return;
7708 mResumeDelayedPlaybackAgent =
7709 MediaPlaybackDelayPolicy::CreateResumeDelayedPlaybackAgent(this,
7710 IsAudible());
7711 if (!mResumeDelayedPlaybackAgent) {
7712 LOG(LogLevel::Debug,
7713 ("%p Failed to create a delayed playback agant", this));
7714 return;
7716 mResumeDelayedPlaybackAgent->GetResumePromise()
7717 ->Then(
7718 AbstractMainThread(), __func__,
7719 [self = RefPtr<HTMLMediaElement>(this)]() {
7720 LOG(LogLevel::Debug, ("%p Resume delayed Play() call", self.get()));
7721 self->mResumePlaybackRequest.Complete();
7722 self->mResumeDelayedPlaybackAgent = nullptr;
7723 IgnoredErrorResult dummy;
7724 RefPtr<Promise> toBeIgnored = self->Play(dummy);
7726 [self = RefPtr<HTMLMediaElement>(this)]() {
7727 LOG(LogLevel::Debug,
7728 ("%p Can not resume delayed Play() call", self.get()));
7729 self->mResumePlaybackRequest.Complete();
7730 self->mResumeDelayedPlaybackAgent = nullptr;
7732 ->Track(mResumePlaybackRequest);
7735 void HTMLMediaElement::ClearResumeDelayedMediaPlaybackAgentIfNeeded() {
7736 if (mResumeDelayedPlaybackAgent) {
7737 mResumePlaybackRequest.DisconnectIfExists();
7738 mResumeDelayedPlaybackAgent = nullptr;
7742 void HTMLMediaElement::NotifyMediaControlPlaybackStateChanged() {
7743 if (!mMediaControlKeyListener->IsStarted()) {
7744 return;
7746 if (mPaused) {
7747 mMediaControlKeyListener->NotifyMediaStoppedPlaying();
7748 } else {
7749 mMediaControlKeyListener->NotifyMediaStartedPlaying();
7753 bool HTMLMediaElement::IsInFullScreen() const {
7754 return State().HasState(ElementState::FULLSCREEN);
7757 bool HTMLMediaElement::IsPlayable() const {
7758 return (mDecoder || mSrcStream) && !HasError();
7761 bool HTMLMediaElement::ShouldStartMediaControlKeyListener() const {
7762 if (!IsPlayable()) {
7763 MEDIACONTROL_LOG("Not start listener because media is not playable");
7764 return false;
7767 if (mSrcStream) {
7768 MEDIACONTROL_LOG("Not listening because media is real-time");
7769 return false;
7772 if (IsBeingUsedInPictureInPictureMode()) {
7773 MEDIACONTROL_LOG("Start listener because of being used in PiP mode");
7774 return true;
7777 if (IsInFullScreen()) {
7778 MEDIACONTROL_LOG("Start listener because of being used in fullscreen");
7779 return true;
7782 // In order to filter out notification-ish sound, we use this pref to set the
7783 // eligible media duration to prevent showing media control for those short
7784 // sound.
7785 if (Duration() <
7786 StaticPrefs::media_mediacontrol_eligible_media_duration_s()) {
7787 MEDIACONTROL_LOG("Not listening because media's duration %f is too short.",
7788 Duration());
7789 return false;
7792 // This includes cases such like `video is muted`, `video has zero volume`,
7793 // `video's audio track is still inaudible` and `tab is muted by audio channel
7794 // (tab sound indicator)`, all these cases would make media inaudible.
7795 // `ComputedVolume()` would return the final volume applied the affection made
7796 // by audio channel, which is used to detect if the tab is muted by audio
7797 // channel.
7798 if (!IsAudible() || ComputedVolume() == 0.0f) {
7799 MEDIACONTROL_LOG("Not listening because media is inaudible");
7800 return false;
7802 return true;
7805 void HTMLMediaElement::StartMediaControlKeyListenerIfNeeded() {
7806 if (!ShouldStartMediaControlKeyListener()) {
7807 return;
7809 mMediaControlKeyListener->Start();
7812 void HTMLMediaElement::UpdateStreamName() {
7813 MOZ_ASSERT(NS_IsMainThread());
7815 nsAutoString aTitle;
7816 OwnerDoc()->GetTitle(aTitle);
7818 if (mDecoder) {
7819 mDecoder->SetStreamName(aTitle);
7823 void HTMLMediaElement::SetSecondaryMediaStreamRenderer(
7824 VideoFrameContainer* aContainer,
7825 FirstFrameVideoOutput* aFirstFrameOutput /* = nullptr */) {
7826 MOZ_ASSERT(mSrcStream);
7827 MOZ_ASSERT(mMediaStreamRenderer);
7828 if (mSecondaryMediaStreamRenderer) {
7829 mSecondaryMediaStreamRenderer->Shutdown();
7830 mSecondaryMediaStreamRenderer = nullptr;
7832 if (aContainer) {
7833 mSecondaryMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
7834 AbstractMainThread(), aContainer, aFirstFrameOutput, this);
7835 if (mSrcStreamIsPlaying) {
7836 mSecondaryMediaStreamRenderer->Start();
7838 if (mSelectedVideoStreamTrack) {
7839 mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
7844 void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() {
7845 if (IsBeingUsedInPictureInPictureMode()) {
7846 // When media enters PIP mode, we should ensure that the listener has been
7847 // started because we always want to control PIP video.
7848 StartMediaControlKeyListenerIfNeeded();
7849 if (!mMediaControlKeyListener->IsStarted()) {
7850 MEDIACONTROL_LOG("Failed to start listener when entering PIP mode");
7852 // Updating controller PIP state no matter the listener starts or not.
7853 mMediaControlKeyListener->SetPictureInPictureModeEnabled(true);
7854 } else {
7855 mMediaControlKeyListener->SetPictureInPictureModeEnabled(false);
7859 bool HTMLMediaElement::IsBeingUsedInPictureInPictureMode() const {
7860 if (!IsVideo()) {
7861 return false;
7863 return static_cast<const HTMLVideoElement*>(this)->IsCloningElementVisually();
7866 void HTMLMediaElement::NodeInfoChanged(Document* aOldDoc) {
7867 if (mMediaSource) {
7868 OwnerDoc()->AddMediaElementWithMSE();
7869 aOldDoc->RemoveMediaElementWithMSE();
7872 nsGenericHTMLElement::NodeInfoChanged(aOldDoc);
7875 #ifdef MOZ_WMF_CDM
7876 bool HTMLMediaElement::IsUsingWMFCDM() const { return mIsUsingWMFCDM; };
7877 #endif
7879 } // namespace mozilla::dom
7881 #undef LOG
7882 #undef LOG_EVENT