Bug 1655413 [wpt PR 24763] - Make CSP default-src without 'unsafe-eval' block eval...
[gecko.git] / dom / html / HTMLMediaElement.h
blob82e6c2e53aa88d5be6a4c380f59adee50682325a
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef mozilla_dom_HTMLMediaElement_h
7 #define mozilla_dom_HTMLMediaElement_h
9 #include "nsGenericHTMLElement.h"
10 #include "AudioChannelService.h"
11 #include "MediaEventSource.h"
12 #include "SeekTarget.h"
13 #include "MediaDecoderOwner.h"
14 #include "MediaPlaybackDelayPolicy.h"
15 #include "MediaPromiseDefs.h"
16 #include "nsCycleCollectionParticipant.h"
17 #include "Visibility.h"
18 #include "mozilla/CORSMode.h"
19 #include "DecoderTraits.h"
20 #include "mozilla/Attributes.h"
21 #include "mozilla/StateWatching.h"
22 #include "mozilla/WeakPtr.h"
23 #include "mozilla/dom/HTMLMediaElementBinding.h"
24 #include "mozilla/dom/MediaDebugInfoBinding.h"
25 #include "mozilla/dom/MediaKeys.h"
26 #include "mozilla/dom/TextTrackManager.h"
27 #include "nsGkAtoms.h"
28 #include "PrincipalChangeObserver.h"
29 #include "nsStubMutationObserver.h"
30 #include "MediaSegment.h" // for PrincipalHandle, GraphTime
32 #include <utility>
34 // X.h on Linux #defines CurrentTime as 0L, so we have to #undef it here.
35 #ifdef CurrentTime
36 # undef CurrentTime
37 #endif
39 // Define to output information on decoding and painting framerate
40 /* #define DEBUG_FRAME_RATE 1 */
42 typedef uint16_t nsMediaNetworkState;
43 typedef uint16_t nsMediaReadyState;
44 typedef uint32_t SuspendTypes;
45 typedef uint32_t AudibleChangedReasons;
47 namespace mozilla {
48 class AbstractThread;
49 class ChannelMediaDecoder;
50 class DecoderDoctorDiagnostics;
51 class DOMMediaStream;
52 class ErrorResult;
53 class MediaResource;
54 class MediaDecoder;
55 class MediaInputPort;
56 class MediaTrack;
57 class MediaTrackGraph;
58 class MediaStreamWindowCapturer;
59 struct SharedDummyTrack;
60 class VideoFrameContainer;
61 namespace dom {
62 class MediaKeys;
63 class TextTrack;
64 class TimeRanges;
65 class WakeLock;
66 class MediaStreamTrack;
67 class MediaStreamTrackSource;
68 class MediaTrack;
69 class VideoStreamTrack;
70 } // namespace dom
71 } // namespace mozilla
73 class AudioDeviceInfo;
74 class nsIChannel;
75 class nsIHttpChannel;
76 class nsILoadGroup;
77 class nsIRunnable;
78 class nsISerialEventTarget;
79 class nsITimer;
80 class nsRange;
82 namespace mozilla {
83 namespace dom {
85 // Number of milliseconds between timeupdate events as defined by spec
86 #define TIMEUPDATE_MS 250
88 class MediaError;
89 class MediaSource;
90 class PlayPromise;
91 class Promise;
92 class TextTrackList;
93 class AudioTrackList;
94 class VideoTrackList;
96 enum class StreamCaptureType : uint8_t { CAPTURE_ALL_TRACKS, CAPTURE_AUDIO };
98 enum class StreamCaptureBehavior : uint8_t {
99 CONTINUE_WHEN_ENDED,
100 FINISH_WHEN_ENDED
103 class HTMLMediaElement : public nsGenericHTMLElement,
104 public MediaDecoderOwner,
105 public PrincipalChangeObserver<MediaStreamTrack>,
106 public SupportsWeakPtr,
107 public nsStubMutationObserver {
108 public:
109 typedef mozilla::TimeStamp TimeStamp;
110 typedef mozilla::layers::ImageContainer ImageContainer;
111 typedef mozilla::VideoFrameContainer VideoFrameContainer;
112 typedef mozilla::MediaResource MediaResource;
113 typedef mozilla::MediaDecoderOwner MediaDecoderOwner;
114 typedef mozilla::MetadataTags MetadataTags;
116 // Helper struct to keep track of the MediaStreams returned by
117 // mozCaptureStream(). For each OutputMediaStream, dom::MediaTracks get
118 // captured into MediaStreamTracks which get added to
119 // OutputMediaStream::mStream.
120 struct OutputMediaStream {
121 OutputMediaStream(RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
122 bool aFinishWhenEnded);
123 ~OutputMediaStream();
125 RefPtr<DOMMediaStream> mStream;
126 nsTArray<RefPtr<MediaStreamTrack>> mLiveTracks;
127 const bool mCapturingAudioOnly;
128 const bool mFinishWhenEnded;
129 // If mFinishWhenEnded is true, this is the URI of the first resource
130 // mStream got tracks for, if not a MediaStream.
131 nsCOMPtr<nsIURI> mFinishWhenEndedLoadingSrc;
132 // If mFinishWhenEnded is true, this is the first MediaStream mStream got
133 // tracks for, if not a resource.
134 RefPtr<DOMMediaStream> mFinishWhenEndedAttrStream;
137 NS_DECL_NSIMUTATIONOBSERVER_CONTENTREMOVED
139 CORSMode GetCORSMode() { return mCORSMode; }
141 explicit HTMLMediaElement(
142 already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo);
143 void Init();
145 void ReportCanPlayTelemetry();
148 * This is used when the browser is constructing a video element to play
149 * a channel that we've already started loading. The src attribute and
150 * <source> children are ignored.
151 * @param aChannel the channel to use
152 * @param aListener returns a stream listener that should receive
153 * notifications for the stream
155 nsresult LoadWithChannel(nsIChannel* aChannel, nsIStreamListener** aListener);
157 // nsISupports
158 NS_DECL_ISUPPORTS_INHERITED
159 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(HTMLMediaElement,
160 nsGenericHTMLElement)
161 NS_IMPL_FROMNODE_HELPER(HTMLMediaElement,
162 IsAnyOfHTMLElements(nsGkAtoms::video,
163 nsGkAtoms::audio))
165 NS_DECL_ADDSIZEOFEXCLUDINGTHIS
167 // EventTarget
168 void GetEventTargetParent(EventChainPreVisitor& aVisitor) override;
170 virtual bool ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
171 const nsAString& aValue,
172 nsIPrincipal* aMaybeScriptedPrincipal,
173 nsAttrValue& aResult) override;
175 virtual nsresult BindToTree(BindContext&, nsINode& aParent) override;
176 virtual void UnbindFromTree(bool aNullParent = true) override;
177 virtual void DoneCreatingElement() override;
179 virtual bool IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
180 int32_t* aTabIndex) override;
181 virtual int32_t TabIndexDefault() override;
183 // Called by the video decoder object, on the main thread,
184 // when it has read the metadata containing video dimensions,
185 // etc.
186 virtual void MetadataLoaded(const MediaInfo* aInfo,
187 UniquePtr<const MetadataTags> aTags) final;
189 // Called by the decoder object, on the main thread,
190 // when it has read the first frame of the video or audio.
191 void FirstFrameLoaded() final;
193 // Called by the video decoder object, on the main thread,
194 // when the resource has a network error during loading.
195 void NetworkError(const MediaResult& aError) final;
197 // Called by the video decoder object, on the main thread, when the
198 // resource has a decode error during metadata loading or decoding.
199 void DecodeError(const MediaResult& aError) final;
201 // Called by the decoder object, on the main thread, when the
202 // resource has a decode issue during metadata loading or decoding, but can
203 // continue decoding.
204 void DecodeWarning(const MediaResult& aError) final;
206 // Return true if error attribute is not null.
207 bool HasError() const final;
209 // Called by the video decoder object, on the main thread, when the
210 // resource load has been cancelled.
211 void LoadAborted() final;
213 // Called by the video decoder object, on the main thread,
214 // when the video playback has ended.
215 void PlaybackEnded() final;
217 // Called by the video decoder object, on the main thread,
218 // when the resource has started seeking.
219 void SeekStarted() final;
221 // Called by the video decoder object, on the main thread,
222 // when the resource has completed seeking.
223 void SeekCompleted() final;
225 // Called by the video decoder object, on the main thread,
226 // when the resource has aborted seeking.
227 void SeekAborted() final;
229 // Called by the media stream, on the main thread, when the download
230 // has been suspended by the cache or because the element itself
231 // asked the decoder to suspend the download.
232 void DownloadSuspended() final;
234 // Called by the media stream, on the main thread, when the download
235 // has been resumed by the cache or because the element itself
236 // asked the decoder to resumed the download.
237 void DownloadResumed();
239 // Called to indicate the download is progressing.
240 void DownloadProgressed() final;
242 // Called by the media decoder to indicate whether the media cache has
243 // suspended the channel.
244 void NotifySuspendedByCache(bool aSuspendedByCache) final;
246 bool IsActive() const;
248 bool IsHidden() const;
250 // Called by the media decoder and the video frame to get the
251 // ImageContainer containing the video data.
252 VideoFrameContainer* GetVideoFrameContainer() final;
253 layers::ImageContainer* GetImageContainer();
256 * Call this to reevaluate whether we should start/stop due to our owner
257 * document being active, inactive, visible or hidden.
259 void NotifyOwnerDocumentActivityChanged();
261 // Called when the media element enters or leaves the fullscreen.
262 void NotifyFullScreenChanged();
264 bool IsInFullScreen() const;
266 // From PrincipalChangeObserver<MediaStreamTrack>.
267 void PrincipalChanged(MediaStreamTrack* aTrack) override;
269 void UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle);
271 // Called after the MediaStream we're playing rendered a frame to aContainer
272 // with a different principalHandle than the previous frame.
273 void PrincipalHandleChangedForVideoFrameContainer(
274 VideoFrameContainer* aContainer,
275 const PrincipalHandle& aNewPrincipalHandle) override;
277 // Dispatch events
278 void DispatchAsyncEvent(const nsAString& aName) final;
280 // Triggers a recomputation of readyState.
281 void UpdateReadyState() override {
282 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
285 // Dispatch events that were raised while in the bfcache
286 nsresult DispatchPendingMediaEvents();
288 // Return true if we can activate autoplay assuming enough data has arrived.
289 bool CanActivateAutoplay();
291 // Notify that state has changed that might cause an autoplay element to
292 // start playing.
293 // If the element is 'autoplay' and is ready to play back (not paused,
294 // autoplay pref enabled, etc), it should start playing back.
295 void CheckAutoplayDataReady();
297 // Check if the media element had crossorigin set when loading started
298 bool ShouldCheckAllowOrigin();
300 // Returns true if the currently loaded resource is CORS same-origin with
301 // respect to the document.
302 bool IsCORSSameOrigin();
304 // Is the media element potentially playing as defined by the HTML 5
305 // specification.
306 // http://www.whatwg.org/specs/web-apps/current-work/#potentially-playing
307 bool IsPotentiallyPlaying() const;
309 // Has playback ended as defined by the HTML 5 specification.
310 // http://www.whatwg.org/specs/web-apps/current-work/#ended
311 bool IsPlaybackEnded() const;
313 // principal of the currently playing resource. Anything accessing the
314 // contents of this element must have a principal that subsumes this
315 // principal. Returns null if nothing is playing.
316 already_AddRefed<nsIPrincipal> GetCurrentPrincipal();
318 // Return true if the loading of this resource required cross-origin
319 // redirects.
320 bool HadCrossOriginRedirects();
322 // Principal of the currently playing video resource. Anything accessing the
323 // image container of this element must have a principal that subsumes this
324 // principal. If there are no live video tracks but content has been rendered
325 // to the image container, we return the last video principal we had. Should
326 // the image container be empty with no live video tracks, we return nullptr.
327 already_AddRefed<nsIPrincipal> GetCurrentVideoPrincipal();
329 // called to notify that the principal of the decoder's media resource has
330 // changed.
331 void NotifyDecoderPrincipalChanged() final;
333 void GetEMEInfo(dom::EMEDebugInfo& aInfo);
335 // Update the visual size of the media. Called from the decoder on the
336 // main thread when/if the size changes.
337 virtual void UpdateMediaSize(const nsIntSize& aSize);
338 // Like UpdateMediaSize, but only updates the size if no size has yet
339 // been set.
340 void UpdateInitialMediaSize(const nsIntSize& aSize);
342 void Invalidate(bool aImageSizeChanged, Maybe<nsIntSize>& aNewIntrinsicSize,
343 bool aForceInvalidate) override;
345 // Returns the CanPlayStatus indicating if we can handle the
346 // full MIME type including the optional codecs parameter.
347 static CanPlayStatus GetCanPlay(const nsAString& aType,
348 DecoderDoctorDiagnostics* aDiagnostics);
351 * Called when a child source element is added to this media element. This
352 * may queue a task to run the select resource algorithm if appropriate.
354 void NotifyAddedSource();
357 * Called when there's been an error fetching the resource. This decides
358 * whether it's appropriate to fire an error event.
360 void NotifyLoadError(const nsACString& aErrorDetails = nsCString());
363 * Called by one of our associated MediaTrackLists (audio/video) when an
364 * AudioTrack is enabled or a VideoTrack is selected.
366 void NotifyMediaTrackEnabled(dom::MediaTrack* aTrack);
369 * Called by one of our associated MediaTrackLists (audio/video) when an
370 * AudioTrack is disabled or a VideoTrack is unselected.
372 void NotifyMediaTrackDisabled(dom::MediaTrack* aTrack);
375 * Returns the current load ID. Asynchronous events store the ID that was
376 * current when they were enqueued, and if it has changed when they come to
377 * fire, they consider themselves cancelled, and don't fire.
379 uint32_t GetCurrentLoadID() { return mCurrentLoadID; }
382 * Returns the load group for this media element's owner document.
383 * XXX XBL2 issue.
385 already_AddRefed<nsILoadGroup> GetDocumentLoadGroup();
388 * Returns true if the media has played or completed a seek.
389 * Used by video frame to determine whether to paint the poster.
391 bool GetPlayedOrSeeked() const { return mHasPlayedOrSeeked; }
393 nsresult CopyInnerTo(Element* aDest);
396 * Sets the Accept header on the HTTP channel to the required
397 * video or audio MIME types.
399 virtual nsresult SetAcceptHeader(nsIHttpChannel* aChannel) = 0;
402 * Sets the required request headers on the HTTP channel for
403 * video or audio requests.
405 void SetRequestHeaders(nsIHttpChannel* aChannel);
408 * Asynchronously awaits a stable state, whereupon aRunnable runs on the main
409 * thread. This adds an event which run aRunnable to the appshell's list of
410 * sections synchronous the next time control returns to the event loop.
412 void RunInStableState(nsIRunnable* aRunnable);
415 * Fires a timeupdate event. If aPeriodic is true, the event will only
416 * be fired if we've not fired a timeupdate event (for any reason) in the
417 * last 250ms, as required by the spec when the current time is periodically
418 * increasing during playback.
420 void FireTimeUpdate(bool aPeriodic) final;
422 // WebIDL
424 MediaError* GetError() const;
426 void GetSrc(nsAString& aSrc) { GetURIAttr(nsGkAtoms::src, nullptr, aSrc); }
427 void SetSrc(const nsAString& aSrc, ErrorResult& aError) {
428 SetHTMLAttr(nsGkAtoms::src, aSrc, aError);
430 void SetSrc(const nsAString& aSrc, nsIPrincipal* aTriggeringPrincipal,
431 ErrorResult& aError) {
432 SetHTMLAttr(nsGkAtoms::src, aSrc, aTriggeringPrincipal, aError);
435 void GetCurrentSrc(nsAString& aCurrentSrc);
437 void GetCrossOrigin(nsAString& aResult) {
438 // Null for both missing and invalid defaults is ok, since we
439 // always parse to an enum value, so we don't need an invalid
440 // default, and we _want_ the missing default to be null.
441 GetEnumAttr(nsGkAtoms::crossorigin, nullptr, aResult);
443 void SetCrossOrigin(const nsAString& aCrossOrigin, ErrorResult& aError) {
444 SetOrRemoveNullableStringAttr(nsGkAtoms::crossorigin, aCrossOrigin, aError);
447 uint16_t NetworkState() const { return mNetworkState; }
449 void NotifyXPCOMShutdown() final;
451 // Called by media decoder when the audible state changed or when input is
452 // a media stream.
453 void SetAudibleState(bool aAudible) final;
455 // Notify agent when the MediaElement changes its audible state.
456 void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason);
458 void GetPreload(nsAString& aValue) {
459 if (mSrcAttrStream) {
460 nsGkAtoms::none->ToString(aValue);
461 return;
463 GetEnumAttr(nsGkAtoms::preload, nullptr, aValue);
465 void SetPreload(const nsAString& aValue, ErrorResult& aRv) {
466 if (mSrcAttrStream) {
467 return;
469 SetHTMLAttr(nsGkAtoms::preload, aValue, aRv);
472 already_AddRefed<TimeRanges> Buffered() const;
474 void Load();
476 void CanPlayType(const nsAString& aType, nsAString& aResult);
478 uint16_t ReadyState() const { return mReadyState; }
480 bool Seeking() const;
482 double CurrentTime() const;
484 void SetCurrentTime(double aCurrentTime, ErrorResult& aRv);
485 void SetCurrentTime(double aCurrentTime) {
486 SetCurrentTime(aCurrentTime, IgnoreErrors());
489 void FastSeek(double aTime, ErrorResult& aRv);
491 already_AddRefed<Promise> SeekToNextFrame(ErrorResult& aRv);
493 double Duration() const;
495 bool HasAudio() const { return mMediaInfo.HasAudio(); }
497 virtual bool IsVideo() const { return false; }
499 bool HasVideo() const { return mMediaInfo.HasVideo(); }
501 bool IsEncrypted() const { return mIsEncrypted; }
503 bool Paused() const { return mPaused; }
505 double DefaultPlaybackRate() const {
506 if (mSrcAttrStream) {
507 return 1.0;
509 return mDefaultPlaybackRate;
512 void SetDefaultPlaybackRate(double aDefaultPlaybackRate, ErrorResult& aRv);
514 double PlaybackRate() const {
515 if (mSrcAttrStream) {
516 return 1.0;
518 return mPlaybackRate;
521 void SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv);
523 already_AddRefed<TimeRanges> Played();
525 already_AddRefed<TimeRanges> Seekable() const;
527 bool Ended();
529 bool Autoplay() const { return GetBoolAttr(nsGkAtoms::autoplay); }
531 void SetAutoplay(bool aValue, ErrorResult& aRv) {
532 SetHTMLBoolAttr(nsGkAtoms::autoplay, aValue, aRv);
535 bool Loop() const { return GetBoolAttr(nsGkAtoms::loop); }
537 void SetLoop(bool aValue, ErrorResult& aRv) {
538 SetHTMLBoolAttr(nsGkAtoms::loop, aValue, aRv);
541 already_AddRefed<Promise> Play(ErrorResult& aRv);
542 void Play() {
543 IgnoredErrorResult dummy;
544 RefPtr<Promise> toBeIgnored = Play(dummy);
547 void Pause(ErrorResult& aRv);
548 void Pause() { Pause(IgnoreErrors()); }
550 bool Controls() const { return GetBoolAttr(nsGkAtoms::controls); }
552 void SetControls(bool aValue, ErrorResult& aRv) {
553 SetHTMLBoolAttr(nsGkAtoms::controls, aValue, aRv);
556 double Volume() const { return mVolume; }
558 void SetVolume(double aVolume, ErrorResult& aRv);
560 bool Muted() const { return mMuted & MUTED_BY_CONTENT; }
561 void SetMuted(bool aMuted);
563 bool DefaultMuted() const { return GetBoolAttr(nsGkAtoms::muted); }
565 void SetDefaultMuted(bool aMuted, ErrorResult& aRv) {
566 SetHTMLBoolAttr(nsGkAtoms::muted, aMuted, aRv);
569 bool MozAllowCasting() const { return mAllowCasting; }
571 void SetMozAllowCasting(bool aShow) { mAllowCasting = aShow; }
573 bool MozIsCasting() const { return mIsCasting; }
575 void SetMozIsCasting(bool aShow) { mIsCasting = aShow; }
577 // Returns whether a call to Play() would be rejected with NotAllowedError.
578 // This assumes "worst case" for unknowns. So if prompting for permission is
579 // enabled and no permission is stored, this behaves as if the user would
580 // opt to block.
581 bool AllowedToPlay() const;
583 already_AddRefed<MediaSource> GetMozMediaSourceObject() const;
585 // Returns a promise which will be resolved after collecting debugging
586 // data from decoder/reader/MDSM. Used for debugging purposes.
587 already_AddRefed<Promise> MozRequestDebugInfo(ErrorResult& aRv);
589 // Enables DecoderDoctorLogger logging. Used for debugging purposes.
590 static void MozEnableDebugLog(const GlobalObject&);
592 // Returns a promise which will be resolved after collecting debugging
593 // log associated with this element. Used for debugging purposes.
594 already_AddRefed<Promise> MozRequestDebugLog(ErrorResult& aRv);
596 // For use by mochitests. Enabling pref "media.test.video-suspend"
597 void SetVisible(bool aVisible);
599 // For use by mochitests. Enabling pref "media.test.video-suspend"
600 bool HasSuspendTaint() const;
602 // For use by mochitests.
603 bool IsVideoDecodingSuspended() const;
605 // For use by mochitests only.
606 bool IsVisible() const;
608 // Synchronously, return the next video frame and mark the element unable to
609 // participate in decode suspending.
611 // This function is synchronous for cases where decoding has been suspended
612 // and JS needs a frame to use in, eg., nsLayoutUtils::SurfaceFromElement()
613 // via drawImage().
614 already_AddRefed<layers::Image> GetCurrentImage();
616 already_AddRefed<DOMMediaStream> GetSrcObject() const;
617 void SetSrcObject(DOMMediaStream& aValue);
618 void SetSrcObject(DOMMediaStream* aValue);
620 bool MozPreservesPitch() const { return mPreservesPitch; }
621 void SetMozPreservesPitch(bool aPreservesPitch);
623 MediaKeys* GetMediaKeys() const;
625 already_AddRefed<Promise> SetMediaKeys(MediaKeys* mediaKeys,
626 ErrorResult& aRv);
628 mozilla::dom::EventHandlerNonNull* GetOnencrypted();
629 void SetOnencrypted(mozilla::dom::EventHandlerNonNull* aCallback);
631 mozilla::dom::EventHandlerNonNull* GetOnwaitingforkey();
632 void SetOnwaitingforkey(mozilla::dom::EventHandlerNonNull* aCallback);
634 void DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
635 const nsAString& aInitDataType) override;
637 bool IsEventAttributeNameInternal(nsAtom* aName) override;
639 bool ContainsRestrictedContent();
641 void NotifyWaitingForKey() override;
643 already_AddRefed<DOMMediaStream> CaptureAudio(ErrorResult& aRv,
644 MediaTrackGraph* aGraph);
646 already_AddRefed<DOMMediaStream> MozCaptureStream(ErrorResult& aRv);
648 already_AddRefed<DOMMediaStream> MozCaptureStreamUntilEnded(ErrorResult& aRv);
650 bool MozAudioCaptured() const { return mAudioCaptured; }
652 void MozGetMetadata(JSContext* aCx, JS::MutableHandle<JSObject*> aResult,
653 ErrorResult& aRv);
655 double MozFragmentEnd();
657 AudioTrackList* AudioTracks();
659 VideoTrackList* VideoTracks();
661 TextTrackList* GetTextTracks();
663 already_AddRefed<TextTrack> AddTextTrack(TextTrackKind aKind,
664 const nsAString& aLabel,
665 const nsAString& aLanguage);
667 void AddTextTrack(TextTrack* aTextTrack) {
668 GetOrCreateTextTrackManager()->AddTextTrack(aTextTrack);
671 void RemoveTextTrack(TextTrack* aTextTrack, bool aPendingListOnly = false) {
672 if (mTextTrackManager) {
673 mTextTrackManager->RemoveTextTrack(aTextTrack, aPendingListOnly);
677 void NotifyCueAdded(TextTrackCue& aCue) {
678 if (mTextTrackManager) {
679 mTextTrackManager->NotifyCueAdded(aCue);
682 void NotifyCueRemoved(TextTrackCue& aCue) {
683 if (mTextTrackManager) {
684 mTextTrackManager->NotifyCueRemoved(aCue);
687 void NotifyCueUpdated(TextTrackCue* aCue) {
688 if (mTextTrackManager) {
689 mTextTrackManager->NotifyCueUpdated(aCue);
693 void NotifyCueDisplayStatesChanged();
695 bool IsBlessed() const { return mIsBlessed; }
697 // A method to check whether we are currently playing.
698 bool IsCurrentlyPlaying() const;
700 // Returns true if the media element is being destroyed. Used in
701 // dormancy checks to prevent dormant processing for an element
702 // that will soon be gone.
703 bool IsBeingDestroyed();
705 void OnVisibilityChange(Visibility aNewVisibility);
707 // Begin testing only methods
708 float ComputedVolume() const;
709 bool ComputedMuted() const;
711 // Return true if the media has been suspended media due to an inactive
712 // document or prohibiting by the docshell.
713 bool IsSuspendedByInactiveDocOrDocShell() const;
714 // End testing only methods
716 void SetMediaInfo(const MediaInfo& aInfo);
718 AbstractThread* AbstractMainThread() const final;
720 // Telemetry: to record the usage of a {visible / invisible} video element as
721 // the source of {drawImage(), createPattern(), createImageBitmap() and
722 // captureStream()} APIs.
723 enum class CallerAPI {
724 DRAW_IMAGE,
725 CREATE_PATTERN,
726 CREATE_IMAGEBITMAP,
727 CAPTURE_STREAM,
729 void MarkAsContentSource(CallerAPI aAPI);
731 Document* GetDocument() const override;
733 already_AddRefed<GMPCrashHelper> CreateGMPCrashHelper() override;
735 nsISerialEventTarget* MainThreadEventTarget() {
736 return mMainThreadEventTarget;
739 // Set the sink id (of the output device) that the audio will play. If aSinkId
740 // is empty the default device will be set.
741 already_AddRefed<Promise> SetSinkId(const nsAString& aSinkId,
742 ErrorResult& aRv);
743 // Get the sink id of the device that audio is being played. Initial value is
744 // empty and the default device is being used.
745 void GetSinkId(nsString& aSinkId) {
746 MOZ_ASSERT(NS_IsMainThread());
747 aSinkId = mSink.first;
750 // This is used to notify MediaElementAudioSourceNode that media element is
751 // allowed to play when media element is used as a source for web audio, so
752 // that we can start AudioContext if it was not allowed to start.
753 RefPtr<GenericNonExclusivePromise> GetAllowedToPlayPromise();
755 bool GetShowPosterFlag() const { return mShowPoster; }
757 bool IsAudible() const;
759 protected:
760 virtual ~HTMLMediaElement();
762 class AudioChannelAgentCallback;
763 class ChannelLoader;
764 class ErrorSink;
765 class MediaElementTrackSource;
766 class MediaLoadListener;
767 class MediaStreamRenderer;
768 class MediaStreamTrackListener;
769 class FirstFrameListener;
770 class ShutdownObserver;
771 class MediaControlKeyListener;
773 MediaDecoderOwner::NextFrameStatus NextFrameStatus();
775 void SetDecoder(MediaDecoder* aDecoder);
777 void PlayInternal(bool aHandlingUserInput);
779 // See spec, https://html.spec.whatwg.org/#internal-pause-steps
780 void PauseInternal();
782 /** Use this method to change the mReadyState member, so required
783 * events can be fired.
785 void ChangeReadyState(nsMediaReadyState aState);
788 * Use this method to change the mNetworkState member, so required
789 * actions will be taken during the transition.
791 void ChangeNetworkState(nsMediaNetworkState aState);
794 * The MediaElement will be responsible for creating and releasing the audio
795 * wakelock depending on the playing and audible state.
797 virtual void WakeLockRelease();
798 virtual void UpdateWakeLock();
800 void CreateAudioWakeLockIfNeeded();
801 void ReleaseAudioWakeLockIfExists();
802 RefPtr<WakeLock> mWakeLock;
805 * Logs a warning message to the web console to report various failures.
806 * aMsg is the localized message identifier, aParams is the parameters to
807 * be substituted into the localized message, and aParamCount is the number
808 * of parameters in aParams.
810 void ReportLoadError(const char* aMsg, const nsTArray<nsString>& aParams =
811 nsTArray<nsString>());
814 * Log message to web console.
816 void ReportToConsole(
817 uint32_t aErrorFlags, const char* aMsg,
818 const nsTArray<nsString>& aParams = nsTArray<nsString>()) const;
821 * Changes mHasPlayedOrSeeked to aValue. If mHasPlayedOrSeeked changes
822 * we'll force a reflow so that the video frame gets reflowed to reflect
823 * the poster hiding or showing immediately.
825 void SetPlayedOrSeeked(bool aValue);
828 * Initialize the media element for playback of aStream
830 void SetupSrcMediaStreamPlayback(DOMMediaStream* aStream);
832 * Stop playback on mSrcStream.
834 void EndSrcMediaStreamPlayback();
836 * Ensure we're playing mSrcStream if and only if we're not paused.
838 enum { REMOVING_SRC_STREAM = 0x1 };
839 void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0);
842 * Ensure currentTime progresses if and only if we're potentially playing
843 * mSrcStream. Called by the watch manager while we're playing mSrcStream, and
844 * one of the inputs to the potentially playing algorithm changes.
846 void UpdateSrcStreamPotentiallyPlaying();
849 * mSrcStream's graph's CurrentTime() has been updated. It might be time to
850 * fire "timeupdate".
852 void UpdateSrcStreamTime();
855 * Called after a tail dispatch when playback of mSrcStream ended, to comply
856 * with the spec where we must start reporting true for the ended attribute
857 * after the event loop returns to step 1. A MediaStream could otherwise be
858 * manipulated to end a HTMLMediaElement synchronously.
860 void UpdateSrcStreamReportPlaybackEnded();
863 * Called by our DOMMediaStream::TrackListener when a new MediaStreamTrack has
864 * been added to the playback stream of |mSrcStream|.
866 void NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aTrack);
869 * Called by our DOMMediaStream::TrackListener when a MediaStreamTrack in
870 * |mSrcStream|'s playback stream has ended.
872 void NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack);
875 * Convenience method to get in a single list all enabled AudioTracks and, if
876 * this is a video element, the selected VideoTrack.
878 void GetAllEnabledMediaTracks(nsTArray<RefPtr<MediaTrack>>& aTracks);
881 * Enables or disables all tracks forwarded from mSrcStream to all
882 * OutputMediaStreams. We do this for muting the tracks when pausing,
883 * and unmuting when playing the media element again.
885 void SetCapturedOutputStreamsEnabled(bool aEnabled);
888 * Returns true if output tracks should be muted, based on the state of this
889 * media element.
891 enum class OutputMuteState { Muted, Unmuted };
892 OutputMuteState OutputTracksMuted();
895 * Sets the muted state of all output track sources. They are muted when we're
896 * paused and unmuted otherwise.
898 void UpdateOutputTracksMuting();
901 * Create a new MediaStreamTrack for the TrackSource corresponding to aTrack
902 * and add it to the DOMMediaStream in aOutputStream. This automatically sets
903 * the output track to enabled or disabled depending on our current playing
904 * state.
906 enum class AddTrackMode { ASYNC, SYNC };
907 void AddOutputTrackSourceToOutputStream(
908 MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
909 AddTrackMode aMode = AddTrackMode::ASYNC);
912 * Creates output track sources when this media element is captured, tracks
913 * exist, playback is not ended and readyState is >= HAVE_METADATA.
915 void UpdateOutputTrackSources();
918 * Returns an DOMMediaStream containing the played contents of this
919 * element. When aBehavior is FINISH_WHEN_ENDED, when this element ends
920 * playback we will finish the stream and not play any more into it. When
921 * aType is CONTINUE_WHEN_ENDED, ending playback does not finish the stream.
922 * The stream will never finish.
924 * When aType is CAPTURE_AUDIO, we stop playout of audio and instead route it
925 * to the DOMMediaStream. Volume and mute state will be applied to the audio
926 * reaching the stream. No video tracks will be captured in this case.
928 already_AddRefed<DOMMediaStream> CaptureStreamInternal(
929 StreamCaptureBehavior aFinishBehavior,
930 StreamCaptureType aStreamCaptureType, MediaTrackGraph* aGraph);
933 * Initialize a decoder as a clone of an existing decoder in another
934 * element.
935 * mLoadingSrc must already be set.
937 nsresult InitializeDecoderAsClone(ChannelMediaDecoder* aOriginal);
940 * Call Load() and FinishDecoderSetup() on the decoder. It also handle
941 * resource cloning if DecoderType is ChannelMediaDecoder.
943 template <typename DecoderType, typename... LoadArgs>
944 nsresult SetupDecoder(DecoderType* aDecoder, LoadArgs&&... aArgs);
947 * Initialize a decoder to load the given channel. The decoder's stream
948 * listener is returned via aListener.
949 * mLoadingSrc must already be set.
951 nsresult InitializeDecoderForChannel(nsIChannel* aChannel,
952 nsIStreamListener** aListener);
955 * Finish setting up the decoder after Load() has been called on it.
956 * Called by InitializeDecoderForChannel/InitializeDecoderAsClone.
958 nsresult FinishDecoderSetup(MediaDecoder* aDecoder);
961 * Call this after setting up mLoadingSrc and mDecoder.
963 void AddMediaElementToURITable();
965 * Call this before modifying mLoadingSrc.
967 void RemoveMediaElementFromURITable();
969 * Call this to find a media element with the same NodePrincipal and
970 * mLoadingSrc set to aURI, and with a decoder on which Load() has been
971 * called.
973 HTMLMediaElement* LookupMediaElementURITable(nsIURI* aURI);
976 * Shutdown and clear mDecoder and maintain associated invariants.
978 void ShutdownDecoder();
980 * Execute the initial steps of the load algorithm that ensure existing
981 * loads are aborted, the element is emptied, and a new load ID is
982 * created.
984 void AbortExistingLoads();
987 * This is the dedicated media source failure steps.
988 * Called when all potential resources are exhausted. Changes network
989 * state to NETWORK_NO_SOURCE, and sends error event with code
990 * MEDIA_ERR_SRC_NOT_SUPPORTED.
992 void NoSupportedMediaSourceError(
993 const nsACString& aErrorDetails = nsCString());
996 * Per spec, Failed with elements: Queue a task, using the DOM manipulation
997 * task source, to fire a simple event named error at the candidate element.
998 * So dispatch |QueueLoadFromSourceTask| to main thread to make sure the task
999 * will be executed later than loadstart event.
1001 void DealWithFailedElement(nsIContent* aSourceElement);
1004 * Attempts to load resources from the <source> children. This is a
1005 * substep of the resource selection algorithm. Do not call this directly,
1006 * call QueueLoadFromSourceTask() instead.
1008 void LoadFromSourceChildren();
1011 * Asynchronously awaits a stable state, and then causes
1012 * LoadFromSourceChildren() to be called on the main threads' event loop.
1014 void QueueLoadFromSourceTask();
1017 * Runs the media resource selection algorithm.
1019 void SelectResource();
1022 * A wrapper function that allows us to cleanly reset flags after a call
1023 * to SelectResource()
1025 void SelectResourceWrapper();
1028 * Asynchronously awaits a stable state, and then causes SelectResource()
1029 * to be run on the main thread's event loop.
1031 void QueueSelectResourceTask();
1034 * When loading a new source on an existing media element, make sure to reset
1035 * everything that is accessible using the media element API.
1037 void ResetState();
1040 * The resource-fetch algorithm step of the load algorithm.
1042 MediaResult LoadResource();
1045 * Selects the next <source> child from which to load a resource. Called
1046 * during the resource selection algorithm. Stores the return value in
1047 * mSourceLoadCandidate before returning.
1049 Element* GetNextSource();
1052 * Changes mDelayingLoadEvent, and will call BlockOnLoad()/UnblockOnLoad()
1053 * on the owning document, so it can delay the load event firing.
1055 void ChangeDelayLoadStatus(bool aDelay);
1058 * If we suspended downloading after the first frame, unsuspend now.
1060 void StopSuspendingAfterFirstFrame();
1063 * Called when our channel is redirected to another channel.
1064 * Updates our mChannel reference to aNewChannel.
1066 nsresult OnChannelRedirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
1067 uint32_t aFlags);
1070 * Call this to reevaluate whether we should be holding a self-reference.
1072 void AddRemoveSelfReference();
1075 * Called when "xpcom-shutdown" event is received.
1077 void NotifyShutdownEvent();
1080 * Possible values of the 'preload' attribute.
1082 enum PreloadAttrValue : uint8_t {
1083 PRELOAD_ATTR_EMPTY, // set to ""
1084 PRELOAD_ATTR_NONE, // set to "none"
1085 PRELOAD_ATTR_METADATA, // set to "metadata"
1086 PRELOAD_ATTR_AUTO // set to "auto"
1090 * The preloading action to perform. These dictate how we react to the
1091 * preload attribute. See mPreloadAction.
1093 enum PreloadAction {
1094 PRELOAD_UNDEFINED = 0, // not determined - used only for initialization
1095 PRELOAD_NONE = 1, // do not preload
1096 PRELOAD_METADATA = 2, // preload only the metadata (and first frame)
1097 PRELOAD_ENOUGH = 3 // preload enough data to allow uninterrupted
1098 // playback
1102 * The guts of Load(). Load() acts as a wrapper around this which sets
1103 * mIsDoingExplicitLoad to true so that when script calls 'load()'
1104 * preload-none will be automatically upgraded to preload-metadata.
1106 void DoLoad();
1109 * Suspends the load of mLoadingSrc, so that it can be resumed later
1110 * by ResumeLoad(). This is called when we have a media with a 'preload'
1111 * attribute value of 'none', during the resource selection algorithm.
1113 void SuspendLoad();
1116 * Resumes a previously suspended load (suspended by SuspendLoad(uri)).
1117 * Will continue running the resource selection algorithm.
1118 * Sets mPreloadAction to aAction.
1120 void ResumeLoad(PreloadAction aAction);
1123 * Handle a change to the preload attribute. Should be called whenever the
1124 * value (or presence) of the preload attribute changes. The change in
1125 * attribute value may cause a change in the mPreloadAction of this
1126 * element. If there is a change then this method will initiate any
1127 * behaviour that is necessary to implement the action.
1129 void UpdatePreloadAction();
1132 * Fire progress events if needed according to the time and byte constraints
1133 * outlined in the specification. aHaveNewProgress is true if progress has
1134 * just been detected. Otherwise the method is called as a result of the
1135 * progress timer.
1137 void CheckProgress(bool aHaveNewProgress);
1138 static void ProgressTimerCallback(nsITimer* aTimer, void* aClosure);
1140 * Start timer to update download progress.
1142 void StartProgressTimer();
1144 * Start sending progress and/or stalled events.
1146 void StartProgress();
1148 * Stop progress information timer and events.
1150 void StopProgress();
1153 * Dispatches an error event to a child source element.
1155 void DispatchAsyncSourceError(nsIContent* aSourceElement);
1158 * Resets the media element for an error condition as per aErrorCode.
1159 * aErrorCode must be one of WebIDL HTMLMediaElement error codes.
1161 void Error(uint16_t aErrorCode,
1162 const nsACString& aErrorDetails = nsCString());
1165 * Returns the URL spec of the currentSrc.
1167 void GetCurrentSpec(nsCString& aString);
1170 * Process any media fragment entries in the URI
1172 void ProcessMediaFragmentURI();
1175 * Mute or unmute the audio and change the value that the |muted| map.
1177 void SetMutedInternal(uint32_t aMuted);
1179 * Update the volume of the output audio stream to match the element's
1180 * current mMuted/mVolume/mAudioChannelFaded state.
1182 void SetVolumeInternal();
1185 * Suspend or resume element playback and resource download. When we suspend
1186 * playback, event delivery would also be suspended (and events queued) until
1187 * the element is resumed.
1189 void SuspendOrResumeElement(bool aSuspendElement);
1191 // Get the HTMLMediaElement object if the decoder is being used from an
1192 // HTML media element, and null otherwise.
1193 HTMLMediaElement* GetMediaElement() final { return this; }
1195 // Return true if decoding should be paused
1196 bool GetPaused() final { return Paused(); }
1199 * Video has been playing while hidden and, if feature was enabled, would
1200 * trigger suspending decoder.
1201 * Used to track hidden-video-decode-suspend telemetry.
1203 static void VideoDecodeSuspendTimerCallback(nsITimer* aTimer, void* aClosure);
1205 * Video is now both: playing and hidden.
1206 * Used to track hidden-video telemetry.
1208 void HiddenVideoStart();
1210 * Video is not playing anymore and/or has become visible.
1211 * Used to track hidden-video telemetry.
1213 void HiddenVideoStop();
1215 void ReportTelemetry();
1217 // Seeks to aTime seconds. aSeekType can be Exact to seek to exactly the
1218 // seek target, or PrevSyncPoint if a quicker but less precise seek is
1219 // desired, and we'll seek to the sync point (keyframe and/or start of the
1220 // next block of audio samples) preceeding seek target.
1221 void Seek(double aTime, SeekTarget::Type aSeekType, ErrorResult& aRv);
1223 // Update the audio channel playing state
1224 void UpdateAudioChannelPlayingState();
1226 // Adds to the element's list of pending text tracks each text track
1227 // in the element's list of text tracks whose text track mode is not disabled
1228 // and whose text track readiness state is loading.
1229 void PopulatePendingTextTrackList();
1231 // Gets a reference to the MediaElement's TextTrackManager. If the
1232 // MediaElement doesn't yet have one then it will create it.
1233 TextTrackManager* GetOrCreateTextTrackManager();
1235 // Recomputes ready state and fires events as necessary based on current
1236 // state.
1237 void UpdateReadyStateInternal();
1239 // Create or destroy the captured stream.
1240 void AudioCaptureTrackChange(bool aCapture);
1242 // If the network state is empty and then we would trigger DoLoad().
1243 void MaybeDoLoad();
1245 // Anything we need to check after played success and not related with spec.
1246 void UpdateCustomPolicyAfterPlayed();
1248 // Returns a StreamCaptureType populated with the right bits, depending on the
1249 // tracks this HTMLMediaElement has.
1250 StreamCaptureType CaptureTypeForElement();
1252 // True if this element can be captured, false otherwise.
1253 bool CanBeCaptured(StreamCaptureType aCaptureType);
1255 class nsAsyncEventRunner;
1256 class nsNotifyAboutPlayingRunner;
1257 class nsResolveOrRejectPendingPlayPromisesRunner;
1258 using nsGenericHTMLElement::DispatchEvent;
1259 // For nsAsyncEventRunner.
1260 nsresult DispatchEvent(const nsAString& aName);
1262 // This method moves the mPendingPlayPromises into a temperate object. So the
1263 // mPendingPlayPromises is cleared after this method call.
1264 nsTArray<RefPtr<PlayPromise>> TakePendingPlayPromises();
1266 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1267 // and queues a task to resolve them.
1268 void AsyncResolvePendingPlayPromises();
1270 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1271 // and queues a task to reject them.
1272 void AsyncRejectPendingPlayPromises(nsresult aError);
1274 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1275 // and queues a task to resolve them also to dispatch a "playing" event.
1276 void NotifyAboutPlaying();
1278 already_AddRefed<Promise> CreateDOMPromise(ErrorResult& aRv) const;
1280 // Pass information for deciding the video decode mode to decoder.
1281 void NotifyDecoderActivityChanges() const;
1283 // Constructs an AudioTrack in mAudioTrackList if aInfo reports that audio is
1284 // available, and a VideoTrack in mVideoTrackList if aInfo reports that video
1285 // is available.
1286 void ConstructMediaTracks(const MediaInfo* aInfo);
1288 // Removes all MediaTracks from mAudioTrackList and mVideoTrackList and fires
1289 // "removetrack" on the lists accordingly.
1290 // Note that by spec, this should not fire "removetrack". However, it appears
1291 // other user agents do, per
1292 // https://wpt.fyi/results/media-source/mediasource-avtracks.html.
1293 void RemoveMediaTracks();
1295 // Mark the decoder owned by the element as tainted so that the
1296 // suspend-video-decoder is disabled.
1297 void MarkAsTainted();
1299 virtual nsresult AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
1300 const nsAttrValue* aValue,
1301 const nsAttrValue* aOldValue,
1302 nsIPrincipal* aMaybeScriptedPrincipal,
1303 bool aNotify) override;
1304 virtual nsresult OnAttrSetButNotChanged(int32_t aNamespaceID, nsAtom* aName,
1305 const nsAttrValueOrString& aValue,
1306 bool aNotify) override;
1308 bool DetachExistingMediaKeys();
1309 bool TryRemoveMediaKeysAssociation();
1310 void RemoveMediaKeys();
1311 bool AttachNewMediaKeys();
1312 bool TryMakeAssociationWithCDM(CDMProxy* aProxy);
1313 void MakeAssociationWithCDMResolved();
1314 void SetCDMProxyFailure(const MediaResult& aResult);
1315 void ResetSetMediaKeysTempVariables();
1317 void PauseIfShouldNotBePlaying();
1319 WatchManager<HTMLMediaElement> mWatchManager;
1321 // Update the silence range of the audio track when the audible status of
1322 // silent audio track changes or seeking to the new position where the audio
1323 // track is silent.
1324 void UpdateAudioTrackSilenceRange(bool aAudible);
1326 // When silent audio track becomes audible or seeking to new place, we would
1327 // end the current silence range and accumulate it to the total silence
1328 // proportion of audio track and update current silence range.
1329 void AccumulateAudioTrackSilence();
1331 // True when the media element's audio track is containing silence now.
1332 bool IsAudioTrackCurrentlySilent() const;
1334 // Calculate the audio track silence proportion and then report the telemetry
1335 // result. we would report the result when decoder is destroyed.
1336 void ReportAudioTrackSilenceProportionTelemetry();
1338 // When the play is not allowed, dispatch related events which are used for
1339 // testing or changing control UI.
1340 void DispatchEventsWhenPlayWasNotAllowed();
1342 // When the doc is blocked permanantly, we would dispatch event to notify
1343 // front-end side to show blocking icon.
1344 void MaybeNotifyAutoplayBlocked();
1346 // When playing state change, we have to notify MediaControl in the chrome
1347 // process in order to keep its playing state correct.
1348 void NotifyMediaControlPlaybackStateChanged();
1350 // Clear the timer when we want to continue listening to the media control
1351 // key events.
1352 void ClearStopMediaControlTimerIfNeeded();
1354 // This function is used to update the status of media control when the media
1355 // changes its status of being used in the Picture-in-Picture mode.
1356 void UpdateMediaControlAfterPictureInPictureModeChanged();
1358 // The current decoder. Load() has been called on this decoder.
1359 // At most one of mDecoder and mSrcStream can be non-null.
1360 RefPtr<MediaDecoder> mDecoder;
1362 // The DocGroup-specific nsISerialEventTarget of this HTML element on the main
1363 // thread.
1364 nsCOMPtr<nsISerialEventTarget> mMainThreadEventTarget;
1366 // The DocGroup-specific AbstractThread::MainThread() of this HTML element.
1367 RefPtr<AbstractThread> mAbstractMainThread;
1369 // A reference to the VideoFrameContainer which contains the current frame
1370 // of video to display.
1371 RefPtr<VideoFrameContainer> mVideoFrameContainer;
1373 // Holds a reference to the MediaStream that has been set in the src
1374 // attribute.
1375 RefPtr<DOMMediaStream> mSrcAttrStream;
1377 // Holds the triggering principal for the src attribute.
1378 nsCOMPtr<nsIPrincipal> mSrcAttrTriggeringPrincipal;
1380 // Holds a reference to the MediaStream that we're actually playing.
1381 // At most one of mDecoder and mSrcStream can be non-null.
1382 RefPtr<DOMMediaStream> mSrcStream;
1384 // The MediaStreamRenderer handles rendering of our selected video track, and
1385 // enabled audio tracks, while mSrcStream is set.
1386 RefPtr<MediaStreamRenderer> mMediaStreamRenderer;
1388 // True once PlaybackEnded() is called and we're playing a MediaStream.
1389 // Reset to false if we start playing mSrcStream again.
1390 Watchable<bool> mSrcStreamPlaybackEnded = {
1391 false, "HTMLMediaElement::mSrcStreamPlaybackEnded"};
1393 // Mirrors mSrcStreamPlaybackEnded after a tail dispatch when set to true,
1394 // but may be be forced to false directly. To accomodate when an application
1395 // ends playback synchronously by manipulating mSrcStream or its tracks,
1396 // e.g., through MediaStream.removeTrack(), or MediaStreamTrack.stop().
1397 bool mSrcStreamReportPlaybackEnded = false;
1399 // Holds a reference to the stream connecting this stream to the window
1400 // capture sink.
1401 UniquePtr<MediaStreamWindowCapturer> mStreamWindowCapturer;
1403 // Holds references to the DOM wrappers for the MediaStreams that we're
1404 // writing to.
1405 nsTArray<OutputMediaStream> mOutputStreams;
1407 // Mapping for output tracks, from dom::MediaTrack ids to the
1408 // MediaElementTrackSource that represents the source of all corresponding
1409 // MediaStreamTracks captured from this element.
1410 nsRefPtrHashtable<nsStringHashKey, MediaElementTrackSource>
1411 mOutputTrackSources;
1413 // Holds a reference to the first-frame-getting track listener attached to
1414 // mSelectedVideoStreamTrack.
1415 RefPtr<FirstFrameListener> mFirstFrameListener;
1416 // The currently selected video stream track.
1417 RefPtr<VideoStreamTrack> mSelectedVideoStreamTrack;
1419 const RefPtr<ShutdownObserver> mShutdownObserver;
1421 // Holds a reference to the MediaSource, if any, referenced by the src
1422 // attribute on the media element.
1423 RefPtr<MediaSource> mSrcMediaSource;
1425 // Holds a reference to the MediaSource supplying data for playback. This
1426 // may either match mSrcMediaSource or come from Source element children.
1427 // This is set when and only when mLoadingSrc corresponds to an object url
1428 // that resolved to a MediaSource.
1429 RefPtr<MediaSource> mMediaSource;
1431 RefPtr<ChannelLoader> mChannelLoader;
1433 // Points to the child source elements, used to iterate through the children
1434 // when selecting a resource to load. This is the previous sibling of the
1435 // child considered the current 'candidate' in:
1436 // https://html.spec.whatwg.org/multipage/media.html#concept-media-load-algorithm
1438 // mSourcePointer == nullptr, we will next try to load |GetFirstChild()|.
1439 // mSourcePointer == GetLastChild(), we've exhausted all sources, waiting
1440 // for new elements to be appended.
1441 nsCOMPtr<nsIContent> mSourcePointer;
1443 // Points to the document whose load we're blocking. This is the document
1444 // we're bound to when loading starts.
1445 nsCOMPtr<Document> mLoadBlockedDoc;
1447 // Contains names of events that have been raised while in the bfcache.
1448 // These events get re-dispatched when the bfcache is exited.
1449 nsTArray<nsString> mPendingEvents;
1451 // Media loading flags. See:
1452 // http://www.whatwg.org/specs/web-apps/current-work/#video)
1453 nsMediaNetworkState mNetworkState = HTMLMediaElement_Binding::NETWORK_EMPTY;
1454 Watchable<nsMediaReadyState> mReadyState = {
1455 HTMLMediaElement_Binding::HAVE_NOTHING, "HTMLMediaElement::mReadyState"};
1457 enum LoadAlgorithmState {
1458 // No load algorithm instance is waiting for a source to be added to the
1459 // media in order to continue loading.
1460 NOT_WAITING,
1461 // We've run the load algorithm, and we tried all source children of the
1462 // media element, and failed to load any successfully. We're waiting for
1463 // another source element to be added to the media element, and will try
1464 // to load any such element when its added.
1465 WAITING_FOR_SOURCE
1468 // The current media load ID. This is incremented every time we start a
1469 // new load. Async events note the ID when they're first sent, and only fire
1470 // if the ID is unchanged when they come to fire.
1471 uint32_t mCurrentLoadID = 0;
1473 // Denotes the waiting state of a load algorithm instance. When the load
1474 // algorithm is waiting for a source element child to be added, this is set
1475 // to WAITING_FOR_SOURCE, otherwise it's NOT_WAITING.
1476 LoadAlgorithmState mLoadWaitStatus = NOT_WAITING;
1478 // Current audio volume
1479 double mVolume = 1.0;
1481 // True if the audio track is not silent.
1482 bool mIsAudioTrackAudible = false;
1484 // Used to mark the start of the silence range of audio track.
1485 double mAudioTrackSilenceStartedTime = 0.0;
1487 // Save all the silence ranges, all ranges would be normalized. That means
1488 // intervals won't overlap or touch each other.
1489 media::TimeIntervals mSilenceTimeRanges;
1491 // True if we have calculated silence range before SeekEnd(). This attribute
1492 // would be reset after seeking completed.
1493 bool mHasAccumulatedSilenceRangeBeforeSeekEnd = false;
1495 enum MutedReasons {
1496 MUTED_BY_CONTENT = 0x01,
1497 MUTED_BY_INVALID_PLAYBACK_RATE = 0x02,
1498 MUTED_BY_AUDIO_CHANNEL = 0x04,
1499 MUTED_BY_AUDIO_TRACK = 0x08
1502 uint32_t mMuted = 0;
1504 UniquePtr<const MetadataTags> mTags;
1506 // URI of the resource we're attempting to load. This stores the value we
1507 // return in the currentSrc attribute. Use GetCurrentSrc() to access the
1508 // currentSrc attribute.
1509 // This is always the original URL we're trying to load --- before
1510 // redirects etc.
1511 nsCOMPtr<nsIURI> mLoadingSrc;
1513 // The triggering principal for the current source.
1514 nsCOMPtr<nsIPrincipal> mLoadingSrcTriggeringPrincipal;
1516 // Stores the current preload action for this element. Initially set to
1517 // PRELOAD_UNDEFINED, its value is changed by calling
1518 // UpdatePreloadAction().
1519 PreloadAction mPreloadAction = PRELOAD_UNDEFINED;
1521 // Time that the last timeupdate event was fired. Read/Write from the
1522 // main thread only.
1523 TimeStamp mTimeUpdateTime;
1525 // Time that the last progress event was fired. Read/Write from the
1526 // main thread only.
1527 TimeStamp mProgressTime;
1529 // Time that data was last read from the media resource. Used for
1530 // computing if the download has stalled and to rate limit progress events
1531 // when data is arriving slower than PROGRESS_MS.
1532 // Read/Write from the main thread only.
1533 TimeStamp mDataTime;
1535 // Media 'currentTime' value when the last timeupdate event occurred.
1536 // Read/Write from the main thread only.
1537 double mLastCurrentTime = 0.0;
1539 // Logical start time of the media resource in seconds as obtained
1540 // from any media fragments. A negative value indicates that no
1541 // fragment time has been set. Read/Write from the main thread only.
1542 double mFragmentStart = -1.0;
1544 // Logical end time of the media resource in seconds as obtained
1545 // from any media fragments. A negative value indicates that no
1546 // fragment time has been set. Read/Write from the main thread only.
1547 double mFragmentEnd = -1.0;
1549 // The defaultPlaybackRate attribute gives the desired speed at which the
1550 // media resource is to play, as a multiple of its intrinsic speed.
1551 double mDefaultPlaybackRate = 1.0;
1553 // The playbackRate attribute gives the speed at which the media resource
1554 // plays, as a multiple of its intrinsic speed. If it is not equal to the
1555 // defaultPlaybackRate, then the implication is that the user is using a
1556 // feature such as fast forward or slow motion playback.
1557 double mPlaybackRate = 1.0;
1559 // True if pitch correction is applied when playbackRate is set to a
1560 // non-intrinsic value.
1561 bool mPreservesPitch = true;
1563 // Reference to the source element last returned by GetNextSource().
1564 // This is the child source element which we're trying to load from.
1565 nsCOMPtr<nsIContent> mSourceLoadCandidate;
1567 // Range of time played.
1568 RefPtr<TimeRanges> mPlayed;
1570 // Timer used for updating progress events.
1571 nsCOMPtr<nsITimer> mProgressTimer;
1573 // Timer used to simulate video-suspend.
1574 nsCOMPtr<nsITimer> mVideoDecodeSuspendTimer;
1576 // Encrypted Media Extension media keys.
1577 RefPtr<MediaKeys> mMediaKeys;
1578 RefPtr<MediaKeys> mIncomingMediaKeys;
1579 // The dom promise is used for HTMLMediaElement::SetMediaKeys.
1580 RefPtr<DetailedPromise> mSetMediaKeysDOMPromise;
1581 // Used to indicate if the MediaKeys attaching operation is on-going or not.
1582 bool mAttachingMediaKey = false;
1583 MozPromiseRequestHolder<SetCDMPromise> mSetCDMRequest;
1585 // Stores the time at the start of the current 'played' range.
1586 double mCurrentPlayRangeStart = 1.0;
1588 // True if loadeddata has been fired.
1589 bool mLoadedDataFired = false;
1591 // Indicates whether current playback is a result of user action
1592 // (ie. calling of the Play method), or automatic playback due to
1593 // the 'autoplay' attribute being set. A true value indicates the
1594 // latter case.
1595 // The 'autoplay' HTML attribute indicates that the video should
1596 // start playing when loaded. The 'autoplay' attribute of the object
1597 // is a mirror of the HTML attribute. These are different from this
1598 // 'mAutoplaying' flag, which indicates whether the current playback
1599 // is a result of the autoplay attribute.
1600 bool mAutoplaying = true;
1602 // Playback of the video is paused either due to calling the
1603 // 'Pause' method, or playback not yet having started.
1604 Watchable<bool> mPaused = {true, "HTMLMediaElement::mPaused"};
1606 // The following two fields are here for the private storage of the builtin
1607 // video controls, and control 'casting' of the video to external devices
1608 // (TVs, projectors etc.)
1609 // True if casting is currently allowed
1610 bool mAllowCasting = false;
1611 // True if currently casting this video
1612 bool mIsCasting = false;
1614 // Set while there are some OutputMediaStreams this media element's enabled
1615 // and selected tracks are captured into. When set, all tracks are captured
1616 // into the graph of this dummy track.
1617 // NB: This is a SharedDummyTrack to allow non-default graphs (AudioContexts
1618 // with an explicit sampleRate defined) to capture this element. When
1619 // cross-graph tracks are supported, this can become a bool.
1620 Watchable<RefPtr<SharedDummyTrack>> mTracksCaptured;
1622 // True if the sound is being captured.
1623 bool mAudioCaptured = false;
1625 // If TRUE then the media element was actively playing before the currently
1626 // in progress seeking. If FALSE then the media element is either not seeking
1627 // or was not actively playing before the current seek. Used to decide whether
1628 // to raise the 'waiting' event as per 4.7.1.8 in HTML 5 specification.
1629 bool mPlayingBeforeSeek = false;
1631 // True if this element is suspended because the document is inactive or the
1632 // inactive docshell is not allowing media to play.
1633 bool mSuspendedByInactiveDocOrDocshell = false;
1635 // True if event delivery is suspended (mSuspendedByInactiveDocOrDocshell
1636 // must also be true).
1637 bool mEventDeliveryPaused = false;
1639 // True if we're running the "load()" method.
1640 bool mIsRunningLoadMethod = false;
1642 // True if we're running or waiting to run queued tasks due to an explicit
1643 // call to "load()".
1644 bool mIsDoingExplicitLoad = false;
1646 // True if we're loading the resource from the child source elements.
1647 bool mIsLoadingFromSourceChildren = false;
1649 // True if we're delaying the "load" event. They are delayed until either
1650 // an error occurs, or the first frame is loaded.
1651 bool mDelayingLoadEvent = false;
1653 // True when we've got a task queued to call SelectResource(),
1654 // or while we're running SelectResource().
1655 bool mIsRunningSelectResource = false;
1657 // True when we already have select resource call queued
1658 bool mHaveQueuedSelectResource = false;
1660 // True if we suspended the decoder because we were paused,
1661 // preloading metadata is enabled, autoplay was not enabled, and we loaded
1662 // the first frame.
1663 bool mSuspendedAfterFirstFrame = false;
1665 // True if we are allowed to suspend the decoder because we were paused,
1666 // preloading metdata was enabled, autoplay was not enabled, and we loaded
1667 // the first frame.
1668 bool mAllowSuspendAfterFirstFrame = true;
1670 // True if we've played or completed a seek. We use this to determine
1671 // when the poster frame should be shown.
1672 bool mHasPlayedOrSeeked = false;
1674 // True if we've added a reference to ourselves to keep the element
1675 // alive while no-one is referencing it but the element may still fire
1676 // events of its own accord.
1677 bool mHasSelfReference = false;
1679 // True if we've received a notification that the engine is shutting
1680 // down.
1681 bool mShuttingDown = false;
1683 // True if we've suspended a load in the resource selection algorithm
1684 // due to loading a preload:none media. When true, the resource we'll
1685 // load when the user initiates either playback or an explicit load is
1686 // stored in mPreloadURI.
1687 bool mSuspendedForPreloadNone = false;
1689 // True if we've connected mSrcStream to the media element output.
1690 bool mSrcStreamIsPlaying = false;
1692 // True if we should set nsIClassOfService::UrgentStart to the channel to
1693 // get the response ASAP for better user responsiveness.
1694 bool mUseUrgentStartForChannel = false;
1696 // The CORS mode when loading the media element
1697 CORSMode mCORSMode = CORS_NONE;
1699 // Info about the played media.
1700 MediaInfo mMediaInfo;
1702 // True if the media has encryption information.
1703 bool mIsEncrypted = false;
1705 enum WaitingForKeyState {
1706 NOT_WAITING_FOR_KEY = 0,
1707 WAITING_FOR_KEY = 1,
1708 WAITING_FOR_KEY_DISPATCHED = 2
1711 // True when the CDM cannot decrypt the current block due to lacking a key.
1712 // Note: the "waitingforkey" event is not dispatched until all decoded data
1713 // has been rendered.
1714 WaitingForKeyState mWaitingForKey = NOT_WAITING_FOR_KEY;
1716 // Listens for waitingForKey events from the owned decoder.
1717 MediaEventListener mWaitingForKeyListener;
1719 // Init Data that needs to be sent in 'encrypted' events in MetadataLoaded().
1720 EncryptionInfo mPendingEncryptedInitData;
1722 // True if the media's channel's download has been suspended.
1723 Watchable<bool> mDownloadSuspendedByCache = {
1724 false, "HTMLMediaElement::mDownloadSuspendedByCache"};
1726 // Disable the video playback by track selection. This flag might not be
1727 // enough if we ever expand the ability of supporting multi-tracks video
1728 // playback.
1729 bool mDisableVideo = false;
1731 RefPtr<TextTrackManager> mTextTrackManager;
1733 RefPtr<AudioTrackList> mAudioTrackList;
1735 RefPtr<VideoTrackList> mVideoTrackList;
1737 UniquePtr<MediaStreamTrackListener> mMediaStreamTrackListener;
1739 // The principal guarding mVideoFrameContainer access when playing a
1740 // MediaStream.
1741 nsCOMPtr<nsIPrincipal> mSrcStreamVideoPrincipal;
1743 // True if the autoplay media was blocked because it hadn't loaded metadata
1744 // yet.
1745 bool mBlockedAsWithoutMetadata = false;
1747 // This promise is used to notify MediaElementAudioSourceNode that media
1748 // element is allowed to play when MediaElement is used as a source for web
1749 // audio.
1750 MozPromiseHolder<GenericNonExclusivePromise> mAllowedToPlayPromise;
1752 // True if media has ever been blocked for autoplay, it's used to notify front
1753 // end to show the correct blocking icon when the document goes back from
1754 // bfcache.
1755 bool mHasEverBeenBlockedForAutoplay = false;
1757 // True if we have dispatched a task for text track changed, will be unset
1758 // when we starts processing text track changed.
1759 // https://html.spec.whatwg.org/multipage/media.html#pending-text-track-change-notification-flag
1760 bool mPendingTextTrackChanged = false;
1762 public:
1763 // This function will be called whenever a text track that is in a media
1764 // element's list of text tracks has its text track mode change value
1765 void NotifyTextTrackModeChanged();
1767 public:
1768 // Helper class to measure times for playback telemetry stats
1769 class TimeDurationAccumulator {
1770 public:
1771 TimeDurationAccumulator() : mCount(0) {}
1772 void Start() {
1773 if (IsStarted()) {
1774 return;
1776 mStartTime = TimeStamp::Now();
1778 void Pause() {
1779 if (!IsStarted()) {
1780 return;
1782 mSum += (TimeStamp::Now() - mStartTime);
1783 mCount++;
1784 mStartTime = TimeStamp();
1786 bool IsStarted() const { return !mStartTime.IsNull(); }
1787 double Total() const {
1788 if (!IsStarted()) {
1789 return mSum.ToSeconds();
1791 // Add current running time until now, but keep it running.
1792 return (mSum + (TimeStamp::Now() - mStartTime)).ToSeconds();
1794 uint32_t Count() const {
1795 if (!IsStarted()) {
1796 return mCount;
1798 // Count current run in this report, without increasing the stored count.
1799 return mCount + 1;
1801 void Reset() {
1802 mStartTime = TimeStamp();
1803 mSum = TimeDuration();
1804 mCount = 0;
1807 private:
1808 TimeStamp mStartTime;
1809 TimeDuration mSum;
1810 uint32_t mCount;
1813 private:
1814 already_AddRefed<PlayPromise> CreatePlayPromise(ErrorResult& aRv) const;
1816 void UpdateHadAudibleAutoplayState();
1818 virtual void MaybeBeginCloningVisually(){};
1820 uint32_t GetPreloadDefault() const;
1821 uint32_t GetPreloadDefaultAuto() const;
1824 * This function is called by AfterSetAttr and OnAttrSetButNotChanged.
1825 * It will not be called if the value is being unset.
1827 * @param aNamespaceID the namespace of the attr being set
1828 * @param aName the localname of the attribute being set
1829 * @param aNotify Whether we plan to notify document observers.
1831 void AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName, bool aNotify);
1833 // Total time a video has spent playing.
1834 TimeDurationAccumulator mPlayTime;
1836 // Total time a video has spent playing while hidden.
1837 TimeDurationAccumulator mHiddenPlayTime;
1839 // Total time a video has (or would have) spent in video-decode-suspend mode.
1840 TimeDurationAccumulator mVideoDecodeSuspendTime;
1842 // Total time a video has spent playing on the current load, it would be reset
1843 // when media aborts the current load; be paused when the docuemt enters the
1844 // bf-cache and be resumed when the docuemt leaves the bf-cache.
1845 TimeDurationAccumulator mCurrentLoadPlayTime;
1847 // True if media has ever been blocked by autoplay policy before.
1848 bool mHasPlayEverBeenBlocked = false;
1850 // Report the Telemetry about whether media played over the specific time
1851 // threshold.
1852 void ReportPlayedTimeAfterBlockedTelemetry();
1854 // True if Init() has been called after construction
1855 bool mInitialized = false;
1857 // True if user has called load(), seek() or element has started playing
1858 // before. It's *only* use for `click-to-play` blocking autoplay policy.
1859 // In addition, we would reset this once media aborts current load.
1860 bool mIsBlessed = false;
1862 // True if the first frame has been successfully loaded.
1863 Watchable<bool> mFirstFrameLoaded = {false,
1864 "HTMLMediaElement::mFirstFrameLoaded"};
1866 // Media elements also have a default playback start position, which must
1867 // initially be set to zero seconds. This time is used to allow the element to
1868 // be seeked even before the media is loaded.
1869 double mDefaultPlaybackStartPosition = 0.0;
1871 // True if media element has been marked as 'tainted' and can't
1872 // participate in video decoder suspending.
1873 bool mHasSuspendTaint = false;
1875 // True if media element has been forced into being considered 'hidden'.
1876 // For use by mochitests. Enabling pref "media.test.video-suspend"
1877 bool mForcedHidden = false;
1879 Visibility mVisibilityState = Visibility::Untracked;
1881 UniquePtr<ErrorSink> mErrorSink;
1883 // This wrapper will handle all audio channel related stuffs, eg. the
1884 // operations of tab audio indicator, Fennec's media control. Note:
1885 // mAudioChannelWrapper might be null after GC happened.
1886 RefPtr<AudioChannelAgentCallback> mAudioChannelWrapper;
1888 // A list of pending play promises. The elements are pushed during the play()
1889 // method call and are resolved/rejected during further playback steps.
1890 nsTArray<RefPtr<PlayPromise>> mPendingPlayPromises;
1892 // A list of already-dispatched but not yet run
1893 // nsResolveOrRejectPendingPlayPromisesRunners.
1894 // Runners whose Run() method is called remove themselves from this list.
1895 // We keep track of these because the load algorithm resolves/rejects all
1896 // already-dispatched pending play promises.
1897 nsTArray<nsResolveOrRejectPendingPlayPromisesRunner*>
1898 mPendingPlayPromisesRunners;
1900 // A pending seek promise which is created at Seek() method call and is
1901 // resolved/rejected at AsyncResolveSeekDOMPromiseIfExists()/
1902 // AsyncRejectSeekDOMPromiseIfExists() methods.
1903 RefPtr<dom::Promise> mSeekDOMPromise;
1905 // Return true if the docshell is inactive and explicitly wants to stop media
1906 // playing in that shell.
1907 bool ShouldBeSuspendedByInactiveDocShell() const;
1909 // For debugging bug 1407148.
1910 void AssertReadyStateIsNothing();
1912 // Contains the unique id of the sink device and the device info.
1913 // The initial value is ("", nullptr) and the default output device is used.
1914 // It can contain an invalid id and info if the device has been
1915 // unplugged. It can be set to ("", nullptr). It follows the spec attribute:
1916 // https://w3c.github.io/mediacapture-output/#htmlmediaelement-extensions
1917 // Read/Write from the main thread only.
1918 std::pair<nsString, RefPtr<AudioDeviceInfo>> mSink;
1920 // This flag is used to control when the user agent is to show a poster frame
1921 // for a video element instead of showing the video contents.
1922 // https://html.spec.whatwg.org/multipage/media.html#show-poster-flag
1923 bool mShowPoster;
1925 // We may delay starting playback of a media for an unvisited tab until it's
1926 // going to foreground. We would create ResumeDelayedMediaPlaybackAgent to
1927 // handle related operations at the time whenever delaying media playback is
1928 // needed.
1929 void CreateResumeDelayedMediaPlaybackAgentIfNeeded();
1930 void ClearResumeDelayedMediaPlaybackAgentIfNeeded();
1931 RefPtr<ResumeDelayedPlaybackAgent> mResumeDelayedPlaybackAgent;
1932 MozPromiseRequestHolder<ResumeDelayedPlaybackAgent::ResumePromise>
1933 mResumePlaybackRequest;
1935 // Return true if the media qualifies for being controlled by media control
1936 // keys.
1937 bool ShouldStartMediaControlKeyListener() const;
1939 // Start the listener if media fits the requirement of being able to be
1940 // controlled be media control keys.
1941 void StartMediaControlKeyListenerIfNeeded();
1943 // It's used to listen media control key, by which we would play or pause
1944 // media element.
1945 RefPtr<MediaControlKeyListener> mMediaControlKeyListener;
1947 // Return true if the media element is being used in picture in picture mode.
1948 bool IsBeingUsedInPictureInPictureMode() const;
1951 // Check if the context is chrome or has the debugger or tabs permission
1952 bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj);
1954 } // namespace dom
1955 } // namespace mozilla
1957 #endif // mozilla_dom_HTMLMediaElement_h