Bug 1795723 - Unified extensions UI should support High Contrast Mode. r=ayeddi,deskt...
[gecko.git] / dom / html / HTMLMediaElement.h
blob09d613504a04a6d8e26ac7b7a4c63ba7e205c1fe
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef mozilla_dom_HTMLMediaElement_h
7 #define mozilla_dom_HTMLMediaElement_h
9 #include "nsGenericHTMLElement.h"
10 #include "AudioChannelService.h"
11 #include "MediaEventSource.h"
12 #include "SeekTarget.h"
13 #include "MediaDecoderOwner.h"
14 #include "MediaElementEventRunners.h"
15 #include "MediaPlaybackDelayPolicy.h"
16 #include "MediaPromiseDefs.h"
17 #include "TelemetryProbesReporter.h"
18 #include "nsCycleCollectionParticipant.h"
19 #include "Visibility.h"
20 #include "mozilla/CORSMode.h"
21 #include "DecoderTraits.h"
22 #include "mozilla/Attributes.h"
23 #include "mozilla/StateWatching.h"
24 #include "mozilla/WeakPtr.h"
25 #include "mozilla/dom/DecoderDoctorNotificationBinding.h"
26 #include "mozilla/dom/HTMLMediaElementBinding.h"
27 #include "mozilla/dom/MediaDebugInfoBinding.h"
28 #include "mozilla/dom/MediaKeys.h"
29 #include "mozilla/dom/TextTrackManager.h"
30 #include "nsGkAtoms.h"
31 #include "PrincipalChangeObserver.h"
32 #include "nsStubMutationObserver.h"
33 #include "MediaSegment.h" // for PrincipalHandle, GraphTime
35 #include <utility>
37 // X.h on Linux #defines CurrentTime as 0L, so we have to #undef it here.
38 #ifdef CurrentTime
39 # undef CurrentTime
40 #endif
42 // Define to output information on decoding and painting framerate
43 /* #define DEBUG_FRAME_RATE 1 */
45 using nsMediaNetworkState = uint16_t;
46 using nsMediaReadyState = uint16_t;
47 using SuspendTypes = uint32_t;
48 using AudibleChangedReasons = uint32_t;
50 class nsIStreamListener;
52 namespace mozilla {
53 class AbstractThread;
54 class ChannelMediaDecoder;
55 class DecoderDoctorDiagnostics;
56 class DOMMediaStream;
57 class ErrorResult;
58 class FirstFrameVideoOutput;
59 class MediaResource;
60 class MediaDecoder;
61 class MediaInputPort;
62 class MediaTrack;
63 class MediaTrackGraph;
64 class MediaStreamWindowCapturer;
65 struct SharedDummyTrack;
66 class VideoFrameContainer;
67 class VideoOutput;
68 namespace dom {
69 class MediaKeys;
70 class TextTrack;
71 class TimeRanges;
72 class WakeLock;
73 class MediaStreamTrack;
74 class MediaStreamTrackSource;
75 class MediaTrack;
76 class VideoStreamTrack;
77 } // namespace dom
78 } // namespace mozilla
80 class AudioDeviceInfo;
81 class nsIChannel;
82 class nsIHttpChannel;
83 class nsILoadGroup;
84 class nsIRunnable;
85 class nsISerialEventTarget;
86 class nsITimer;
87 class nsRange;
89 namespace mozilla::dom {
91 // Number of milliseconds between timeupdate events as defined by spec
92 #define TIMEUPDATE_MS 250
94 class MediaError;
95 class MediaSource;
96 class PlayPromise;
97 class Promise;
98 class TextTrackList;
99 class AudioTrackList;
100 class VideoTrackList;
102 enum class StreamCaptureType : uint8_t { CAPTURE_ALL_TRACKS, CAPTURE_AUDIO };
104 enum class StreamCaptureBehavior : uint8_t {
105 CONTINUE_WHEN_ENDED,
106 FINISH_WHEN_ENDED
109 class HTMLMediaElement : public nsGenericHTMLElement,
110 public MediaDecoderOwner,
111 public PrincipalChangeObserver<MediaStreamTrack>,
112 public SupportsWeakPtr,
113 public nsStubMutationObserver,
114 public TelemetryProbesReporterOwner {
115 public:
116 using TimeStamp = mozilla::TimeStamp;
117 using ImageContainer = mozilla::layers::ImageContainer;
118 using VideoFrameContainer = mozilla::VideoFrameContainer;
119 using MediaResource = mozilla::MediaResource;
120 using MediaDecoderOwner = mozilla::MediaDecoderOwner;
121 using MetadataTags = mozilla::MetadataTags;
123 // Helper struct to keep track of the MediaStreams returned by
124 // mozCaptureStream(). For each OutputMediaStream, dom::MediaTracks get
125 // captured into MediaStreamTracks which get added to
126 // OutputMediaStream::mStream.
127 struct OutputMediaStream {
128 OutputMediaStream(RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
129 bool aFinishWhenEnded);
130 ~OutputMediaStream();
132 RefPtr<DOMMediaStream> mStream;
133 nsTArray<RefPtr<MediaStreamTrack>> mLiveTracks;
134 const bool mCapturingAudioOnly;
135 const bool mFinishWhenEnded;
136 // If mFinishWhenEnded is true, this is the URI of the first resource
137 // mStream got tracks for.
138 nsCOMPtr<nsIURI> mFinishWhenEndedLoadingSrc;
139 // If mFinishWhenEnded is true, this is the first MediaStream mStream got
140 // tracks for.
141 RefPtr<DOMMediaStream> mFinishWhenEndedAttrStream;
142 // If mFinishWhenEnded is true, this is the MediaSource being played.
143 RefPtr<MediaSource> mFinishWhenEndedMediaSource;
146 NS_DECL_NSIMUTATIONOBSERVER_CONTENTREMOVED
148 CORSMode GetCORSMode() { return mCORSMode; }
150 explicit HTMLMediaElement(
151 already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo);
152 void Init();
154 // `eMandatory`: `timeupdate` occurs according to the spec requirement.
155 // Eg.
156 // https://html.spec.whatwg.org/multipage/media.html#seeking:event-media-timeupdate
157 // `ePeriodic` : `timeupdate` occurs regularly and should follow the rule
158 // of not dispatching that event within 250 ms. Eg.
159 // https://html.spec.whatwg.org/multipage/media.html#offsets-into-the-media-resource:event-media-timeupdate
160 enum class TimeupdateType : bool {
161 eMandatory = false,
162 ePeriodic = true,
165 // This is used for event runner creation. Currently only timeupdate needs
166 // that, but it can be used to extend for other events in the future if
167 // necessary.
168 enum class EventFlag : uint8_t {
169 eNone = 0,
170 eMandatory = 1,
174 * This is used when the browser is constructing a video element to play
175 * a channel that we've already started loading. The src attribute and
176 * <source> children are ignored.
177 * @param aChannel the channel to use
178 * @param aListener returns a stream listener that should receive
179 * notifications for the stream
181 nsresult LoadWithChannel(nsIChannel* aChannel, nsIStreamListener** aListener);
183 // nsISupports
184 NS_DECL_ISUPPORTS_INHERITED
185 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(HTMLMediaElement,
186 nsGenericHTMLElement)
187 NS_IMPL_FROMNODE_HELPER(HTMLMediaElement,
188 IsAnyOfHTMLElements(nsGkAtoms::video,
189 nsGkAtoms::audio))
191 NS_DECL_ADDSIZEOFEXCLUDINGTHIS
193 // EventTarget
194 void GetEventTargetParent(EventChainPreVisitor& aVisitor) override;
196 void NodeInfoChanged(Document* aOldDoc) override;
198 virtual bool ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
199 const nsAString& aValue,
200 nsIPrincipal* aMaybeScriptedPrincipal,
201 nsAttrValue& aResult) override;
203 virtual nsresult BindToTree(BindContext&, nsINode& aParent) override;
204 virtual void UnbindFromTree(bool aNullParent = true) override;
205 virtual void DoneCreatingElement() override;
207 virtual bool IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
208 int32_t* aTabIndex) override;
209 virtual int32_t TabIndexDefault() override;
211 // Called by the video decoder object, on the main thread,
212 // when it has read the metadata containing video dimensions,
213 // etc.
214 virtual void MetadataLoaded(const MediaInfo* aInfo,
215 UniquePtr<const MetadataTags> aTags) final;
217 // Called by the decoder object, on the main thread,
218 // when it has read the first frame of the video or audio.
219 void FirstFrameLoaded() final;
221 // Called by the video decoder object, on the main thread,
222 // when the resource has a network error during loading.
223 void NetworkError(const MediaResult& aError) final;
225 // Called by the video decoder object, on the main thread, when the
226 // resource has a decode error during metadata loading or decoding.
227 void DecodeError(const MediaResult& aError) final;
229 // Called by the decoder object, on the main thread, when the
230 // resource has a decode issue during metadata loading or decoding, but can
231 // continue decoding.
232 void DecodeWarning(const MediaResult& aError) final;
234 // Return true if error attribute is not null.
235 bool HasError() const final;
237 // Called by the video decoder object, on the main thread, when the
238 // resource load has been cancelled.
239 void LoadAborted() final;
241 // Called by the video decoder object, on the main thread,
242 // when the video playback has ended.
243 void PlaybackEnded() final;
245 // Called by the video decoder object, on the main thread,
246 // when the resource has started seeking.
247 void SeekStarted() final;
249 // Called by the video decoder object, on the main thread,
250 // when the resource has completed seeking.
251 void SeekCompleted() final;
253 // Called by the video decoder object, on the main thread,
254 // when the resource has aborted seeking.
255 void SeekAborted() final;
257 // Called by the media stream, on the main thread, when the download
258 // has been suspended by the cache or because the element itself
259 // asked the decoder to suspend the download.
260 void DownloadSuspended() final;
262 // Called by the media stream, on the main thread, when the download
263 // has been resumed by the cache or because the element itself
264 // asked the decoder to resumed the download.
265 void DownloadResumed();
267 // Called to indicate the download is progressing.
268 void DownloadProgressed() final;
270 // Called by the media decoder to indicate whether the media cache has
271 // suspended the channel.
272 void NotifySuspendedByCache(bool aSuspendedByCache) final;
274 // Return true if the media element is actually invisible to users.
275 bool IsActuallyInvisible() const override;
277 // Return true if the element is in the view port.
278 bool IsInViewPort() const;
280 // Called by the media decoder and the video frame to get the
281 // ImageContainer containing the video data.
282 VideoFrameContainer* GetVideoFrameContainer() final;
283 layers::ImageContainer* GetImageContainer();
286 * Call this to reevaluate whether we should start/stop due to our owner
287 * document being active, inactive, visible or hidden.
289 void NotifyOwnerDocumentActivityChanged();
291 // Called when the media element enters or leaves the fullscreen.
292 void NotifyFullScreenChanged();
294 bool IsInFullScreen() const;
296 // From PrincipalChangeObserver<MediaStreamTrack>.
297 void PrincipalChanged(MediaStreamTrack* aTrack) override;
299 void UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle);
301 // Called after the MediaStream we're playing rendered a frame to aContainer
302 // with a different principalHandle than the previous frame.
303 void PrincipalHandleChangedForVideoFrameContainer(
304 VideoFrameContainer* aContainer,
305 const PrincipalHandle& aNewPrincipalHandle) override;
307 // Dispatch events
308 void DispatchAsyncEvent(const nsAString& aName) final;
309 void DispatchAsyncEvent(RefPtr<nsMediaEventRunner> aRunner);
311 // Triggers a recomputation of readyState.
312 void UpdateReadyState() override {
313 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
316 // Dispatch events that were raised while in the bfcache
317 nsresult DispatchPendingMediaEvents();
319 // Return true if we can activate autoplay assuming enough data has arrived.
320 bool CanActivateAutoplay();
322 // Notify that state has changed that might cause an autoplay element to
323 // start playing.
324 // If the element is 'autoplay' and is ready to play back (not paused,
325 // autoplay pref enabled, etc), it should start playing back.
326 void CheckAutoplayDataReady();
328 // Check if the media element had crossorigin set when loading started
329 bool ShouldCheckAllowOrigin();
331 // Returns true if the currently loaded resource is CORS same-origin with
332 // respect to the document.
333 bool IsCORSSameOrigin();
335 // Is the media element potentially playing as defined by the HTML 5
336 // specification.
337 // http://www.whatwg.org/specs/web-apps/current-work/#potentially-playing
338 bool IsPotentiallyPlaying() const;
340 // Has playback ended as defined by the HTML 5 specification.
341 // http://www.whatwg.org/specs/web-apps/current-work/#ended
342 bool IsPlaybackEnded() const;
344 // principal of the currently playing resource. Anything accessing the
345 // contents of this element must have a principal that subsumes this
346 // principal. Returns null if nothing is playing.
347 already_AddRefed<nsIPrincipal> GetCurrentPrincipal();
349 // Return true if the loading of this resource required cross-origin
350 // redirects.
351 bool HadCrossOriginRedirects();
353 // Principal of the currently playing video resource. Anything accessing the
354 // image container of this element must have a principal that subsumes this
355 // principal. If there are no live video tracks but content has been rendered
356 // to the image container, we return the last video principal we had. Should
357 // the image container be empty with no live video tracks, we return nullptr.
358 already_AddRefed<nsIPrincipal> GetCurrentVideoPrincipal();
360 // called to notify that the principal of the decoder's media resource has
361 // changed.
362 void NotifyDecoderPrincipalChanged() final;
364 void GetEMEInfo(dom::EMEDebugInfo& aInfo);
366 // Update the visual size of the media. Called from the decoder on the
367 // main thread when/if the size changes.
368 virtual void UpdateMediaSize(const nsIntSize& aSize);
369 // Like UpdateMediaSize, but only updates the size if no size has yet
370 // been set.
371 void UpdateInitialMediaSize(const nsIntSize& aSize);
373 void Invalidate(bool aImageSizeChanged, Maybe<nsIntSize>& aNewIntrinsicSize,
374 bool aForceInvalidate) override;
376 // Returns the CanPlayStatus indicating if we can handle the
377 // full MIME type including the optional codecs parameter.
378 static CanPlayStatus GetCanPlay(const nsAString& aType,
379 DecoderDoctorDiagnostics* aDiagnostics);
382 * Called when a child source element is added to this media element. This
383 * may queue a task to run the select resource algorithm if appropriate.
385 void NotifyAddedSource();
388 * Called when there's been an error fetching the resource. This decides
389 * whether it's appropriate to fire an error event.
391 void NotifyLoadError(const nsACString& aErrorDetails = nsCString());
394 * Called by one of our associated MediaTrackLists (audio/video) when a
395 * MediaTrack is added.
397 void NotifyMediaTrackAdded(dom::MediaTrack* aTrack);
400 * Called by one of our associated MediaTrackLists (audio/video) when a
401 * MediaTrack is removed.
403 void NotifyMediaTrackRemoved(dom::MediaTrack* aTrack);
406 * Called by one of our associated MediaTrackLists (audio/video) when an
407 * AudioTrack is enabled or a VideoTrack is selected.
409 void NotifyMediaTrackEnabled(dom::MediaTrack* aTrack);
412 * Called by one of our associated MediaTrackLists (audio/video) when an
413 * AudioTrack is disabled or a VideoTrack is unselected.
415 void NotifyMediaTrackDisabled(dom::MediaTrack* aTrack);
418 * Returns the current load ID. Asynchronous events store the ID that was
419 * current when they were enqueued, and if it has changed when they come to
420 * fire, they consider themselves cancelled, and don't fire.
422 uint32_t GetCurrentLoadID() const { return mCurrentLoadID; }
425 * Returns the load group for this media element's owner document.
426 * XXX XBL2 issue.
428 already_AddRefed<nsILoadGroup> GetDocumentLoadGroup();
431 * Returns true if the media has played or completed a seek.
432 * Used by video frame to determine whether to paint the poster.
434 bool GetPlayedOrSeeked() const { return mHasPlayedOrSeeked; }
436 nsresult CopyInnerTo(Element* aDest);
439 * Sets the Accept header on the HTTP channel to the required
440 * video or audio MIME types.
442 virtual nsresult SetAcceptHeader(nsIHttpChannel* aChannel) = 0;
445 * Sets the required request headers on the HTTP channel for
446 * video or audio requests.
448 void SetRequestHeaders(nsIHttpChannel* aChannel);
451 * Asynchronously awaits a stable state, whereupon aRunnable runs on the main
452 * thread. This adds an event which run aRunnable to the appshell's list of
453 * sections synchronous the next time control returns to the event loop.
455 void RunInStableState(nsIRunnable* aRunnable);
458 * Fires a timeupdate event. If aPeriodic is true, the event will only
459 * be fired if we've not fired a timeupdate event (for any reason) in the
460 * last 250ms, as required by the spec when the current time is periodically
461 * increasing during playback.
463 void FireTimeUpdate(TimeupdateType aType);
465 void MaybeQueueTimeupdateEvent() final {
466 FireTimeUpdate(TimeupdateType::ePeriodic);
469 const TimeStamp& LastTimeupdateDispatchTime() const;
470 void UpdateLastTimeupdateDispatchTime();
472 // WebIDL
474 MediaError* GetError() const;
476 void GetSrc(nsAString& aSrc) { GetURIAttr(nsGkAtoms::src, nullptr, aSrc); }
477 void SetSrc(const nsAString& aSrc, ErrorResult& aError) {
478 SetHTMLAttr(nsGkAtoms::src, aSrc, aError);
480 void SetSrc(const nsAString& aSrc, nsIPrincipal* aTriggeringPrincipal,
481 ErrorResult& aError) {
482 SetHTMLAttr(nsGkAtoms::src, aSrc, aTriggeringPrincipal, aError);
485 void GetCurrentSrc(nsAString& aCurrentSrc);
487 void GetCrossOrigin(nsAString& aResult) {
488 // Null for both missing and invalid defaults is ok, since we
489 // always parse to an enum value, so we don't need an invalid
490 // default, and we _want_ the missing default to be null.
491 GetEnumAttr(nsGkAtoms::crossorigin, nullptr, aResult);
493 void SetCrossOrigin(const nsAString& aCrossOrigin, ErrorResult& aError) {
494 SetOrRemoveNullableStringAttr(nsGkAtoms::crossorigin, aCrossOrigin, aError);
497 uint16_t NetworkState() const { return mNetworkState; }
499 void NotifyXPCOMShutdown() final;
501 // Called by media decoder when the audible state changed or when input is
502 // a media stream.
503 void SetAudibleState(bool aAudible) final;
505 // Notify agent when the MediaElement changes its audible state.
506 void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason);
508 void GetPreload(nsAString& aValue) {
509 if (mSrcAttrStream) {
510 nsGkAtoms::none->ToString(aValue);
511 return;
513 GetEnumAttr(nsGkAtoms::preload, nullptr, aValue);
515 void SetPreload(const nsAString& aValue, ErrorResult& aRv) {
516 if (mSrcAttrStream) {
517 return;
519 SetHTMLAttr(nsGkAtoms::preload, aValue, aRv);
522 already_AddRefed<TimeRanges> Buffered() const;
524 void Load();
526 void CanPlayType(const nsAString& aType, nsAString& aResult);
528 uint16_t ReadyState() const { return mReadyState; }
530 bool Seeking() const;
532 double CurrentTime() const;
534 void SetCurrentTime(double aCurrentTime, ErrorResult& aRv);
535 void SetCurrentTime(double aCurrentTime) {
536 SetCurrentTime(aCurrentTime, IgnoreErrors());
539 void FastSeek(double aTime, ErrorResult& aRv);
541 already_AddRefed<Promise> SeekToNextFrame(ErrorResult& aRv);
543 double Duration() const;
545 bool HasAudio() const { return mMediaInfo.HasAudio(); }
547 virtual bool IsVideo() const { return false; }
549 bool HasVideo() const { return mMediaInfo.HasVideo(); }
551 bool IsEncrypted() const override { return mIsEncrypted; }
553 bool Paused() const { return mPaused; }
555 double DefaultPlaybackRate() const {
556 if (mSrcAttrStream) {
557 return 1.0;
559 return mDefaultPlaybackRate;
562 void SetDefaultPlaybackRate(double aDefaultPlaybackRate, ErrorResult& aRv);
564 double PlaybackRate() const {
565 if (mSrcAttrStream) {
566 return 1.0;
568 return mPlaybackRate;
571 void SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv);
573 already_AddRefed<TimeRanges> Played();
575 already_AddRefed<TimeRanges> Seekable() const;
577 bool Ended();
579 bool Autoplay() const { return GetBoolAttr(nsGkAtoms::autoplay); }
581 void SetAutoplay(bool aValue, ErrorResult& aRv) {
582 SetHTMLBoolAttr(nsGkAtoms::autoplay, aValue, aRv);
585 bool Loop() const { return GetBoolAttr(nsGkAtoms::loop); }
587 void SetLoop(bool aValue, ErrorResult& aRv) {
588 SetHTMLBoolAttr(nsGkAtoms::loop, aValue, aRv);
591 already_AddRefed<Promise> Play(ErrorResult& aRv);
592 void Play() {
593 IgnoredErrorResult dummy;
594 RefPtr<Promise> toBeIgnored = Play(dummy);
597 void Pause(ErrorResult& aRv);
598 void Pause() { Pause(IgnoreErrors()); }
600 bool Controls() const { return GetBoolAttr(nsGkAtoms::controls); }
602 void SetControls(bool aValue, ErrorResult& aRv) {
603 SetHTMLBoolAttr(nsGkAtoms::controls, aValue, aRv);
606 double Volume() const { return mVolume; }
608 void SetVolume(double aVolume, ErrorResult& aRv);
610 bool Muted() const { return mMuted & MUTED_BY_CONTENT; }
611 void SetMuted(bool aMuted);
613 bool DefaultMuted() const { return GetBoolAttr(nsGkAtoms::muted); }
615 void SetDefaultMuted(bool aMuted, ErrorResult& aRv) {
616 SetHTMLBoolAttr(nsGkAtoms::muted, aMuted, aRv);
619 bool MozAllowCasting() const { return mAllowCasting; }
621 void SetMozAllowCasting(bool aShow) { mAllowCasting = aShow; }
623 bool MozIsCasting() const { return mIsCasting; }
625 void SetMozIsCasting(bool aShow) { mIsCasting = aShow; }
627 // Returns whether a call to Play() would be rejected with NotAllowedError.
628 // This assumes "worst case" for unknowns. So if prompting for permission is
629 // enabled and no permission is stored, this behaves as if the user would
630 // opt to block.
631 bool AllowedToPlay() const;
633 already_AddRefed<MediaSource> GetMozMediaSourceObject() const;
635 // Returns a promise which will be resolved after collecting debugging
636 // data from decoder/reader/MDSM. Used for debugging purposes.
637 already_AddRefed<Promise> MozRequestDebugInfo(ErrorResult& aRv);
639 // Enables DecoderDoctorLogger logging. Used for debugging purposes.
640 static void MozEnableDebugLog(const GlobalObject&);
642 // Returns a promise which will be resolved after collecting debugging
643 // log associated with this element. Used for debugging purposes.
644 already_AddRefed<Promise> MozRequestDebugLog(ErrorResult& aRv);
646 // For use by mochitests. Enabling pref "media.test.video-suspend"
647 void SetVisible(bool aVisible);
649 // For use by mochitests. Enabling pref "media.test.video-suspend"
650 bool HasSuspendTaint() const;
652 // For use by mochitests.
653 bool IsVideoDecodingSuspended() const;
655 // These functions return accumulated time, which are used for the telemetry
656 // usage. Return -1 for error.
657 double TotalVideoPlayTime() const;
658 double TotalVideoHDRPlayTime() const;
659 double VisiblePlayTime() const;
660 double InvisiblePlayTime() const;
661 double VideoDecodeSuspendedTime() const;
662 double TotalAudioPlayTime() const;
663 double AudiblePlayTime() const;
664 double InaudiblePlayTime() const;
665 double MutedPlayTime() const;
667 // Test methods for decoder doctor.
668 void SetFormatDiagnosticsReportForMimeType(const nsAString& aMimeType,
669 DecoderDoctorReportType aType);
670 void SetDecodeError(const nsAString& aError, ErrorResult& aRv);
671 void SetAudioSinkFailedStartup();
673 // Synchronously, return the next video frame and mark the element unable to
674 // participate in decode suspending.
676 // This function is synchronous for cases where decoding has been suspended
677 // and JS needs a frame to use in, eg., nsLayoutUtils::SurfaceFromElement()
678 // via drawImage().
679 already_AddRefed<layers::Image> GetCurrentImage();
681 already_AddRefed<DOMMediaStream> GetSrcObject() const;
682 void SetSrcObject(DOMMediaStream& aValue);
683 void SetSrcObject(DOMMediaStream* aValue);
685 bool PreservesPitch() const { return mPreservesPitch; }
686 void SetPreservesPitch(bool aPreservesPitch);
688 MediaKeys* GetMediaKeys() const;
690 already_AddRefed<Promise> SetMediaKeys(MediaKeys* mediaKeys,
691 ErrorResult& aRv);
693 mozilla::dom::EventHandlerNonNull* GetOnencrypted();
694 void SetOnencrypted(mozilla::dom::EventHandlerNonNull* aCallback);
696 mozilla::dom::EventHandlerNonNull* GetOnwaitingforkey();
697 void SetOnwaitingforkey(mozilla::dom::EventHandlerNonNull* aCallback);
699 void DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
700 const nsAString& aInitDataType) override;
702 bool IsEventAttributeNameInternal(nsAtom* aName) override;
704 bool ContainsRestrictedContent() const;
706 void NotifyWaitingForKey() override;
708 already_AddRefed<DOMMediaStream> CaptureAudio(ErrorResult& aRv,
709 MediaTrackGraph* aGraph);
711 already_AddRefed<DOMMediaStream> MozCaptureStream(ErrorResult& aRv);
713 already_AddRefed<DOMMediaStream> MozCaptureStreamUntilEnded(ErrorResult& aRv);
715 bool MozAudioCaptured() const { return mAudioCaptured; }
717 void MozGetMetadata(JSContext* aCx, JS::MutableHandle<JSObject*> aResult,
718 ErrorResult& aRv);
720 double MozFragmentEnd();
722 AudioTrackList* AudioTracks();
724 VideoTrackList* VideoTracks();
726 TextTrackList* GetTextTracks();
728 already_AddRefed<TextTrack> AddTextTrack(TextTrackKind aKind,
729 const nsAString& aLabel,
730 const nsAString& aLanguage);
732 void AddTextTrack(TextTrack* aTextTrack) {
733 GetOrCreateTextTrackManager()->AddTextTrack(aTextTrack);
736 void RemoveTextTrack(TextTrack* aTextTrack, bool aPendingListOnly = false) {
737 if (mTextTrackManager) {
738 mTextTrackManager->RemoveTextTrack(aTextTrack, aPendingListOnly);
742 void NotifyCueAdded(TextTrackCue& aCue) {
743 if (mTextTrackManager) {
744 mTextTrackManager->NotifyCueAdded(aCue);
747 void NotifyCueRemoved(TextTrackCue& aCue) {
748 if (mTextTrackManager) {
749 mTextTrackManager->NotifyCueRemoved(aCue);
752 void NotifyCueUpdated(TextTrackCue* aCue) {
753 if (mTextTrackManager) {
754 mTextTrackManager->NotifyCueUpdated(aCue);
758 void NotifyCueDisplayStatesChanged();
760 bool IsBlessed() const { return mIsBlessed; }
762 // A method to check whether we are currently playing.
763 bool IsCurrentlyPlaying() const;
765 // Returns true if the media element is being destroyed. Used in
766 // dormancy checks to prevent dormant processing for an element
767 // that will soon be gone.
768 bool IsBeingDestroyed();
770 void OnVisibilityChange(Visibility aNewVisibility);
772 // Begin testing only methods
773 float ComputedVolume() const;
774 bool ComputedMuted() const;
776 // Return true if the media has been suspended media due to an inactive
777 // document or prohibiting by the docshell.
778 bool IsSuspendedByInactiveDocOrDocShell() const;
779 // End testing only methods
781 void SetMediaInfo(const MediaInfo& aInfo);
782 MediaInfo GetMediaInfo() const override;
784 // Gives access to the decoder's frame statistics, if present.
785 FrameStatistics* GetFrameStatistics() const override;
787 void DispatchAsyncTestingEvent(const nsAString& aName) override;
789 AbstractThread* AbstractMainThread() const final;
791 // Log the usage of a {visible / invisible} video element as
792 // the source of {drawImage(), createPattern(), createImageBitmap() and
793 // captureStream()} APIs. This function can be used to collect telemetries for
794 // bug 1352007.
795 enum class CallerAPI {
796 DRAW_IMAGE,
797 CREATE_PATTERN,
798 CREATE_IMAGEBITMAP,
799 CAPTURE_STREAM,
800 CREATE_VIDEOFRAME,
802 void LogVisibility(CallerAPI aAPI);
804 Document* GetDocument() const override;
806 already_AddRefed<GMPCrashHelper> CreateGMPCrashHelper() override;
808 nsISerialEventTarget* MainThreadEventTarget() {
809 return mMainThreadEventTarget;
812 // Set the sink id (of the output device) that the audio will play. If aSinkId
813 // is empty the default device will be set.
814 already_AddRefed<Promise> SetSinkId(const nsAString& aSinkId,
815 ErrorResult& aRv);
816 // Get the sink id of the device that audio is being played. Initial value is
817 // empty and the default device is being used.
818 void GetSinkId(nsString& aSinkId) const {
819 MOZ_ASSERT(NS_IsMainThread());
820 aSinkId = mSink.first;
823 // This is used to notify MediaElementAudioSourceNode that media element is
824 // allowed to play when media element is used as a source for web audio, so
825 // that we can start AudioContext if it was not allowed to start.
826 RefPtr<GenericNonExclusivePromise> GetAllowedToPlayPromise();
828 bool GetShowPosterFlag() const { return mShowPoster; }
830 bool IsAudible() const;
832 // Return key system in use if we have one, otherwise return nothing.
833 Maybe<nsAutoString> GetKeySystem() const override;
835 protected:
836 virtual ~HTMLMediaElement();
838 class AudioChannelAgentCallback;
839 class ChannelLoader;
840 class ErrorSink;
841 class MediaElementTrackSource;
842 class MediaLoadListener;
843 class MediaStreamRenderer;
844 class MediaStreamTrackListener;
845 class ShutdownObserver;
846 class TitleChangeObserver;
847 class MediaControlKeyListener;
849 MediaDecoderOwner::NextFrameStatus NextFrameStatus();
851 void SetDecoder(MediaDecoder* aDecoder);
853 void PlayInternal(bool aHandlingUserInput);
855 // See spec, https://html.spec.whatwg.org/#internal-pause-steps
856 void PauseInternal();
858 /** Use this method to change the mReadyState member, so required
859 * events can be fired.
861 void ChangeReadyState(nsMediaReadyState aState);
864 * Use this method to change the mNetworkState member, so required
865 * actions will be taken during the transition.
867 void ChangeNetworkState(nsMediaNetworkState aState);
870 * The MediaElement will be responsible for creating and releasing the audio
871 * wakelock depending on the playing and audible state.
873 virtual void WakeLockRelease();
874 virtual void UpdateWakeLock();
876 void CreateAudioWakeLockIfNeeded();
877 void ReleaseAudioWakeLockIfExists();
878 RefPtr<WakeLock> mWakeLock;
881 * Logs a warning message to the web console to report various failures.
882 * aMsg is the localized message identifier, aParams is the parameters to
883 * be substituted into the localized message, and aParamCount is the number
884 * of parameters in aParams.
886 void ReportLoadError(const char* aMsg, const nsTArray<nsString>& aParams =
887 nsTArray<nsString>());
890 * Log message to web console.
892 void ReportToConsole(
893 uint32_t aErrorFlags, const char* aMsg,
894 const nsTArray<nsString>& aParams = nsTArray<nsString>()) const;
897 * Changes mHasPlayedOrSeeked to aValue. If mHasPlayedOrSeeked changes
898 * we'll force a reflow so that the video frame gets reflowed to reflect
899 * the poster hiding or showing immediately.
901 void SetPlayedOrSeeked(bool aValue);
904 * Initialize the media element for playback of aStream
906 void SetupSrcMediaStreamPlayback(DOMMediaStream* aStream);
908 * Stop playback on mSrcStream.
910 void EndSrcMediaStreamPlayback();
912 * Ensure we're playing mSrcStream if and only if we're not paused.
914 enum { REMOVING_SRC_STREAM = 0x1 };
915 void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0);
918 * Ensure currentTime progresses if and only if we're potentially playing
919 * mSrcStream. Called by the watch manager while we're playing mSrcStream, and
920 * one of the inputs to the potentially playing algorithm changes.
922 void UpdateSrcStreamPotentiallyPlaying();
925 * mSrcStream's graph's CurrentTime() has been updated. It might be time to
926 * fire "timeupdate".
928 void UpdateSrcStreamTime();
931 * Called after a tail dispatch when playback of mSrcStream ended, to comply
932 * with the spec where we must start reporting true for the ended attribute
933 * after the event loop returns to step 1. A MediaStream could otherwise be
934 * manipulated to end a HTMLMediaElement synchronously.
936 void UpdateSrcStreamReportPlaybackEnded();
939 * Called by our DOMMediaStream::TrackListener when a new MediaStreamTrack has
940 * been added to the playback stream of |mSrcStream|.
942 void NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aTrack);
945 * Called by our DOMMediaStream::TrackListener when a MediaStreamTrack in
946 * |mSrcStream|'s playback stream has ended.
948 void NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack);
951 * Convenience method to get in a single list all enabled AudioTracks and, if
952 * this is a video element, the selected VideoTrack.
954 void GetAllEnabledMediaTracks(nsTArray<RefPtr<MediaTrack>>& aTracks);
957 * Enables or disables all tracks forwarded from mSrcStream to all
958 * OutputMediaStreams. We do this for muting the tracks when pausing,
959 * and unmuting when playing the media element again.
961 void SetCapturedOutputStreamsEnabled(bool aEnabled);
964 * Returns true if output tracks should be muted, based on the state of this
965 * media element.
967 enum class OutputMuteState { Muted, Unmuted };
968 OutputMuteState OutputTracksMuted();
971 * Sets the muted state of all output track sources. They are muted when we're
972 * paused and unmuted otherwise.
974 void UpdateOutputTracksMuting();
977 * Create a new MediaStreamTrack for the TrackSource corresponding to aTrack
978 * and add it to the DOMMediaStream in aOutputStream. This automatically sets
979 * the output track to enabled or disabled depending on our current playing
980 * state.
982 enum class AddTrackMode { ASYNC, SYNC };
983 void AddOutputTrackSourceToOutputStream(
984 MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
985 AddTrackMode aMode = AddTrackMode::ASYNC);
988 * Creates output track sources when this media element is captured, tracks
989 * exist, playback is not ended and readyState is >= HAVE_METADATA.
991 void UpdateOutputTrackSources();
994 * Returns an DOMMediaStream containing the played contents of this
995 * element. When aBehavior is FINISH_WHEN_ENDED, when this element ends
996 * playback we will finish the stream and not play any more into it. When
997 * aType is CONTINUE_WHEN_ENDED, ending playback does not finish the stream.
998 * The stream will never finish.
1000 * When aType is CAPTURE_AUDIO, we stop playout of audio and instead route it
1001 * to the DOMMediaStream. Volume and mute state will be applied to the audio
1002 * reaching the stream. No video tracks will be captured in this case.
1004 already_AddRefed<DOMMediaStream> CaptureStreamInternal(
1005 StreamCaptureBehavior aFinishBehavior,
1006 StreamCaptureType aStreamCaptureType, MediaTrackGraph* aGraph);
1009 * Initialize a decoder as a clone of an existing decoder in another
1010 * element.
1011 * mLoadingSrc must already be set.
1013 nsresult InitializeDecoderAsClone(ChannelMediaDecoder* aOriginal);
1016 * Call Load() and FinishDecoderSetup() on the decoder. It also handle
1017 * resource cloning if DecoderType is ChannelMediaDecoder.
1019 template <typename DecoderType, typename... LoadArgs>
1020 nsresult SetupDecoder(DecoderType* aDecoder, LoadArgs&&... aArgs);
1023 * Initialize a decoder to load the given channel. The decoder's stream
1024 * listener is returned via aListener.
1025 * mLoadingSrc must already be set.
1027 nsresult InitializeDecoderForChannel(nsIChannel* aChannel,
1028 nsIStreamListener** aListener);
1031 * Finish setting up the decoder after Load() has been called on it.
1032 * Called by InitializeDecoderForChannel/InitializeDecoderAsClone.
1034 nsresult FinishDecoderSetup(MediaDecoder* aDecoder);
1037 * Call this after setting up mLoadingSrc and mDecoder.
1039 void AddMediaElementToURITable();
1041 * Call this before modifying mLoadingSrc.
1043 void RemoveMediaElementFromURITable();
1045 * Call this to find a media element with the same NodePrincipal and
1046 * mLoadingSrc set to aURI, and with a decoder on which Load() has been
1047 * called.
1049 HTMLMediaElement* LookupMediaElementURITable(nsIURI* aURI);
1052 * Shutdown and clear mDecoder and maintain associated invariants.
1054 void ShutdownDecoder();
1056 * Execute the initial steps of the load algorithm that ensure existing
1057 * loads are aborted, the element is emptied, and a new load ID is
1058 * created.
1060 void AbortExistingLoads();
1063 * This is the dedicated media source failure steps.
1064 * Called when all potential resources are exhausted. Changes network
1065 * state to NETWORK_NO_SOURCE, and sends error event with code
1066 * MEDIA_ERR_SRC_NOT_SUPPORTED.
1068 void NoSupportedMediaSourceError(
1069 const nsACString& aErrorDetails = nsCString());
1072 * Per spec, Failed with elements: Queue a task, using the DOM manipulation
1073 * task source, to fire a simple event named error at the candidate element.
1074 * So dispatch |QueueLoadFromSourceTask| to main thread to make sure the task
1075 * will be executed later than loadstart event.
1077 void DealWithFailedElement(nsIContent* aSourceElement);
1080 * Attempts to load resources from the <source> children. This is a
1081 * substep of the resource selection algorithm. Do not call this directly,
1082 * call QueueLoadFromSourceTask() instead.
1084 void LoadFromSourceChildren();
1087 * Asynchronously awaits a stable state, and then causes
1088 * LoadFromSourceChildren() to be called on the main threads' event loop.
1090 void QueueLoadFromSourceTask();
1093 * Runs the media resource selection algorithm.
1095 void SelectResource();
1098 * A wrapper function that allows us to cleanly reset flags after a call
1099 * to SelectResource()
1101 void SelectResourceWrapper();
1104 * Asynchronously awaits a stable state, and then causes SelectResource()
1105 * to be run on the main thread's event loop.
1107 void QueueSelectResourceTask();
1110 * When loading a new source on an existing media element, make sure to reset
1111 * everything that is accessible using the media element API.
1113 void ResetState();
1116 * The resource-fetch algorithm step of the load algorithm.
1118 MediaResult LoadResource();
1121 * Selects the next <source> child from which to load a resource. Called
1122 * during the resource selection algorithm. Stores the return value in
1123 * mSourceLoadCandidate before returning.
1125 Element* GetNextSource();
1128 * Changes mDelayingLoadEvent, and will call BlockOnLoad()/UnblockOnLoad()
1129 * on the owning document, so it can delay the load event firing.
1131 void ChangeDelayLoadStatus(bool aDelay);
1134 * If we suspended downloading after the first frame, unsuspend now.
1136 void StopSuspendingAfterFirstFrame();
1139 * Called when our channel is redirected to another channel.
1140 * Updates our mChannel reference to aNewChannel.
1142 nsresult OnChannelRedirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
1143 uint32_t aFlags);
1146 * Call this to reevaluate whether we should be holding a self-reference.
1148 void AddRemoveSelfReference();
1151 * Called when "xpcom-shutdown" event is received.
1153 void NotifyShutdownEvent();
1156 * Possible values of the 'preload' attribute.
1158 enum PreloadAttrValue : uint8_t {
1159 PRELOAD_ATTR_EMPTY, // set to ""
1160 PRELOAD_ATTR_NONE, // set to "none"
1161 PRELOAD_ATTR_METADATA, // set to "metadata"
1162 PRELOAD_ATTR_AUTO // set to "auto"
1166 * The preloading action to perform. These dictate how we react to the
1167 * preload attribute. See mPreloadAction.
1169 enum PreloadAction {
1170 PRELOAD_UNDEFINED = 0, // not determined - used only for initialization
1171 PRELOAD_NONE = 1, // do not preload
1172 PRELOAD_METADATA = 2, // preload only the metadata (and first frame)
1173 PRELOAD_ENOUGH = 3 // preload enough data to allow uninterrupted
1174 // playback
1178 * The guts of Load(). Load() acts as a wrapper around this which sets
1179 * mIsDoingExplicitLoad to true so that when script calls 'load()'
1180 * preload-none will be automatically upgraded to preload-metadata.
1182 void DoLoad();
1185 * Suspends the load of mLoadingSrc, so that it can be resumed later
1186 * by ResumeLoad(). This is called when we have a media with a 'preload'
1187 * attribute value of 'none', during the resource selection algorithm.
1189 void SuspendLoad();
1192 * Resumes a previously suspended load (suspended by SuspendLoad(uri)).
1193 * Will continue running the resource selection algorithm.
1194 * Sets mPreloadAction to aAction.
1196 void ResumeLoad(PreloadAction aAction);
1199 * Handle a change to the preload attribute. Should be called whenever the
1200 * value (or presence) of the preload attribute changes. The change in
1201 * attribute value may cause a change in the mPreloadAction of this
1202 * element. If there is a change then this method will initiate any
1203 * behaviour that is necessary to implement the action.
1205 void UpdatePreloadAction();
1208 * Fire progress events if needed according to the time and byte constraints
1209 * outlined in the specification. aHaveNewProgress is true if progress has
1210 * just been detected. Otherwise the method is called as a result of the
1211 * progress timer.
1213 void CheckProgress(bool aHaveNewProgress);
1214 static void ProgressTimerCallback(nsITimer* aTimer, void* aClosure);
1216 * Start timer to update download progress.
1218 void StartProgressTimer();
1220 * Start sending progress and/or stalled events.
1222 void StartProgress();
1224 * Stop progress information timer and events.
1226 void StopProgress();
1229 * Dispatches an error event to a child source element.
1231 void DispatchAsyncSourceError(nsIContent* aSourceElement);
1234 * Resets the media element for an error condition as per aErrorCode.
1235 * aErrorCode must be one of WebIDL HTMLMediaElement error codes.
1237 void Error(uint16_t aErrorCode,
1238 const nsACString& aErrorDetails = nsCString());
1241 * Returns the URL spec of the currentSrc.
1243 void GetCurrentSpec(nsCString& aString);
1246 * Process any media fragment entries in the URI
1248 void ProcessMediaFragmentURI();
1251 * Mute or unmute the audio and change the value that the |muted| map.
1253 void SetMutedInternal(uint32_t aMuted);
1255 * Update the volume of the output audio stream to match the element's
1256 * current mMuted/mVolume/mAudioChannelFaded state.
1258 void SetVolumeInternal();
1261 * Suspend or resume element playback and resource download. When we suspend
1262 * playback, event delivery would also be suspended (and events queued) until
1263 * the element is resumed.
1265 void SuspendOrResumeElement(bool aSuspendElement);
1267 // Get the HTMLMediaElement object if the decoder is being used from an
1268 // HTML media element, and null otherwise.
1269 HTMLMediaElement* GetMediaElement() final { return this; }
1271 // Return true if decoding should be paused
1272 bool GetPaused() final { return Paused(); }
1274 // Seeks to aTime seconds. aSeekType can be Exact to seek to exactly the
1275 // seek target, or PrevSyncPoint if a quicker but less precise seek is
1276 // desired, and we'll seek to the sync point (keyframe and/or start of the
1277 // next block of audio samples) preceeding seek target.
1278 void Seek(double aTime, SeekTarget::Type aSeekType, ErrorResult& aRv);
1280 // Update the audio channel playing state
1281 void UpdateAudioChannelPlayingState();
1283 // Adds to the element's list of pending text tracks each text track
1284 // in the element's list of text tracks whose text track mode is not disabled
1285 // and whose text track readiness state is loading.
1286 void PopulatePendingTextTrackList();
1288 // Gets a reference to the MediaElement's TextTrackManager. If the
1289 // MediaElement doesn't yet have one then it will create it.
1290 TextTrackManager* GetOrCreateTextTrackManager();
1292 // Recomputes ready state and fires events as necessary based on current
1293 // state.
1294 void UpdateReadyStateInternal();
1296 // Create or destroy the captured stream.
1297 void AudioCaptureTrackChange(bool aCapture);
1299 // If the network state is empty and then we would trigger DoLoad().
1300 void MaybeDoLoad();
1302 // Anything we need to check after played success and not related with spec.
1303 void UpdateCustomPolicyAfterPlayed();
1305 // Returns a StreamCaptureType populated with the right bits, depending on the
1306 // tracks this HTMLMediaElement has.
1307 StreamCaptureType CaptureTypeForElement();
1309 // True if this element can be captured, false otherwise.
1310 bool CanBeCaptured(StreamCaptureType aCaptureType);
1312 using nsGenericHTMLElement::DispatchEvent;
1313 // For nsAsyncEventRunner.
1314 nsresult DispatchEvent(const nsAString& aName);
1316 already_AddRefed<nsMediaEventRunner> GetEventRunner(
1317 const nsAString& aName, EventFlag aFlag = EventFlag::eNone);
1319 // This method moves the mPendingPlayPromises into a temperate object. So the
1320 // mPendingPlayPromises is cleared after this method call.
1321 nsTArray<RefPtr<PlayPromise>> TakePendingPlayPromises();
1323 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1324 // and queues a task to resolve them.
1325 void AsyncResolvePendingPlayPromises();
1327 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1328 // and queues a task to reject them.
1329 void AsyncRejectPendingPlayPromises(nsresult aError);
1331 // This method snapshots the mPendingPlayPromises by TakePendingPlayPromises()
1332 // and queues a task to resolve them also to dispatch a "playing" event.
1333 void NotifyAboutPlaying();
1335 already_AddRefed<Promise> CreateDOMPromise(ErrorResult& aRv) const;
1337 // Pass information for deciding the video decode mode to decoder.
1338 void NotifyDecoderActivityChanges() const;
1340 // Constructs an AudioTrack in mAudioTrackList if aInfo reports that audio is
1341 // available, and a VideoTrack in mVideoTrackList if aInfo reports that video
1342 // is available.
1343 void ConstructMediaTracks(const MediaInfo* aInfo);
1345 // Removes all MediaTracks from mAudioTrackList and mVideoTrackList and fires
1346 // "removetrack" on the lists accordingly.
1347 // Note that by spec, this should not fire "removetrack". However, it appears
1348 // other user agents do, per
1349 // https://wpt.fyi/results/media-source/mediasource-avtracks.html.
1350 void RemoveMediaTracks();
1352 // Mark the decoder owned by the element as tainted so that the
1353 // suspend-video-decoder is disabled.
1354 void MarkAsTainted();
1356 virtual nsresult AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
1357 const nsAttrValue* aValue,
1358 const nsAttrValue* aOldValue,
1359 nsIPrincipal* aMaybeScriptedPrincipal,
1360 bool aNotify) override;
1361 virtual nsresult OnAttrSetButNotChanged(int32_t aNamespaceID, nsAtom* aName,
1362 const nsAttrValueOrString& aValue,
1363 bool aNotify) override;
1365 bool DetachExistingMediaKeys();
1366 bool TryRemoveMediaKeysAssociation();
1367 void RemoveMediaKeys();
1368 bool AttachNewMediaKeys();
1369 bool TryMakeAssociationWithCDM(CDMProxy* aProxy);
1370 void MakeAssociationWithCDMResolved();
1371 void SetCDMProxyFailure(const MediaResult& aResult);
1372 void ResetSetMediaKeysTempVariables();
1374 void PauseIfShouldNotBePlaying();
1376 WatchManager<HTMLMediaElement> mWatchManager;
1378 // When the play is not allowed, dispatch related events which are used for
1379 // testing or changing control UI.
1380 void DispatchEventsWhenPlayWasNotAllowed();
1382 // When the doc is blocked permanantly, we would dispatch event to notify
1383 // front-end side to show blocking icon.
1384 void MaybeNotifyAutoplayBlocked();
1386 // Dispatch event for video control when video gets blocked in order to show
1387 // the click-to-play icon.
1388 void DispatchBlockEventForVideoControl();
1390 // When playing state change, we have to notify MediaControl in the chrome
1391 // process in order to keep its playing state correct.
1392 void NotifyMediaControlPlaybackStateChanged();
1394 // Clear the timer when we want to continue listening to the media control
1395 // key events.
1396 void ClearStopMediaControlTimerIfNeeded();
1398 // Sets a secondary renderer for mSrcStream, so this media element can be
1399 // rendered in Picture-in-Picture mode when playing a MediaStream. A null
1400 // aContainer will unset the secondary renderer. aFirstFrameOutput allows
1401 // for injecting a listener of the callers choice for rendering the first
1402 // frame.
1403 void SetSecondaryMediaStreamRenderer(
1404 VideoFrameContainer* aContainer,
1405 FirstFrameVideoOutput* aFirstFrameOutput = nullptr);
1407 // This function is used to update the status of media control when the media
1408 // changes its status of being used in the Picture-in-Picture mode.
1409 void UpdateMediaControlAfterPictureInPictureModeChanged();
1411 // The current decoder. Load() has been called on this decoder.
1412 // At most one of mDecoder and mSrcStream can be non-null.
1413 RefPtr<MediaDecoder> mDecoder;
1415 // The DocGroup-specific nsISerialEventTarget of this HTML element on the main
1416 // thread.
1417 nsCOMPtr<nsISerialEventTarget> mMainThreadEventTarget;
1419 // The DocGroup-specific AbstractThread::MainThread() of this HTML element.
1420 RefPtr<AbstractThread> mAbstractMainThread;
1422 // A reference to the VideoFrameContainer which contains the current frame
1423 // of video to display.
1424 RefPtr<VideoFrameContainer> mVideoFrameContainer;
1426 // Holds a reference to the MediaStream that has been set in the src
1427 // attribute.
1428 RefPtr<DOMMediaStream> mSrcAttrStream;
1430 // Holds the triggering principal for the src attribute.
1431 nsCOMPtr<nsIPrincipal> mSrcAttrTriggeringPrincipal;
1433 // Holds a reference to the MediaStream that we're actually playing.
1434 // At most one of mDecoder and mSrcStream can be non-null.
1435 RefPtr<DOMMediaStream> mSrcStream;
1437 // The MediaStreamRenderer handles rendering of our selected video track, and
1438 // enabled audio tracks, while mSrcStream is set.
1439 RefPtr<MediaStreamRenderer> mMediaStreamRenderer;
1441 // The secondary MediaStreamRenderer handles rendering of our selected video
1442 // track to a secondary VideoFrameContainer, while mSrcStream is set.
1443 RefPtr<MediaStreamRenderer> mSecondaryMediaStreamRenderer;
1445 // True once PlaybackEnded() is called and we're playing a MediaStream.
1446 // Reset to false if we start playing mSrcStream again.
1447 Watchable<bool> mSrcStreamPlaybackEnded = {
1448 false, "HTMLMediaElement::mSrcStreamPlaybackEnded"};
1450 // Mirrors mSrcStreamPlaybackEnded after a tail dispatch when set to true,
1451 // but may be be forced to false directly. To accomodate when an application
1452 // ends playback synchronously by manipulating mSrcStream or its tracks,
1453 // e.g., through MediaStream.removeTrack(), or MediaStreamTrack.stop().
1454 bool mSrcStreamReportPlaybackEnded = false;
1456 // Holds a reference to the stream connecting this stream to the window
1457 // capture sink.
1458 UniquePtr<MediaStreamWindowCapturer> mStreamWindowCapturer;
1460 // Holds references to the DOM wrappers for the MediaStreams that we're
1461 // writing to.
1462 nsTArray<OutputMediaStream> mOutputStreams;
1464 // Mapping for output tracks, from dom::MediaTrack ids to the
1465 // MediaElementTrackSource that represents the source of all corresponding
1466 // MediaStreamTracks captured from this element.
1467 nsRefPtrHashtable<nsStringHashKey, MediaElementTrackSource>
1468 mOutputTrackSources;
1470 // The currently selected video stream track.
1471 RefPtr<VideoStreamTrack> mSelectedVideoStreamTrack;
1473 const RefPtr<ShutdownObserver> mShutdownObserver;
1475 const RefPtr<TitleChangeObserver> mTitleChangeObserver;
1477 // Holds a reference to the MediaSource, if any, referenced by the src
1478 // attribute on the media element.
1479 RefPtr<MediaSource> mSrcMediaSource;
1481 // Holds a reference to the MediaSource supplying data for playback. This
1482 // may either match mSrcMediaSource or come from Source element children.
1483 // This is set when and only when mLoadingSrc corresponds to an object url
1484 // that resolved to a MediaSource.
1485 RefPtr<MediaSource> mMediaSource;
1487 RefPtr<ChannelLoader> mChannelLoader;
1489 // Points to the child source elements, used to iterate through the children
1490 // when selecting a resource to load. This is the previous sibling of the
1491 // child considered the current 'candidate' in:
1492 // https://html.spec.whatwg.org/multipage/media.html#concept-media-load-algorithm
1494 // mSourcePointer == nullptr, we will next try to load |GetFirstChild()|.
1495 // mSourcePointer == GetLastChild(), we've exhausted all sources, waiting
1496 // for new elements to be appended.
1497 nsCOMPtr<nsIContent> mSourcePointer;
1499 // Points to the document whose load we're blocking. This is the document
1500 // we're bound to when loading starts.
1501 nsCOMPtr<Document> mLoadBlockedDoc;
1503 // This is used to help us block/resume the event delivery.
1504 class EventBlocker;
1505 RefPtr<EventBlocker> mEventBlocker;
1507 // Media loading flags. See:
1508 // http://www.whatwg.org/specs/web-apps/current-work/#video)
1509 nsMediaNetworkState mNetworkState = HTMLMediaElement_Binding::NETWORK_EMPTY;
1510 Watchable<nsMediaReadyState> mReadyState = {
1511 HTMLMediaElement_Binding::HAVE_NOTHING, "HTMLMediaElement::mReadyState"};
1513 enum LoadAlgorithmState {
1514 // No load algorithm instance is waiting for a source to be added to the
1515 // media in order to continue loading.
1516 NOT_WAITING,
1517 // We've run the load algorithm, and we tried all source children of the
1518 // media element, and failed to load any successfully. We're waiting for
1519 // another source element to be added to the media element, and will try
1520 // to load any such element when its added.
1521 WAITING_FOR_SOURCE
1524 // The current media load ID. This is incremented every time we start a
1525 // new load. Async events note the ID when they're first sent, and only fire
1526 // if the ID is unchanged when they come to fire.
1527 uint32_t mCurrentLoadID = 0;
1529 // Denotes the waiting state of a load algorithm instance. When the load
1530 // algorithm is waiting for a source element child to be added, this is set
1531 // to WAITING_FOR_SOURCE, otherwise it's NOT_WAITING.
1532 LoadAlgorithmState mLoadWaitStatus = NOT_WAITING;
1534 // Current audio volume
1535 double mVolume = 1.0;
1537 // True if the audio track is not silent.
1538 bool mIsAudioTrackAudible = false;
1540 enum MutedReasons {
1541 MUTED_BY_CONTENT = 0x01,
1542 MUTED_BY_INVALID_PLAYBACK_RATE = 0x02,
1543 MUTED_BY_AUDIO_CHANNEL = 0x04,
1544 MUTED_BY_AUDIO_TRACK = 0x08
1547 uint32_t mMuted = 0;
1549 UniquePtr<const MetadataTags> mTags;
1551 // URI of the resource we're attempting to load. This stores the value we
1552 // return in the currentSrc attribute. Use GetCurrentSrc() to access the
1553 // currentSrc attribute.
1554 // This is always the original URL we're trying to load --- before
1555 // redirects etc.
1556 nsCOMPtr<nsIURI> mLoadingSrc;
1558 // The triggering principal for the current source.
1559 nsCOMPtr<nsIPrincipal> mLoadingSrcTriggeringPrincipal;
1561 // Stores the current preload action for this element. Initially set to
1562 // PRELOAD_UNDEFINED, its value is changed by calling
1563 // UpdatePreloadAction().
1564 PreloadAction mPreloadAction = PRELOAD_UNDEFINED;
1566 // Time that the last timeupdate event was queued. Read/Write from the
1567 // main thread only.
1568 TimeStamp mQueueTimeUpdateRunnerTime;
1570 // Time that the last timeupdate event was fired. Read/Write from the
1571 // main thread only.
1572 TimeStamp mLastTimeUpdateDispatchTime;
1574 // Time that the last progress event was fired. Read/Write from the
1575 // main thread only.
1576 TimeStamp mProgressTime;
1578 // Time that data was last read from the media resource. Used for
1579 // computing if the download has stalled and to rate limit progress events
1580 // when data is arriving slower than PROGRESS_MS.
1581 // Read/Write from the main thread only.
1582 TimeStamp mDataTime;
1584 // Media 'currentTime' value when the last timeupdate event was queued.
1585 // Read/Write from the main thread only.
1586 double mLastCurrentTime = 0.0;
1588 // Logical start time of the media resource in seconds as obtained
1589 // from any media fragments. A negative value indicates that no
1590 // fragment time has been set. Read/Write from the main thread only.
1591 double mFragmentStart = -1.0;
1593 // Logical end time of the media resource in seconds as obtained
1594 // from any media fragments. A negative value indicates that no
1595 // fragment time has been set. Read/Write from the main thread only.
1596 double mFragmentEnd = -1.0;
1598 // The defaultPlaybackRate attribute gives the desired speed at which the
1599 // media resource is to play, as a multiple of its intrinsic speed.
1600 double mDefaultPlaybackRate = 1.0;
1602 // The playbackRate attribute gives the speed at which the media resource
1603 // plays, as a multiple of its intrinsic speed. If it is not equal to the
1604 // defaultPlaybackRate, then the implication is that the user is using a
1605 // feature such as fast forward or slow motion playback.
1606 double mPlaybackRate = 1.0;
1608 // True if pitch correction is applied when playbackRate is set to a
1609 // non-intrinsic value.
1610 bool mPreservesPitch = true;
1612 // Reference to the source element last returned by GetNextSource().
1613 // This is the child source element which we're trying to load from.
1614 nsCOMPtr<nsIContent> mSourceLoadCandidate;
1616 // Range of time played.
1617 RefPtr<TimeRanges> mPlayed;
1619 // Timer used for updating progress events.
1620 nsCOMPtr<nsITimer> mProgressTimer;
1622 // Encrypted Media Extension media keys.
1623 RefPtr<MediaKeys> mMediaKeys;
1624 RefPtr<MediaKeys> mIncomingMediaKeys;
1625 // The dom promise is used for HTMLMediaElement::SetMediaKeys.
1626 RefPtr<DetailedPromise> mSetMediaKeysDOMPromise;
1627 // Used to indicate if the MediaKeys attaching operation is on-going or not.
1628 bool mAttachingMediaKey = false;
1629 MozPromiseRequestHolder<SetCDMPromise> mSetCDMRequest;
1631 // Stores the time at the start of the current 'played' range.
1632 double mCurrentPlayRangeStart = 1.0;
1634 // True if loadeddata has been fired.
1635 bool mLoadedDataFired = false;
1637 // Indicates whether current playback is a result of user action
1638 // (ie. calling of the Play method), or automatic playback due to
1639 // the 'autoplay' attribute being set. A true value indicates the
1640 // latter case.
1641 // The 'autoplay' HTML attribute indicates that the video should
1642 // start playing when loaded. The 'autoplay' attribute of the object
1643 // is a mirror of the HTML attribute. These are different from this
1644 // 'mAutoplaying' flag, which indicates whether the current playback
1645 // is a result of the autoplay attribute.
1646 bool mAutoplaying = true;
1648 // Playback of the video is paused either due to calling the
1649 // 'Pause' method, or playback not yet having started.
1650 Watchable<bool> mPaused = {true, "HTMLMediaElement::mPaused"};
1652 // The following two fields are here for the private storage of the builtin
1653 // video controls, and control 'casting' of the video to external devices
1654 // (TVs, projectors etc.)
1655 // True if casting is currently allowed
1656 bool mAllowCasting = false;
1657 // True if currently casting this video
1658 bool mIsCasting = false;
1660 // Set while there are some OutputMediaStreams this media element's enabled
1661 // and selected tracks are captured into. When set, all tracks are captured
1662 // into the graph of this dummy track.
1663 // NB: This is a SharedDummyTrack to allow non-default graphs (AudioContexts
1664 // with an explicit sampleRate defined) to capture this element. When
1665 // cross-graph tracks are supported, this can become a bool.
1666 Watchable<RefPtr<SharedDummyTrack>> mTracksCaptured;
1668 // True if the sound is being captured.
1669 bool mAudioCaptured = false;
1671 // If TRUE then the media element was actively playing before the currently
1672 // in progress seeking. If FALSE then the media element is either not seeking
1673 // or was not actively playing before the current seek. Used to decide whether
1674 // to raise the 'waiting' event as per 4.7.1.8 in HTML 5 specification.
1675 bool mPlayingBeforeSeek = false;
1677 // True if this element is suspended because the document is inactive or the
1678 // inactive docshell is not allowing media to play.
1679 bool mSuspendedByInactiveDocOrDocshell = false;
1681 // True if we're running the "load()" method.
1682 bool mIsRunningLoadMethod = false;
1684 // True if we're running or waiting to run queued tasks due to an explicit
1685 // call to "load()".
1686 bool mIsDoingExplicitLoad = false;
1688 // True if we're loading the resource from the child source elements.
1689 bool mIsLoadingFromSourceChildren = false;
1691 // True if we're delaying the "load" event. They are delayed until either
1692 // an error occurs, or the first frame is loaded.
1693 bool mDelayingLoadEvent = false;
1695 // True when we've got a task queued to call SelectResource(),
1696 // or while we're running SelectResource().
1697 bool mIsRunningSelectResource = false;
1699 // True when we already have select resource call queued
1700 bool mHaveQueuedSelectResource = false;
1702 // True if we suspended the decoder because we were paused,
1703 // preloading metadata is enabled, autoplay was not enabled, and we loaded
1704 // the first frame.
1705 bool mSuspendedAfterFirstFrame = false;
1707 // True if we are allowed to suspend the decoder because we were paused,
1708 // preloading metdata was enabled, autoplay was not enabled, and we loaded
1709 // the first frame.
1710 bool mAllowSuspendAfterFirstFrame = true;
1712 // True if we've played or completed a seek. We use this to determine
1713 // when the poster frame should be shown.
1714 bool mHasPlayedOrSeeked = false;
1716 // True if we've added a reference to ourselves to keep the element
1717 // alive while no-one is referencing it but the element may still fire
1718 // events of its own accord.
1719 bool mHasSelfReference = false;
1721 // True if we've received a notification that the engine is shutting
1722 // down.
1723 bool mShuttingDown = false;
1725 // True if we've suspended a load in the resource selection algorithm
1726 // due to loading a preload:none media. When true, the resource we'll
1727 // load when the user initiates either playback or an explicit load is
1728 // stored in mPreloadURI.
1729 bool mSuspendedForPreloadNone = false;
1731 // True if we've connected mSrcStream to the media element output.
1732 bool mSrcStreamIsPlaying = false;
1734 // True if we should set nsIClassOfService::UrgentStart to the channel to
1735 // get the response ASAP for better user responsiveness.
1736 bool mUseUrgentStartForChannel = false;
1738 // The CORS mode when loading the media element
1739 CORSMode mCORSMode = CORS_NONE;
1741 // Info about the played media.
1742 MediaInfo mMediaInfo;
1744 // True if the media has encryption information.
1745 bool mIsEncrypted = false;
1747 enum WaitingForKeyState {
1748 NOT_WAITING_FOR_KEY = 0,
1749 WAITING_FOR_KEY = 1,
1750 WAITING_FOR_KEY_DISPATCHED = 2
1753 // True when the CDM cannot decrypt the current block due to lacking a key.
1754 // Note: the "waitingforkey" event is not dispatched until all decoded data
1755 // has been rendered.
1756 WaitingForKeyState mWaitingForKey = NOT_WAITING_FOR_KEY;
1758 // Listens for waitingForKey events from the owned decoder.
1759 MediaEventListener mWaitingForKeyListener;
1761 // Init Data that needs to be sent in 'encrypted' events in MetadataLoaded().
1762 EncryptionInfo mPendingEncryptedInitData;
1764 // True if the media's channel's download has been suspended.
1765 Watchable<bool> mDownloadSuspendedByCache = {
1766 false, "HTMLMediaElement::mDownloadSuspendedByCache"};
1768 // Disable the video playback by track selection. This flag might not be
1769 // enough if we ever expand the ability of supporting multi-tracks video
1770 // playback.
1771 bool mDisableVideo = false;
1773 RefPtr<TextTrackManager> mTextTrackManager;
1775 RefPtr<AudioTrackList> mAudioTrackList;
1777 RefPtr<VideoTrackList> mVideoTrackList;
1779 UniquePtr<MediaStreamTrackListener> mMediaStreamTrackListener;
1781 // The principal guarding mVideoFrameContainer access when playing a
1782 // MediaStream.
1783 nsCOMPtr<nsIPrincipal> mSrcStreamVideoPrincipal;
1785 // True if the autoplay media was blocked because it hadn't loaded metadata
1786 // yet.
1787 bool mBlockedAsWithoutMetadata = false;
1789 // This promise is used to notify MediaElementAudioSourceNode that media
1790 // element is allowed to play when MediaElement is used as a source for web
1791 // audio.
1792 MozPromiseHolder<GenericNonExclusivePromise> mAllowedToPlayPromise;
1794 // True if media has ever been blocked for autoplay, it's used to notify front
1795 // end to show the correct blocking icon when the document goes back from
1796 // bfcache.
1797 bool mHasEverBeenBlockedForAutoplay = false;
1799 // True if we have dispatched a task for text track changed, will be unset
1800 // when we starts processing text track changed.
1801 // https://html.spec.whatwg.org/multipage/media.html#pending-text-track-change-notification-flag
1802 bool mPendingTextTrackChanged = false;
1804 public:
1805 // This function will be called whenever a text track that is in a media
1806 // element's list of text tracks has its text track mode change value
1807 void NotifyTextTrackModeChanged();
1809 private:
1810 friend class nsMediaEventRunner;
1811 friend class nsResolveOrRejectPendingPlayPromisesRunner;
1813 already_AddRefed<PlayPromise> CreatePlayPromise(ErrorResult& aRv) const;
1815 virtual void MaybeBeginCloningVisually(){};
1817 uint32_t GetPreloadDefault() const;
1818 uint32_t GetPreloadDefaultAuto() const;
1821 * This function is called by AfterSetAttr and OnAttrSetButNotChanged.
1822 * It will not be called if the value is being unset.
1824 * @param aNamespaceID the namespace of the attr being set
1825 * @param aName the localname of the attribute being set
1826 * @param aNotify Whether we plan to notify document observers.
1828 void AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName, bool aNotify);
1830 // True if Init() has been called after construction
1831 bool mInitialized = false;
1833 // True if user has called load(), seek() or element has started playing
1834 // before. It's *only* use for `click-to-play` blocking autoplay policy.
1835 // In addition, we would reset this once media aborts current load.
1836 bool mIsBlessed = false;
1838 // True if the first frame has been successfully loaded.
1839 Watchable<bool> mFirstFrameLoaded = {false,
1840 "HTMLMediaElement::mFirstFrameLoaded"};
1842 // Media elements also have a default playback start position, which must
1843 // initially be set to zero seconds. This time is used to allow the element to
1844 // be seeked even before the media is loaded.
1845 double mDefaultPlaybackStartPosition = 0.0;
1847 // True if media element has been marked as 'tainted' and can't
1848 // participate in video decoder suspending.
1849 bool mHasSuspendTaint = false;
1851 // True if media element has been forced into being considered 'hidden'.
1852 // For use by mochitests. Enabling pref "media.test.video-suspend"
1853 bool mForcedHidden = false;
1855 Visibility mVisibilityState = Visibility::Untracked;
1857 UniquePtr<ErrorSink> mErrorSink;
1859 // This wrapper will handle all audio channel related stuffs, eg. the
1860 // operations of tab audio indicator, Fennec's media control. Note:
1861 // mAudioChannelWrapper might be null after GC happened.
1862 RefPtr<AudioChannelAgentCallback> mAudioChannelWrapper;
1864 // A list of pending play promises. The elements are pushed during the play()
1865 // method call and are resolved/rejected during further playback steps.
1866 nsTArray<RefPtr<PlayPromise>> mPendingPlayPromises;
1868 // A list of already-dispatched but not yet run
1869 // nsResolveOrRejectPendingPlayPromisesRunners.
1870 // Runners whose Run() method is called remove themselves from this list.
1871 // We keep track of these because the load algorithm resolves/rejects all
1872 // already-dispatched pending play promises.
1873 nsTArray<nsResolveOrRejectPendingPlayPromisesRunner*>
1874 mPendingPlayPromisesRunners;
1876 // A pending seek promise which is created at Seek() method call and is
1877 // resolved/rejected at AsyncResolveSeekDOMPromiseIfExists()/
1878 // AsyncRejectSeekDOMPromiseIfExists() methods.
1879 RefPtr<dom::Promise> mSeekDOMPromise;
1881 // Return true if the docshell is inactive and explicitly wants to stop media
1882 // playing in that shell.
1883 bool ShouldBeSuspendedByInactiveDocShell() const;
1885 // For debugging bug 1407148.
1886 void AssertReadyStateIsNothing();
1888 // Contains the unique id of the sink device and the device info.
1889 // The initial value is ("", nullptr) and the default output device is used.
1890 // It can contain an invalid id and info if the device has been
1891 // unplugged. It can be set to ("", nullptr). It follows the spec attribute:
1892 // https://w3c.github.io/mediacapture-output/#htmlmediaelement-extensions
1893 // Read/Write from the main thread only.
1894 std::pair<nsString, RefPtr<AudioDeviceInfo>> mSink;
1896 // This flag is used to control when the user agent is to show a poster frame
1897 // for a video element instead of showing the video contents.
1898 // https://html.spec.whatwg.org/multipage/media.html#show-poster-flag
1899 bool mShowPoster;
1901 // We may delay starting playback of a media for an unvisited tab until it's
1902 // going to foreground. We would create ResumeDelayedMediaPlaybackAgent to
1903 // handle related operations at the time whenever delaying media playback is
1904 // needed.
1905 void CreateResumeDelayedMediaPlaybackAgentIfNeeded();
1906 void ClearResumeDelayedMediaPlaybackAgentIfNeeded();
1907 RefPtr<ResumeDelayedPlaybackAgent> mResumeDelayedPlaybackAgent;
1908 MozPromiseRequestHolder<ResumeDelayedPlaybackAgent::ResumePromise>
1909 mResumePlaybackRequest;
1911 // Return true if we have already a decoder or a src stream and don't have any
1912 // error.
1913 bool IsPlayable() const;
1915 // Return true if the media qualifies for being controlled by media control
1916 // keys.
1917 bool ShouldStartMediaControlKeyListener() const;
1919 // Start the listener if media fits the requirement of being able to be
1920 // controlled be media control keys.
1921 void StartMediaControlKeyListenerIfNeeded();
1923 // It's used to listen media control key, by which we would play or pause
1924 // media element.
1925 RefPtr<MediaControlKeyListener> mMediaControlKeyListener;
1927 // Method to update audio stream name
1928 void UpdateStreamName();
1930 // Return true if the media element is being used in picture in picture mode.
1931 bool IsBeingUsedInPictureInPictureMode() const;
1933 // Return true if we should queue a 'timeupdate' event runner to main thread.
1934 bool ShouldQueueTimeupdateAsyncTask(TimeupdateType aType) const;
1937 // Check if the context is chrome or has the debugger or tabs permission
1938 bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj);
1940 } // namespace mozilla::dom
1942 #endif // mozilla_dom_HTMLMediaElement_h