1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #if !defined(MediaDecoder_h_)
8 # define MediaDecoder_h_
10 # include "BackgroundVideoDecodingPermissionObserver.h"
11 # include "DecoderDoctorDiagnostics.h"
12 # include "MediaContainerType.h"
13 # include "MediaDecoderOwner.h"
14 # include "MediaEventSource.h"
15 # include "MediaMetadataManager.h"
16 # include "MediaPromiseDefs.h"
17 # include "MediaResource.h"
18 # include "MediaStatistics.h"
19 # include "SeekTarget.h"
20 # include "TelemetryProbesReporter.h"
21 # include "TimeUnits.h"
22 # include "mozilla/Atomics.h"
23 # include "mozilla/CDMProxy.h"
24 # include "mozilla/MozPromise.h"
25 # include "mozilla/ReentrantMonitor.h"
26 # include "mozilla/StateMirroring.h"
27 # include "mozilla/StateWatching.h"
28 # include "mozilla/dom/MediaDebugInfoBinding.h"
29 # include "nsCOMPtr.h"
30 # include "nsIObserver.h"
31 # include "nsISupports.h"
32 # include "nsITimer.h"
34 class AudioDeviceInfo
;
41 class DecoderBenchmark
;
42 class ProcessedMediaTrack
;
43 class FrameStatistics
;
44 class VideoFrameContainer
;
45 class MediaFormatReader
;
46 class MediaDecoderStateMachineBase
;
47 struct MediaPlaybackEvent
;
48 struct SharedDummyTrack
;
51 struct DurationToType
{
52 double operator()(double aDouble
);
53 double operator()(const media::TimeUnit
& aTimeUnit
);
57 struct DurationToType
<double> {
58 double operator()(double aDouble
) { return aDouble
; }
59 double operator()(const media::TimeUnit
& aTimeUnit
) {
60 if (aTimeUnit
.IsValid()) {
61 if (aTimeUnit
.IsPosInf()) {
62 return std::numeric_limits
<double>::infinity();
64 if (aTimeUnit
.IsNegInf()) {
65 return -std::numeric_limits
<double>::infinity();
67 return aTimeUnit
.ToSeconds();
69 return std::numeric_limits
<double>::quiet_NaN();
73 using DurationToDouble
= DurationToType
<double>;
76 struct DurationToType
<media::TimeUnit
> {
77 media::TimeUnit
operator()(double aDouble
) {
78 return media::TimeUnit::FromSeconds(aDouble
);
80 media::TimeUnit
operator()(const media::TimeUnit
& aTimeUnit
) {
85 using DurationToTimeUnit
= DurationToType
<media::TimeUnit
>;
87 struct MOZ_STACK_CLASS MediaDecoderInit
{
88 MediaDecoderOwner
* const mOwner
;
89 TelemetryProbesReporterOwner
* const mReporterOwner
;
91 const bool mPreservesPitch
;
92 const double mPlaybackRate
;
93 const bool mMinimizePreroll
;
94 const bool mHasSuspendTaint
;
96 const MediaContainerType mContainerType
;
97 const nsAutoString mStreamName
;
99 MediaDecoderInit(MediaDecoderOwner
* aOwner
,
100 TelemetryProbesReporterOwner
* aReporterOwner
, double aVolume
,
101 bool aPreservesPitch
, double aPlaybackRate
,
102 bool aMinimizePreroll
, bool aHasSuspendTaint
, bool aLooping
,
103 const MediaContainerType
& aContainerType
)
105 mReporterOwner(aReporterOwner
),
107 mPreservesPitch(aPreservesPitch
),
108 mPlaybackRate(aPlaybackRate
),
109 mMinimizePreroll(aMinimizePreroll
),
110 mHasSuspendTaint(aHasSuspendTaint
),
112 mContainerType(aContainerType
) {}
115 DDLoggedTypeDeclName(MediaDecoder
);
117 class MediaDecoder
: public DecoderDoctorLifeLogger
<MediaDecoder
> {
119 typedef MozPromise
<bool /* aIgnored */, bool /* aIgnored */,
120 /* IsExclusive = */ true>
123 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoder
)
125 // Enumeration for the valid play states (see mPlayState)
134 // Must be called exactly once, on the main thread, during startup.
135 static void InitStatics();
137 explicit MediaDecoder(MediaDecoderInit
& aInit
);
139 // Returns the container content type of the resource.
140 // Safe to call from any thread.
141 const MediaContainerType
& ContainerType() const { return mContainerType
; }
143 // Cleanup internal data structures. Must be called on the main
144 // thread by the owning object before that object disposes of this object.
145 virtual void Shutdown();
147 // Notified by the shutdown manager that XPCOM shutdown has begun.
148 // The decoder should notify its owner to drop the reference to the decoder
149 // to prevent further calls into the decoder.
150 void NotifyXPCOMShutdown();
152 // Called if the media file encounters a network error.
153 void NetworkError(const MediaResult
& aError
);
155 // Return the principal of the current URI being played or downloaded.
156 virtual already_AddRefed
<nsIPrincipal
> GetCurrentPrincipal() = 0;
158 // Return true if the loading of this resource required cross-origin
160 virtual bool HadCrossOriginRedirects() = 0;
162 // Return the time position in the video stream being
163 // played measured in seconds.
164 virtual double GetCurrentTime();
166 // Seek to the time position in (seconds) from the start of the video.
167 // If aDoFastSeek is true, we'll seek to the sync point/keyframe preceeding
169 void Seek(double aTime
, SeekTarget::Type aSeekType
);
171 // Start playback of a video. 'Load' must have previously been
175 // Notify activity of the decoder owner is changed.
176 virtual void NotifyOwnerActivityChanged(bool aIsOwnerInvisible
,
177 bool aIsOwnerConnected
);
179 // Pause video playback.
180 virtual void Pause();
181 // Adjust the speed of the playback, optionally with pitch correction,
182 void SetVolume(double aVolume
);
184 void SetPlaybackRate(double aPlaybackRate
);
185 void SetPreservesPitch(bool aPreservesPitch
);
186 void SetLooping(bool aLooping
);
187 void SetStreamName(const nsAutoString
& aStreamName
);
189 // Set the given device as the output device.
190 RefPtr
<GenericPromise
> SetSink(AudioDeviceInfo
* aSinkDevice
);
192 bool GetMinimizePreroll() const { return mMinimizePreroll
; }
194 // When we enable delay seek mode, media decoder won't actually ask MDSM to do
195 // seeking. During this period, we would store the latest seeking target and
196 // perform the seek to that target when we leave the mode. If we have any
197 // delayed seeks stored `IsSeeking()` will return true. E.g. During delay
198 // seeking mode, if we get seek target to 5s, 10s, 7s. When we stop delaying
199 // seeking, we would only seek to 7s.
200 void SetDelaySeekMode(bool aShouldDelaySeek
);
202 // All MediaStream-related data is protected by mReentrantMonitor.
203 // We have at most one DecodedStreamData per MediaDecoder. Its stream
204 // is used as the input for each ProcessedMediaTrack created by calls to
205 // captureStream(UntilEnded). Seeking creates a new source stream, as does
206 // replaying after the input as ended. In the latter case, the new source is
207 // not connected to streams created by captureStreamUntilEnded.
209 enum class OutputCaptureState
{ Capture
, Halt
, None
};
210 const char* OutputCaptureStateToStr(OutputCaptureState aState
) const {
212 case OutputCaptureState::Capture
:
214 case OutputCaptureState::Halt
:
216 case OutputCaptureState::None
:
219 MOZ_ASSERT_UNREACHABLE("Not defined state!");
220 return "Not-defined";
224 // Set the output capture state of this decoder.
225 // @param aState Capture: Output is captured into output tracks, and
226 // aDummyTrack must be provided.
227 // Halt: A capturing media sink is used, but capture is
229 // None: Output is not captured.
230 // @param aDummyTrack A SharedDummyTrack the capturing media sink can use to
231 // access a MediaTrackGraph, so it can create tracks even
232 // when there are no output tracks available.
233 void SetOutputCaptureState(OutputCaptureState aState
,
234 SharedDummyTrack
* aDummyTrack
= nullptr);
235 // Add an output track. All decoder output for the track's media type will be
236 // sent to the track.
237 // Note that only one audio track and one video track is supported by
238 // MediaDecoder at this time. Passing in more of one type, or passing in a
239 // type that metadata says we are not decoding, is an error.
240 void AddOutputTrack(RefPtr
<ProcessedMediaTrack
> aTrack
);
241 // Remove an output track added with AddOutputTrack.
242 void RemoveOutputTrack(const RefPtr
<ProcessedMediaTrack
>& aTrack
);
243 // Update the principal for any output tracks.
244 void SetOutputTracksPrincipal(const RefPtr
<nsIPrincipal
>& aPrincipal
);
246 // Return the duration of the video in seconds.
247 virtual double GetDuration();
249 // Return true if the stream is infinite.
250 bool IsInfinite() const;
252 // Return true if we are currently seeking in the media resource.
253 // Call on the main thread only.
254 bool IsSeeking() const;
256 // Return true if the decoder has reached the end of playback.
257 bool IsEnded() const;
259 // True if we are playing a MediaSource object.
260 virtual bool IsMSE() const { return false; }
262 // Return true if the MediaDecoderOwner's error attribute is not null.
263 // Must be called before Shutdown().
264 bool OwnerHasError() const;
266 // Returns true if this media supports random seeking. False for example with
267 // chained ogg files.
268 bool IsMediaSeekable();
269 // Returns true if seeking is supported on a transport level (e.g. the server
270 // supports range requests, we are playing a file, etc.).
271 virtual bool IsTransportSeekable() = 0;
273 // Return the time ranges that can be seeked into, in TimeUnits.
274 virtual media::TimeIntervals
GetSeekable();
275 // Return the time ranges that can be seeked into, in seconds, double
277 virtual media::TimeRanges
GetSeekableTimeRanges();
279 template <typename T
>
282 // Set the end time of the media resource. When playback reaches
283 // this point the media pauses. aTime is in seconds.
284 virtual void SetFragmentEndTime(double aTime
);
286 // Invalidate the frame.
288 void InvalidateWithFlags(uint32_t aFlags
);
290 // Suspend any media downloads that are in progress. Called by the
291 // media element when it is sent to the bfcache, or when we need
292 // to throttle the download. Call on the main thread only. This can
293 // be called multiple times, there's an internal "suspend count".
294 // When it is called the internal system audio resource are cleaned up.
295 virtual void Suspend();
297 // Resume any media downloads that have been suspended. Called by the
298 // media element when it is restored from the bfcache, or when we need
299 // to stop throttling the download. Call on the main thread only.
300 // The download will only actually resume once as many Resume calls
301 // have been made as Suspend calls.
302 virtual void Resume();
304 // Moves any existing channel loads into or out of background. Background
305 // loads don't block the load event. This is called when we stop or restart
306 // delaying the load event. This also determines whether any new loads
307 // initiated (for example to seek) will be in the background. This calls
308 // SetLoadInBackground() on mResource.
309 virtual void SetLoadInBackground(bool aLoadInBackground
) {}
311 MediaDecoderStateMachineBase
* GetStateMachine() const;
312 void SetStateMachine(MediaDecoderStateMachineBase
* aStateMachine
);
314 // Constructs the time ranges representing what segments of the media
315 // are buffered and playable.
316 virtual media::TimeIntervals
GetBuffered();
318 // Returns the size, in bytes, of the heap memory used by the currently
319 // queued decoded video and audio data.
320 size_t SizeOfVideoQueue();
321 size_t SizeOfAudioQueue();
323 // Helper struct for accumulating resource sizes that need to be measured
324 // asynchronously. Once all references are dropped the callback will be
326 struct ResourceSizes
{
327 typedef MozPromise
<size_t, size_t, true> SizeOfPromise
;
328 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ResourceSizes
)
329 explicit ResourceSizes(MallocSizeOf aMallocSizeOf
)
330 : mMallocSizeOf(aMallocSizeOf
), mByteSize(0), mCallback() {}
332 mozilla::MallocSizeOf mMallocSizeOf
;
333 mozilla::Atomic
<size_t> mByteSize
;
335 RefPtr
<SizeOfPromise
> Promise() { return mCallback
.Ensure(__func__
); }
338 ~ResourceSizes() { mCallback
.ResolveIfExists(mByteSize
, __func__
); }
340 MozPromiseHolder
<SizeOfPromise
> mCallback
;
343 virtual void AddSizeOfResources(ResourceSizes
* aSizes
) = 0;
345 VideoFrameContainer
* GetVideoFrameContainer() { return mVideoFrameContainer
; }
347 layers::ImageContainer
* GetImageContainer();
349 // Returns true if we can play the entire media through without stopping
350 // to buffer, given the current download and playback rates.
351 bool CanPlayThrough();
353 // Called from HTMLMediaElement when owner document activity changes
354 virtual void SetElementVisibility(bool aIsOwnerInvisible
,
355 bool aIsOwnerConnected
);
357 // Force override the visible state to hidden.
358 // Called from HTMLMediaElement when testing of video decode suspend from
360 void SetForcedHidden(bool aForcedHidden
);
362 // Mark the decoder as tainted, meaning suspend-video-decoder is disabled.
363 void SetSuspendTaint(bool aTaint
);
365 // Returns true if the decoder can't participate in suspend-video-decoder.
366 bool HasSuspendTaint() const;
368 void UpdateVideoDecodeMode();
370 void SetSecondaryVideoContainer(
371 const RefPtr
<VideoFrameContainer
>& aSecondaryVideoContainer
);
373 void SetIsBackgroundVideoDecodingAllowed(bool aAllowed
);
375 bool IsVideoDecodingSuspended() const;
377 // The MediaDecoderOwner of this decoder wants to resist fingerprinting.
378 bool ShouldResistFingerprinting() const {
379 return mShouldResistFingerprinting
;
383 * The following methods must only be called on the main
387 // Change to a new play state. This updates the mState variable and
388 // notifies any thread blocking on this object's monitor of the
389 // change. Call on the main thread only.
390 virtual void ChangeState(PlayState aState
);
392 // Called when the video has completed playing.
393 // Call on the main thread only.
394 void PlaybackEnded();
396 void OnSeekRejected();
397 void OnSeekResolved();
399 // Seeking has started. Inform the element on the main thread.
400 void SeekingStarted();
402 void UpdateLogicalPositionInternal();
403 void UpdateLogicalPosition() {
404 MOZ_ASSERT(NS_IsMainThread());
405 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
406 // Per spec, offical position remains stable during pause and seek.
407 if (mPlayState
== PLAY_STATE_PAUSED
|| IsSeeking()) {
410 UpdateLogicalPositionInternal();
413 // Find the end of the cached data starting at the current decoder
415 int64_t GetDownloadPosition();
417 // Notifies the element that decoding has failed.
418 void DecodeError(const MediaResult
& aError
);
420 // Indicate whether the media is same-origin with the element.
421 void UpdateSameOriginStatus(bool aSameOrigin
);
423 MediaDecoderOwner
* GetOwner() const;
425 AbstractThread
* AbstractMainThread() const { return mAbstractMainThread
; }
427 RefPtr
<SetCDMPromise
> SetCDMProxy(CDMProxy
* aProxy
);
429 void EnsureTelemetryReported();
431 static bool IsOggEnabled();
432 static bool IsOpusEnabled();
433 static bool IsWaveEnabled();
434 static bool IsWebMEnabled();
436 // Return the frame decode/paint related statistics.
437 FrameStatistics
& GetFrameStatistics() { return *mFrameStats
; }
439 void UpdateReadyState() {
440 MOZ_ASSERT(NS_IsMainThread());
441 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
442 GetOwner()->UpdateReadyState();
445 MediaDecoderOwner::NextFrameStatus
NextFrameStatus() const {
446 return mNextFrameStatus
;
449 virtual MediaDecoderOwner::NextFrameStatus
NextFrameBufferedStatus();
451 RefPtr
<GenericPromise
> RequestDebugInfo(dom::MediaDecoderDebugInfo
& aInfo
);
453 void GetDebugInfo(dom::MediaDecoderDebugInfo
& aInfo
);
455 virtual bool IsHLSDecoder() const { return false; }
458 virtual ~MediaDecoder();
460 // Called when the first audio and/or video from the media file has been
461 // loaded by the state machine. Call on the main thread only.
462 virtual void FirstFrameLoaded(UniquePtr
<MediaInfo
> aInfo
,
463 MediaDecoderEventVisibility aEventVisibility
);
465 // Return error if fail to init the state machine.
466 nsresult
CreateAndInitStateMachine(bool aIsLiveStream
,
467 bool aDisableExternalEngine
= false);
469 // Always return a state machine. If the decoder supports using external
470 // engine, `aDisableExternalEngine` can disable the external engine if needed.
471 virtual MediaDecoderStateMachineBase
* CreateStateMachine(
472 bool aDisableExternalEngine
) MOZ_NONNULL_RETURN
= 0;
474 void SetStateMachineParameters();
476 // Disconnect any events before shutting down the state machine.
477 void DisconnectEvents();
478 RefPtr
<ShutdownPromise
> ShutdownStateMachine();
480 // Called when MediaDecoder shutdown is finished. Subclasses use this to clean
481 // up internal structures, and unregister potential shutdown blockers when
483 virtual void ShutdownInternal();
485 bool IsShutdown() const;
487 // Called to notify the decoder that the duration has changed.
488 virtual void DurationChanged();
490 // State-watching manager.
491 WatchManager
<MediaDecoder
> mWatchManager
;
493 double ExplicitDuration() { return mExplicitDuration
.ref(); }
495 void SetExplicitDuration(double aValue
) {
496 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
497 mExplicitDuration
= Some(aValue
);
499 // We Invoke DurationChanged explicitly, rather than using a watcher, so
500 // that it takes effect immediately, rather than at the end of the current
505 virtual void OnPlaybackEvent(MediaPlaybackEvent
&& aEvent
);
507 // Called when the metadata from the media file has been loaded by the
508 // state machine. Call on the main thread only.
509 virtual void MetadataLoaded(UniquePtr
<MediaInfo
> aInfo
,
510 UniquePtr
<MetadataTags
> aTags
,
511 MediaDecoderEventVisibility aEventVisibility
);
513 void SetLogicalPosition(const media::TimeUnit
& aNewPosition
);
516 * The following members should be accessed with the decoder lock held.
519 // The logical playback position of the media resource in units of
520 // seconds. This corresponds to the "official position" in HTML5. Note that
521 // we need to store this as a double, rather than an int64_t (like
522 // mCurrentPosition), so that |v.currentTime = foo; v.currentTime == foo|
523 // returns true without being affected by rounding errors.
524 double mLogicalPosition
;
526 // The current playback position of the underlying playback infrastructure.
527 // This corresponds to the "current position" in HTML5.
528 // We allow omx subclasses to substitute an alternative current position for
529 // usage with the audio offload player.
530 virtual media::TimeUnit
CurrentPosition() { return mCurrentPosition
.Ref(); }
532 already_AddRefed
<layers::KnowsCompositor
> GetCompositor();
534 // Official duration of the media resource as observed by script.
535 // This can be a TimeUnit representing the exact duration found by demuxing,
536 // as a TimeUnit. This can also be a duration set explicitly by script, as a
538 Variant
<media::TimeUnit
, double> mDuration
;
541 * The following member variables can be accessed from any thread.
544 RefPtr
<MediaFormatReader
> mReader
;
546 // Amount of buffered data ahead of current time required to consider that
547 // the next frame is available.
548 // An arbitrary value of 250ms is used.
549 static constexpr auto DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED
=
550 media::TimeUnit::FromMicroseconds(250000);
553 // Called when the owner's activity changed.
554 void NotifyCompositor();
556 void OnPlaybackErrorEvent(const MediaResult
& aError
);
558 void OnDecoderDoctorEvent(DecoderDoctorEvent aEvent
);
560 void OnMediaNotSeekable() { mMediaSeekable
= false; }
562 void OnNextFrameStatus(MediaDecoderOwner::NextFrameStatus
);
564 void OnTrackInfoUpdated(const VideoInfo
& aVideoInfo
,
565 const AudioInfo
& aAudioInfo
);
567 void OnSecondaryVideoContainerInstalled(
568 const RefPtr
<VideoFrameContainer
>& aSecondaryVideoContainer
);
570 void OnStoreDecoderBenchmark(const VideoInfo
& aInfo
);
572 void FinishShutdown();
574 void ConnectMirrors(MediaDecoderStateMachineBase
* aObject
);
575 void DisconnectMirrors();
577 virtual bool CanPlayThroughImpl() = 0;
579 // The state machine object for handling the decoding. It is safe to
580 // call methods of this object from other threads. Its internal data
581 // is synchronised on a monitor. The lifetime of this object is
582 // after mPlayState is LOADING and before mPlayState is SHUTDOWN. It
583 // is safe to access it during this period.
585 // Explicitly prievate to force access via accessors.
586 RefPtr
<MediaDecoderStateMachineBase
> mDecoderStateMachine
;
589 void NotifyReaderDataArrived();
590 void DiscardOngoingSeekIfExists();
591 void CallSeek(const SeekTarget
& aTarget
);
593 // Called by MediaResource when the principal of the resource has
594 // changed. Called on main thread only.
595 virtual void NotifyPrincipalChanged();
597 MozPromiseRequestHolder
<SeekPromise
> mSeekRequest
;
599 const char* PlayStateStr();
601 void OnMetadataUpdate(TimedMetadata
&& aMetadata
);
603 // This should only ever be accessed from the main thread.
604 // It is set in the constructor and cleared in Shutdown when the element goes
605 // away. The decoder does not add a reference the element.
606 MediaDecoderOwner
* mOwner
;
608 // The AbstractThread from mOwner.
609 const RefPtr
<AbstractThread
> mAbstractMainThread
;
611 // Counters related to decode and presentation of frames.
612 const RefPtr
<FrameStatistics
> mFrameStats
;
614 // Store a benchmark of the decoder based on FrameStatistics.
615 RefPtr
<DecoderBenchmark
> mDecoderBenchmark
;
617 RefPtr
<VideoFrameContainer
> mVideoFrameContainer
;
619 // True if the decoder has been directed to minimize its preroll before
620 // playback starts. After the first time playback starts, we don't attempt
621 // to minimize preroll, as we assume the user is likely to keep playing,
622 // or play the media again.
623 const bool mMinimizePreroll
;
625 // True if we've already fired metadataloaded.
626 bool mFiredMetadataLoaded
;
628 // True if the media is seekable (i.e. supports random access).
629 bool mMediaSeekable
= true;
631 // True if the media is only seekable within its buffered ranges
632 // like WebMs with no cues.
633 bool mMediaSeekableOnlyInBufferedRanges
= false;
635 // Stores media info, including info of audio tracks and video tracks, should
636 // only be accessed from main thread.
637 UniquePtr
<MediaInfo
> mInfo
;
639 // True if the owner element is actually visible to users.
640 bool mIsOwnerInvisible
;
642 // True if the owner element is connected to a document tree.
643 // https://dom.spec.whatwg.org/#connected
644 bool mIsOwnerConnected
;
646 // If true, forces the decoder to be considered hidden.
649 // True if the decoder has a suspend taint - meaning suspend-video-decoder is
651 bool mHasSuspendTaint
;
653 // If true, the decoder should resist fingerprinting.
654 const bool mShouldResistFingerprinting
;
656 MediaDecoderOwner::NextFrameStatus mNextFrameStatus
=
657 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE
;
659 // A listener to receive metadata updates from MDSM.
660 MediaEventListener mTimedMetadataListener
;
662 MediaEventListener mMetadataLoadedListener
;
663 MediaEventListener mFirstFrameLoadedListener
;
665 MediaEventListener mOnPlaybackEvent
;
666 MediaEventListener mOnPlaybackErrorEvent
;
667 MediaEventListener mOnDecoderDoctorEvent
;
668 MediaEventListener mOnMediaNotSeekable
;
669 MediaEventListener mOnEncrypted
;
670 MediaEventListener mOnWaitingForKey
;
671 MediaEventListener mOnDecodeWarning
;
672 MediaEventListener mOnNextFrameStatus
;
673 MediaEventListener mOnTrackInfoUpdated
;
674 MediaEventListener mOnSecondaryVideoContainerInstalled
;
675 MediaEventListener mOnStoreDecoderBenchmark
;
677 // True if we have suspended video decoding.
678 bool mIsVideoDecodingSuspended
= false;
681 // PlaybackRate and pitch preservation status we should start at.
682 double mPlaybackRate
;
684 // True if the decoder is seeking.
685 Watchable
<bool> mLogicallySeeking
;
687 // Buffered range, mirrored from the reader.
688 Mirror
<media::TimeIntervals
> mBuffered
;
690 // NB: Don't use mCurrentPosition directly, but rather CurrentPosition().
691 Mirror
<media::TimeUnit
> mCurrentPosition
;
693 // Duration of the media resource according to the state machine.
694 Mirror
<media::NullableTimeUnit
> mStateMachineDuration
;
696 // Used to distinguish whether the audio is producing sound.
697 Mirror
<bool> mIsAudioDataAudible
;
699 // Volume of playback. 0.0 = muted. 1.0 = full volume.
700 Canonical
<double> mVolume
;
702 Canonical
<bool> mPreservesPitch
;
704 Canonical
<bool> mLooping
;
706 Canonical
<nsAutoString
> mStreamName
;
708 // The device used with SetSink, or nullptr if no explicit device has been
710 Canonical
<RefPtr
<AudioDeviceInfo
>> mSinkDevice
;
712 // Set if the decoder is sending video to a secondary container. While set we
713 // should not suspend the decoder.
714 Canonical
<RefPtr
<VideoFrameContainer
>> mSecondaryVideoContainer
;
716 // Whether this MediaDecoder's output is captured, halted or not captured.
717 // When captured, all decoded data must be played out through mOutputTracks.
718 Canonical
<OutputCaptureState
> mOutputCaptureState
;
720 // A dummy track used to access the right MediaTrackGraph instance. Needed
721 // since there's no guarantee that output tracks are present.
722 Canonical
<nsMainThreadPtrHandle
<SharedDummyTrack
>> mOutputDummyTrack
;
724 // Tracks that, if set, will get data routed through them.
725 Canonical
<CopyableTArray
<RefPtr
<ProcessedMediaTrack
>>> mOutputTracks
;
727 // PrincipalHandle to be used when feeding data into mOutputTracks.
728 Canonical
<PrincipalHandle
> mOutputPrincipal
;
730 // Media duration set explicitly by JS. At present, this is only ever present
732 Maybe
<double> mExplicitDuration
;
734 // Set to one of the valid play states.
735 // This can only be changed on the main thread while holding the decoder
736 // monitor. Thus, it can be safely read while holding the decoder monitor
737 // OR on the main thread.
738 Canonical
<PlayState
> mPlayState
;
740 // This can only be changed on the main thread.
741 PlayState mNextState
= PLAY_STATE_PAUSED
;
743 // True if the media is same-origin with the element. Data can only be
744 // passed to MediaStreams when this is true.
745 bool mSameOriginMedia
;
747 // We can allow video decoding in background when we match some special
748 // conditions, eg. when the cursor is hovering over the tab. This observer is
749 // used to listen the related events.
750 RefPtr
<BackgroundVideoDecodingPermissionObserver
> mVideoDecodingOberver
;
752 // True if we want to resume video decoding even the media element is in the
754 bool mIsBackgroundVideoDecodingAllowed
;
756 // True if we want to delay seeking, and and save the latest seeking target to
757 // resume to when we stop delaying seeking.
758 bool mShouldDelaySeek
= false;
759 Maybe
<SeekTarget
> mDelayedSeekTarget
;
761 # ifdef MOZ_WMF_MEDIA_ENGINE
762 // True if we've ever recreated a new state machine due to the previous state
763 // didn't support the media format or key system.
764 bool mStateMachineRecreated
= false;
768 Canonical
<double>& CanonicalVolume() { return mVolume
; }
769 Canonical
<bool>& CanonicalPreservesPitch() { return mPreservesPitch
; }
770 Canonical
<bool>& CanonicalLooping() { return mLooping
; }
771 Canonical
<nsAutoString
>& CanonicalStreamName() { return mStreamName
; }
772 Canonical
<RefPtr
<AudioDeviceInfo
>>& CanonicalSinkDevice() {
775 Canonical
<RefPtr
<VideoFrameContainer
>>& CanonicalSecondaryVideoContainer() {
776 return mSecondaryVideoContainer
;
778 Canonical
<OutputCaptureState
>& CanonicalOutputCaptureState() {
779 return mOutputCaptureState
;
781 Canonical
<nsMainThreadPtrHandle
<SharedDummyTrack
>>&
782 CanonicalOutputDummyTrack() {
783 return mOutputDummyTrack
;
785 Canonical
<CopyableTArray
<RefPtr
<ProcessedMediaTrack
>>>&
786 CanonicalOutputTracks() {
787 return mOutputTracks
;
789 Canonical
<PrincipalHandle
>& CanonicalOutputPrincipal() {
790 return mOutputPrincipal
;
792 Canonical
<PlayState
>& CanonicalPlayState() { return mPlayState
; }
794 void UpdateTelemetryHelperBasedOnPlayState(PlayState aState
) const;
796 TelemetryProbesReporter::Visibility
OwnerVisibility() const;
798 // Those methods exist to report telemetry related metrics.
799 double GetTotalVideoPlayTimeInSeconds() const;
800 double GetTotalVideoHDRPlayTimeInSeconds() const;
801 double GetVisibleVideoPlayTimeInSeconds() const;
802 double GetInvisibleVideoPlayTimeInSeconds() const;
803 double GetVideoDecodeSuspendedTimeInSeconds() const;
804 double GetTotalAudioPlayTimeInSeconds() const;
805 double GetAudiblePlayTimeInSeconds() const;
806 double GetInaudiblePlayTimeInSeconds() const;
807 double GetMutedPlayTimeInSeconds() const;
811 * This enum describes the reason why we need to update the logical position.
812 * ePeriodicUpdate : the position grows periodically during playback
813 * eSeamlessLoopingSeeking : the position changes due to demuxer level seek.
814 * eOther : due to normal seeking or other attributes changes, eg. playstate
816 enum class PositionUpdate
{
818 eSeamlessLoopingSeeking
,
821 PositionUpdate
GetPositionUpdateReason(double aPrevPos
,
822 const media::TimeUnit
& aCurPos
) const;
824 // Notify owner when the audible state changed
825 void NotifyAudibleStateChanged();
827 void NotifyVolumeChanged();
829 bool mTelemetryReported
;
830 const MediaContainerType mContainerType
;
831 bool mCanPlayThrough
= false;
833 UniquePtr
<TelemetryProbesReporter
> mTelemetryProbesReporter
;
835 # ifdef MOZ_WMF_MEDIA_ENGINE
836 // True when we need to update the newly created MDSM's status to make it
837 // consistent with the previous destroyed one.
838 bool mPendingStatusUpdateForNewlyCreatedStateMachine
= false;
841 // The time of creating the media decoder state machine, it's used to record
842 // the probe for measuring the first video frame loaded time. Reset after
843 // reporting the measurement to avoid a dulpicated report.
844 Maybe
<TimeStamp
> mMDSMCreationTime
;
847 } // namespace mozilla