1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "mozilla/dom/HTMLVideoElement.h"
9 #include "mozilla/AsyncEventDispatcher.h"
10 #include "mozilla/dom/HTMLVideoElementBinding.h"
11 #include "nsGenericHTMLElement.h"
12 #include "nsGkAtoms.h"
15 #include "nsNodeInfoManager.h"
18 #include "nsThreadUtils.h"
19 #include "ImageContainer.h"
20 #include "VideoFrameContainer.h"
21 #include "VideoOutput.h"
23 #include "FrameStatistics.h"
24 #include "MediaError.h"
25 #include "MediaDecoder.h"
26 #include "MediaDecoderStateMachine.h"
27 #include "mozilla/Preferences.h"
28 #include "mozilla/dom/WakeLock.h"
29 #include "mozilla/dom/power/PowerManagerService.h"
30 #include "mozilla/dom/Performance.h"
31 #include "mozilla/dom/TimeRanges.h"
32 #include "mozilla/dom/VideoPlaybackQuality.h"
33 #include "mozilla/dom/VideoStreamTrack.h"
34 #include "mozilla/StaticPrefs_media.h"
35 #include "mozilla/Unused.h"
40 nsGenericHTMLElement
* NS_NewHTMLVideoElement(
41 already_AddRefed
<mozilla::dom::NodeInfo
>&& aNodeInfo
,
42 mozilla::dom::FromParser aFromParser
) {
43 RefPtr
<mozilla::dom::NodeInfo
> nodeInfo(aNodeInfo
);
44 auto* nim
= nodeInfo
->NodeInfoManager();
45 mozilla::dom::HTMLVideoElement
* element
=
46 new (nim
) mozilla::dom::HTMLVideoElement(nodeInfo
.forget());
54 class HTMLVideoElement::SecondaryVideoOutput
: public VideoOutput
{
56 WeakPtr
<HTMLMediaElement
> mElement
;
59 SecondaryVideoOutput(HTMLMediaElement
* aElement
,
60 VideoFrameContainer
* aContainer
,
61 AbstractThread
* aMainThread
)
62 : VideoOutput(aContainer
, aMainThread
), mElement(aElement
) {}
65 MOZ_ASSERT(NS_IsMainThread());
69 void NotifyDirectListenerInstalled(InstallationResult aResult
) override
{
70 if (aResult
== InstallationResult::SUCCESS
) {
71 mMainThread
->Dispatch(NS_NewRunnableFunction(
72 "HTMLMediaElement::OnSecondaryVideoContainerInstalled",
73 [self
= RefPtr
<SecondaryVideoOutput
>(this), this] {
75 mElement
->OnSecondaryVideoContainerInstalled(
76 mVideoFrameContainer
);
83 nsresult
HTMLVideoElement::Clone(mozilla::dom::NodeInfo
* aNodeInfo
,
84 nsINode
** aResult
) const {
86 RefPtr
<mozilla::dom::NodeInfo
> ni(aNodeInfo
);
87 auto* nim
= ni
->NodeInfoManager();
88 HTMLVideoElement
* it
= new (nim
) HTMLVideoElement(ni
.forget());
90 nsCOMPtr
<nsINode
> kungFuDeathGrip
= it
;
91 nsresult rv
= const_cast<HTMLVideoElement
*>(this)->CopyInnerTo(it
);
92 if (NS_SUCCEEDED(rv
)) {
93 kungFuDeathGrip
.swap(*aResult
);
98 NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLVideoElement
,
101 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLVideoElement
)
103 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(HTMLVideoElement
)
104 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTarget
)
105 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTargetPromise
)
106 if (tmp
->mSecondaryVideoOutput
) {
107 MOZ_DIAGNOSTIC_ASSERT(tmp
->mSelectedVideoStreamTrack
);
108 tmp
->mSelectedVideoStreamTrack
->RemoveVideoOutput(
109 tmp
->mSecondaryVideoOutput
);
110 tmp
->mSecondaryVideoOutput
->Forget();
111 tmp
->mSecondaryVideoOutput
= nullptr;
113 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneSource
)
114 NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(HTMLMediaElement
)
116 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement
,
118 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneTarget
)
119 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneTargetPromise
)
120 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneSource
)
121 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
123 HTMLVideoElement::HTMLVideoElement(already_AddRefed
<NodeInfo
>&& aNodeInfo
)
124 : HTMLMediaElement(std::move(aNodeInfo
)), mIsOrientationLocked(false) {
125 DecoderDoctorLogger::LogConstruction(this);
128 HTMLVideoElement::~HTMLVideoElement() {
129 DecoderDoctorLogger::LogDestruction(this);
132 void HTMLVideoElement::UpdateMediaSize(const nsIntSize
& aSize
) {
133 HTMLMediaElement::UpdateMediaSize(aSize
);
134 // If we have a clone target, we should update its size as well.
135 if (mVisualCloneTarget
) {
136 Maybe
<nsIntSize
> newSize
= Some(aSize
);
137 mVisualCloneTarget
->Invalidate(true, newSize
, true);
141 nsresult
HTMLVideoElement::GetVideoSize(nsIntSize
* size
) {
142 if (!mMediaInfo
.HasVideo()) {
143 return NS_ERROR_FAILURE
;
147 return NS_ERROR_FAILURE
;
150 switch (mMediaInfo
.mVideo
.mRotation
) {
151 case VideoInfo::Rotation::kDegree_90
:
152 case VideoInfo::Rotation::kDegree_270
: {
153 size
->width
= mMediaInfo
.mVideo
.mDisplay
.height
;
154 size
->height
= mMediaInfo
.mVideo
.mDisplay
.width
;
157 case VideoInfo::Rotation::kDegree_0
:
158 case VideoInfo::Rotation::kDegree_180
:
160 size
->height
= mMediaInfo
.mVideo
.mDisplay
.height
;
161 size
->width
= mMediaInfo
.mVideo
.mDisplay
.width
;
168 void HTMLVideoElement::Invalidate(bool aImageSizeChanged
,
169 Maybe
<nsIntSize
>& aNewIntrinsicSize
,
170 bool aForceInvalidate
) {
171 HTMLMediaElement::Invalidate(aImageSizeChanged
, aNewIntrinsicSize
,
173 if (mVisualCloneTarget
) {
174 VideoFrameContainer
* container
=
175 mVisualCloneTarget
->GetVideoFrameContainer();
177 container
->Invalidate();
182 bool HTMLVideoElement::ParseAttribute(int32_t aNamespaceID
, nsAtom
* aAttribute
,
183 const nsAString
& aValue
,
184 nsIPrincipal
* aMaybeScriptedPrincipal
,
185 nsAttrValue
& aResult
) {
186 if (aAttribute
== nsGkAtoms::width
|| aAttribute
== nsGkAtoms::height
) {
187 return aResult
.ParseHTMLDimension(aValue
);
190 return HTMLMediaElement::ParseAttribute(aNamespaceID
, aAttribute
, aValue
,
191 aMaybeScriptedPrincipal
, aResult
);
194 void HTMLVideoElement::MapAttributesIntoRule(
195 const nsMappedAttributes
* aAttributes
, MappedDeclarations
& aDecls
) {
196 nsGenericHTMLElement::MapImageSizeAttributesInto(aAttributes
, aDecls
);
197 nsGenericHTMLElement::MapCommonAttributesInto(aAttributes
, aDecls
);
201 HTMLVideoElement::IsAttributeMapped(const nsAtom
* aAttribute
) const {
202 static const MappedAttributeEntry attributes
[] = {
203 {nsGkAtoms::width
}, {nsGkAtoms::height
}, {nullptr}};
205 static const MappedAttributeEntry
* const map
[] = {attributes
,
206 sCommonAttributeMap
};
208 return FindAttributeDependence(aAttribute
, map
);
211 nsMapRuleToAttributesFunc
HTMLVideoElement::GetAttributeMappingFunction()
213 return &MapAttributesIntoRule
;
216 void HTMLVideoElement::UnbindFromTree(bool aNullParent
) {
217 if (mVisualCloneSource
) {
218 mVisualCloneSource
->EndCloningVisually();
219 } else if (mVisualCloneTarget
) {
220 RefPtr
<AsyncEventDispatcher
> asyncDispatcher
=
221 new AsyncEventDispatcher(this, u
"MozStopPictureInPicture"_ns
,
222 CanBubble::eNo
, ChromeOnlyDispatch::eYes
);
223 asyncDispatcher
->RunDOMEventWhenSafe();
225 EndCloningVisually();
228 HTMLMediaElement::UnbindFromTree(aNullParent
);
231 nsresult
HTMLVideoElement::SetAcceptHeader(nsIHttpChannel
* aChannel
) {
236 "application/ogg;q=0.7,"
237 "audio/*;q=0.6,*/*;q=0.5");
239 return aChannel
->SetRequestHeader("Accept"_ns
, value
, false);
242 bool HTMLVideoElement::IsInteractiveHTMLContent() const {
243 return HasAttr(kNameSpaceID_None
, nsGkAtoms::controls
) ||
244 HTMLMediaElement::IsInteractiveHTMLContent();
247 uint32_t HTMLVideoElement::MozParsedFrames() const {
248 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
249 if (!IsVideoStatsEnabled()) {
253 if (nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
254 return nsRFPService::GetSpoofedTotalFrames(TotalPlayTime());
257 return mDecoder
? mDecoder
->GetFrameStatistics().GetParsedFrames() : 0;
260 uint32_t HTMLVideoElement::MozDecodedFrames() const {
261 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
262 if (!IsVideoStatsEnabled()) {
266 if (nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
267 return nsRFPService::GetSpoofedTotalFrames(TotalPlayTime());
270 return mDecoder
? mDecoder
->GetFrameStatistics().GetDecodedFrames() : 0;
273 uint32_t HTMLVideoElement::MozPresentedFrames() const {
274 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
275 if (!IsVideoStatsEnabled()) {
279 if (nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
280 return nsRFPService::GetSpoofedPresentedFrames(TotalPlayTime(),
281 VideoWidth(), VideoHeight());
284 return mDecoder
? mDecoder
->GetFrameStatistics().GetPresentedFrames() : 0;
287 uint32_t HTMLVideoElement::MozPaintedFrames() {
288 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
289 if (!IsVideoStatsEnabled()) {
293 if (nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
294 return nsRFPService::GetSpoofedPresentedFrames(TotalPlayTime(),
295 VideoWidth(), VideoHeight());
298 layers::ImageContainer
* container
= GetImageContainer();
299 return container
? container
->GetPaintCount() : 0;
302 double HTMLVideoElement::MozFrameDelay() {
303 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
305 if (!IsVideoStatsEnabled() ||
306 nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
310 VideoFrameContainer
* container
= GetVideoFrameContainer();
311 // Hide negative delays. Frame timing tweaks in the compositor (e.g.
312 // adding a bias value to prevent multiple dropped/duped frames when
313 // frame times are aligned with composition times) may produce apparent
314 // negative delay, but we shouldn't report that.
315 return container
? std::max(0.0, container
->GetFrameDelay()) : 0.0;
318 bool HTMLVideoElement::MozHasAudio() const {
319 MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
323 JSObject
* HTMLVideoElement::WrapNode(JSContext
* aCx
,
324 JS::Handle
<JSObject
*> aGivenProto
) {
325 return HTMLVideoElement_Binding::Wrap(aCx
, this, aGivenProto
);
328 FrameStatistics
* HTMLVideoElement::GetFrameStatistics() {
329 return mDecoder
? &(mDecoder
->GetFrameStatistics()) : nullptr;
332 already_AddRefed
<VideoPlaybackQuality
>
333 HTMLVideoElement::GetVideoPlaybackQuality() {
334 DOMHighResTimeStamp creationTime
= 0;
335 uint32_t totalFrames
= 0;
336 uint32_t droppedFrames
= 0;
338 if (IsVideoStatsEnabled()) {
339 if (nsPIDOMWindowInner
* window
= OwnerDoc()->GetInnerWindow()) {
340 Performance
* perf
= window
->GetPerformance();
342 creationTime
= perf
->Now();
347 if (nsContentUtils::ShouldResistFingerprinting(OwnerDoc())) {
348 totalFrames
= nsRFPService::GetSpoofedTotalFrames(TotalPlayTime());
349 droppedFrames
= nsRFPService::GetSpoofedDroppedFrames(
350 TotalPlayTime(), VideoWidth(), VideoHeight());
352 FrameStatistics
* stats
= &mDecoder
->GetFrameStatistics();
353 if (sizeof(totalFrames
) >= sizeof(stats
->GetParsedFrames())) {
354 totalFrames
= stats
->GetTotalFrames();
355 droppedFrames
= stats
->GetDroppedFrames();
357 uint64_t total
= stats
->GetTotalFrames();
358 const auto maxNumber
= std::numeric_limits
<uint32_t>::max();
359 if (total
<= maxNumber
) {
360 totalFrames
= uint32_t(total
);
361 droppedFrames
= uint32_t(stats
->GetDroppedFrames());
363 // Too big number(s) -> Resize everything to fit in 32 bits.
364 double ratio
= double(maxNumber
) / double(total
);
365 totalFrames
= maxNumber
; // === total * ratio
366 droppedFrames
= uint32_t(double(stats
->GetDroppedFrames()) * ratio
);
373 RefPtr
<VideoPlaybackQuality
> playbackQuality
=
374 new VideoPlaybackQuality(this, creationTime
, totalFrames
, droppedFrames
);
375 return playbackQuality
.forget();
378 void HTMLVideoElement::WakeLockRelease() {
379 HTMLMediaElement::WakeLockRelease();
380 ReleaseVideoWakeLockIfExists();
383 void HTMLVideoElement::UpdateWakeLock() {
384 HTMLMediaElement::UpdateWakeLock();
386 CreateVideoWakeLockIfNeeded();
388 ReleaseVideoWakeLockIfExists();
392 bool HTMLVideoElement::ShouldCreateVideoWakeLock() const {
393 // Only request wake lock for video with audio or video from media stream,
394 // because non-stream video without audio is often used as a background image.
396 // Some web conferencing sites route audio outside the video element, and
397 // would not be detected unless we check for media stream, so do that below.
399 // Media streams generally aren't used as background images, though if they
400 // were we'd get false positives. If this is an issue, we could check for
401 // media stream AND document has audio playing (but that was tricky to do).
402 return HasVideo() && (mSrcStream
|| HasAudio());
405 void HTMLVideoElement::CreateVideoWakeLockIfNeeded() {
406 if (!mScreenWakeLock
&& ShouldCreateVideoWakeLock()) {
407 RefPtr
<power::PowerManagerService
> pmService
=
408 power::PowerManagerService::GetInstance();
409 NS_ENSURE_TRUE_VOID(pmService
);
412 mScreenWakeLock
= pmService
->NewWakeLock(u
"video-playing"_ns
,
413 OwnerDoc()->GetInnerWindow(), rv
);
417 void HTMLVideoElement::ReleaseVideoWakeLockIfExists() {
418 if (mScreenWakeLock
) {
420 mScreenWakeLock
->Unlock(rv
);
421 rv
.SuppressException();
422 mScreenWakeLock
= nullptr;
427 bool HTMLVideoElement::SetVisualCloneTarget(
428 RefPtr
<HTMLVideoElement
> aVisualCloneTarget
,
429 RefPtr
<Promise
> aVisualCloneTargetPromise
) {
430 MOZ_DIAGNOSTIC_ASSERT(
431 !aVisualCloneTarget
|| aVisualCloneTarget
->IsInComposedDoc(),
432 "Can't set the clone target to a disconnected video "
434 MOZ_DIAGNOSTIC_ASSERT(!mVisualCloneSource
,
435 "Can't clone a video element that is already a clone.");
436 if (!aVisualCloneTarget
||
437 (aVisualCloneTarget
->IsInComposedDoc() && !mVisualCloneSource
)) {
438 mVisualCloneTarget
= std::move(aVisualCloneTarget
);
439 mVisualCloneTargetPromise
= std::move(aVisualCloneTargetPromise
);
445 bool HTMLVideoElement::SetVisualCloneSource(
446 RefPtr
<HTMLVideoElement
> aVisualCloneSource
) {
447 MOZ_DIAGNOSTIC_ASSERT(
448 !aVisualCloneSource
|| aVisualCloneSource
->IsInComposedDoc(),
449 "Can't set the clone source to a disconnected video "
451 MOZ_DIAGNOSTIC_ASSERT(!mVisualCloneTarget
,
452 "Can't clone a video element that is already a "
454 if (!aVisualCloneSource
||
455 (aVisualCloneSource
->IsInComposedDoc() && !mVisualCloneTarget
)) {
456 mVisualCloneSource
= std::move(aVisualCloneSource
);
463 bool HTMLVideoElement::IsVideoStatsEnabled() {
464 return StaticPrefs::media_video_stats_enabled();
467 double HTMLVideoElement::TotalPlayTime() const {
471 uint32_t timeRangeCount
= mPlayed
->Length();
473 for (uint32_t i
= 0; i
< timeRangeCount
; i
++) {
474 double begin
= mPlayed
->Start(i
);
475 double end
= mPlayed
->End(i
);
476 total
+= end
- begin
;
479 if (mCurrentPlayRangeStart
!= -1.0) {
480 double now
= CurrentTime();
481 if (mCurrentPlayRangeStart
!= now
) {
482 total
+= now
- mCurrentPlayRangeStart
;
490 already_AddRefed
<Promise
> HTMLVideoElement::CloneElementVisually(
491 HTMLVideoElement
& aTargetVideo
, ErrorResult
& aRv
) {
492 MOZ_ASSERT(IsInComposedDoc(),
493 "Can't clone a video that's not bound to a DOM tree.");
494 MOZ_ASSERT(aTargetVideo
.IsInComposedDoc(),
495 "Can't clone to a video that's not bound to a DOM tree.");
496 if (!IsInComposedDoc() || !aTargetVideo
.IsInComposedDoc()) {
497 aRv
.Throw(NS_ERROR_UNEXPECTED
);
501 nsPIDOMWindowInner
* win
= OwnerDoc()->GetInnerWindow();
503 aRv
.Throw(NS_ERROR_UNEXPECTED
);
507 RefPtr
<Promise
> promise
= Promise::Create(win
->AsGlobal(), aRv
);
512 // Do we already have a visual clone target? If so, shut it down.
513 if (mVisualCloneTarget
) {
514 EndCloningVisually();
517 // If there's a poster set on the target video, clear it, otherwise
518 // it'll display over top of the cloned frames.
519 aTargetVideo
.UnsetHTMLAttr(nsGkAtoms::poster
, aRv
);
524 if (!SetVisualCloneTarget(&aTargetVideo
, promise
)) {
525 aRv
.Throw(NS_ERROR_FAILURE
);
529 if (!aTargetVideo
.SetVisualCloneSource(this)) {
530 mVisualCloneTarget
= nullptr;
531 aRv
.Throw(NS_ERROR_FAILURE
);
535 aTargetVideo
.SetMediaInfo(mMediaInfo
);
537 if (IsInComposedDoc() && !StaticPrefs::media_cloneElementVisually_testing()) {
538 NotifyUAWidgetSetupOrChange();
541 MaybeBeginCloningVisually();
543 return promise
.forget();
546 void HTMLVideoElement::StopCloningElementVisually() {
547 if (mVisualCloneTarget
) {
548 EndCloningVisually();
552 void HTMLVideoElement::MaybeBeginCloningVisually() {
553 if (!mVisualCloneTarget
) {
558 VideoFrameContainer
* container
=
559 mVisualCloneTarget
->GetVideoFrameContainer();
561 mDecoder
->SetSecondaryVideoContainer(container
);
563 UpdateMediaControlAfterPictureInPictureModeChanged();
564 } else if (mSrcStream
) {
565 VideoFrameContainer
* container
=
566 mVisualCloneTarget
->GetVideoFrameContainer();
567 if (container
&& mSelectedVideoStreamTrack
) {
568 MOZ_DIAGNOSTIC_ASSERT(!mSecondaryVideoOutput
);
569 mSecondaryVideoOutput
= MakeRefPtr
<SecondaryVideoOutput
>(
570 this, container
, mAbstractMainThread
);
571 mSelectedVideoStreamTrack
->AddVideoOutput(mSecondaryVideoOutput
);
573 UpdateMediaControlAfterPictureInPictureModeChanged();
577 void HTMLVideoElement::EndCloningVisually() {
578 MOZ_ASSERT(mVisualCloneTarget
);
581 mDecoder
->SetSecondaryVideoContainer(nullptr);
582 } else if (mSrcStream
) {
583 if (mSecondaryVideoOutput
&&
584 mVisualCloneTarget
->mSelectedVideoStreamTrack
) {
585 mVisualCloneTarget
->mSelectedVideoStreamTrack
->RemoveVideoOutput(
586 mSecondaryVideoOutput
);
587 mSecondaryVideoOutput
->Forget();
588 mSecondaryVideoOutput
= nullptr;
592 Unused
<< mVisualCloneTarget
->SetVisualCloneSource(nullptr);
593 Unused
<< SetVisualCloneTarget(nullptr);
595 UpdateMediaControlAfterPictureInPictureModeChanged();
597 if (IsInComposedDoc() && !StaticPrefs::media_cloneElementVisually_testing()) {
598 NotifyUAWidgetSetupOrChange();
602 void HTMLVideoElement::OnSecondaryVideoContainerInstalled(
603 const RefPtr
<VideoFrameContainer
>& aSecondaryContainer
) {
604 MOZ_ASSERT(NS_IsMainThread());
605 MOZ_DIAGNOSTIC_ASSERT_IF(mVisualCloneTargetPromise
, mVisualCloneTarget
);
606 if (!mVisualCloneTargetPromise
) {
607 // Clone target was unset.
611 VideoFrameContainer
* container
= mVisualCloneTarget
->GetVideoFrameContainer();
612 if (NS_WARN_IF(container
!= aSecondaryContainer
)) {
613 // Not the right container.
617 mMainThreadEventTarget
->Dispatch(NewRunnableMethod(
618 "Promise::MaybeResolveWithUndefined", mVisualCloneTargetPromise
,
619 &Promise::MaybeResolveWithUndefined
));
620 mVisualCloneTargetPromise
= nullptr;
624 } // namespace mozilla