1 /* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8 ChromeUtils.defineESModuleGetters(lazy, {
9 ContentDOMReference: "resource://gre/modules/ContentDOMReference.sys.mjs",
10 DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs",
11 KEYBOARD_CONTROLS: "resource://gre/modules/PictureInPictureControls.sys.mjs",
12 NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs",
13 Rect: "resource://gre/modules/Geometry.sys.mjs",
14 TOGGLE_POLICIES: "resource://gre/modules/PictureInPictureControls.sys.mjs",
15 TOGGLE_POLICY_STRINGS:
16 "resource://gre/modules/PictureInPictureControls.sys.mjs",
19 import { WebVTT } from "resource://gre/modules/vtt.sys.mjs";
20 import { setTimeout, clearTimeout } from "resource://gre/modules/Timer.sys.mjs";
21 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
23 XPCOMUtils.defineLazyPreferenceGetter(
25 "DISPLAY_TEXT_TRACKS_PREF",
26 "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
29 XPCOMUtils.defineLazyPreferenceGetter(
31 "IMPROVED_CONTROLS_ENABLED_PREF",
32 "media.videocontrols.picture-in-picture.improved-video-controls.enabled",
35 XPCOMUtils.defineLazyPreferenceGetter(
38 "media.videocontrols.picture-in-picture.video-toggle.min-video-secs",
41 XPCOMUtils.defineLazyPreferenceGetter(
43 "PIP_TOGGLE_ALWAYS_SHOW",
44 "media.videocontrols.picture-in-picture.video-toggle.always-show",
47 XPCOMUtils.defineLazyPreferenceGetter(
50 "media.videocontrols.picture-in-picture.urlbar-button.enabled",
54 const PIP_ENABLED_PREF = "media.videocontrols.picture-in-picture.enabled";
55 const TOGGLE_ENABLED_PREF =
56 "media.videocontrols.picture-in-picture.video-toggle.enabled";
57 const TOGGLE_FIRST_SEEN_PREF =
58 "media.videocontrols.picture-in-picture.video-toggle.first-seen-secs";
59 const TOGGLE_FIRST_TIME_DURATION_DAYS = 28;
60 const TOGGLE_HAS_USED_PREF =
61 "media.videocontrols.picture-in-picture.video-toggle.has-used";
62 const TOGGLE_TESTING_PREF =
63 "media.videocontrols.picture-in-picture.video-toggle.testing";
64 const TOGGLE_VISIBILITY_THRESHOLD_PREF =
65 "media.videocontrols.picture-in-picture.video-toggle.visibility-threshold";
66 const TEXT_TRACK_FONT_SIZE =
67 "media.videocontrols.picture-in-picture.display-text-tracks.size";
69 const MOUSEMOVE_PROCESSING_DELAY_MS = 50;
70 const TOGGLE_HIDING_TIMEOUT_MS = 3000;
71 // If you change this, also change VideoControlsWidget.SEEK_TIME_SECS:
72 const SEEK_TIME_SECS = 5;
73 const EMPTIED_TIMEOUT_MS = 1000;
75 // The ToggleChild does not want to capture events from the PiP
76 // windows themselves. This set contains all currently open PiP
77 // players' content windows
78 var gPlayerContents = new WeakSet();
80 // To make it easier to write tests, we have a process-global
81 // WeakSet of all <video> elements that are being tracked for
83 var gWeakIntersectingVideosForTesting = new WeakSet();
85 // Overrides are expected to stay constant for the lifetime of a
86 // content process, so we set this as a lazy process global.
87 // See PictureInPictureToggleChild.getSiteOverrides for a
88 // sense of what the return types are.
89 ChromeUtils.defineLazyGetter(lazy, "gSiteOverrides", () => {
90 return PictureInPictureToggleChild.getSiteOverrides();
93 ChromeUtils.defineLazyGetter(lazy, "logConsole", () => {
94 return console.createInstance({
95 prefix: "PictureInPictureChild",
96 maxLogLevel: Services.prefs.getBoolPref(
97 "media.videocontrols.picture-in-picture.log",
106 * Creates and returns an instance of the PictureInPictureChildVideoWrapper class responsible
107 * for applying site-specific wrapper methods around the original video.
109 * The Picture-In-Picture add-on can use this to provide site-specific wrappers for
110 * sites that require special massaging to control.
111 * @param {Object} pipChild reference to PictureInPictureChild class calling this function
112 * @param {Element} originatingVideo
113 * The <video> element to wrap.
114 * @returns {PictureInPictureChildVideoWrapper} instance of PictureInPictureChildVideoWrapper
116 function applyWrapper(pipChild, originatingVideo) {
117 let originatingDoc = originatingVideo.ownerDocument;
118 let originatingDocumentURI = originatingDoc.documentURI;
120 let overrides = lazy.gSiteOverrides.find(([matcher]) => {
121 return matcher.matches(originatingDocumentURI);
124 // gSiteOverrides is a list of tuples where the first element is the MatchPattern
125 // for a supported site and the second is the actual overrides object for it.
126 let wrapperPath = overrides ? overrides[1].videoWrapperScriptPath : null;
127 return new PictureInPictureChildVideoWrapper(
134 export class PictureInPictureLauncherChild extends JSWindowActorChild {
136 switch (event.type) {
137 case "MozTogglePictureInPicture": {
138 if (event.isTrusted) {
139 this.togglePictureInPicture({
141 reason: event.detail?.reason,
142 eventExtraKeys: event.detail?.eventExtraKeys,
150 receiveMessage(message) {
151 switch (message.name) {
152 case "PictureInPicture:KeyToggle": {
160 * Tells the parent to open a Picture-in-Picture window hosting
161 * a clone of the passed video. If we know about a pre-existing
162 * Picture-in-Picture window existing, this tells the parent to
163 * close it before opening the new one.
165 * @param {Object} pipObject
166 * @param {HTMLVideoElement} pipObject.video
167 * @param {String} pipObject.reason What toggled PiP, e.g. "shortcut"
168 * @param {Object} pipObject.eventExtraKeys Extra telemetry keys to record
171 * @resolves {undefined} Once the new Picture-in-Picture window
172 * has been requested.
174 async togglePictureInPicture(pipObject) {
175 let { video, reason, eventExtraKeys = {} } = pipObject;
176 if (video.isCloningElementVisually) {
177 // The only way we could have entered here for the same video is if
178 // we are toggling via the context menu or via the urlbar button,
179 // since we hide the inline Picture-in-Picture toggle when a video
180 // is being displayed in Picture-in-Picture. Turn off PiP in this case
181 const stopPipEvent = new this.contentWindow.CustomEvent(
182 "MozStopPictureInPicture",
188 video.dispatchEvent(stopPipEvent);
192 if (!PictureInPictureChild.videoWrapper) {
193 PictureInPictureChild.videoWrapper = applyWrapper(
194 PictureInPictureChild,
199 let timestamp = undefined;
200 let scrubberPosition = undefined;
202 if (lazy.IMPROVED_CONTROLS_ENABLED_PREF) {
203 timestamp = PictureInPictureChild.videoWrapper.formatTimestamp(
204 PictureInPictureChild.videoWrapper.getCurrentTime(video),
205 PictureInPictureChild.videoWrapper.getDuration(video)
208 // Scrubber is hidden if undefined, so only set it to something else
209 // if the timestamp is not undefined.
211 timestamp === undefined
213 : PictureInPictureChild.videoWrapper.getCurrentTime(video) /
214 PictureInPictureChild.videoWrapper.getDuration(video);
217 // All other requests to toggle PiP should open a new PiP
219 const videoRef = lazy.ContentDOMReference.get(video);
220 this.sendAsyncMessage("PictureInPicture:Request", {
221 isMuted: PictureInPictureChild.videoIsMuted(video),
222 playing: PictureInPictureChild.videoIsPlaying(video),
223 videoHeight: video.videoHeight,
224 videoWidth: video.videoWidth,
226 ccEnabled: lazy.DISPLAY_TEXT_TRACKS_PREF,
227 webVTTSubtitles: !!video.textTracks?.length,
230 volume: PictureInPictureChild.videoWrapper.getVolume(video),
233 Services.telemetry.recordEvent(
239 firstTimeToggle: (!Services.prefs.getBoolPref(
248 * The keyboard was used to attempt to open Picture-in-Picture. If a video is focused,
249 * select that video. Otherwise find the first playing video, or if none, the largest
250 * dimension video. We suspect this heuristic will handle most cases, though we
251 * might refine this later on. Note that we assume that this method will only be
252 * called for the focused document.
255 let doc = this.document;
257 let video = doc.activeElement;
258 if (!HTMLVideoElement.isInstance(video)) {
259 let listOfVideos = [...doc.querySelectorAll("video")].filter(
260 video => !isNaN(video.duration)
262 // Get the first non-paused video, otherwise the longest video. This
263 // fallback is designed to skip over "preview"-style videos on sidebars.
265 listOfVideos.filter(v => !v.paused)[0] ||
266 listOfVideos.sort((a, b) => b.duration - a.duration)[0];
269 this.togglePictureInPicture({ video, reason: "shortcut" });
276 * The PictureInPictureToggleChild is responsible for displaying the overlaid
277 * Picture-in-Picture toggle over top of <video> elements that the mouse is
280 export class PictureInPictureToggleChild extends JSWindowActorChild {
283 // We need to maintain some state about various things related to the
284 // Picture-in-Picture toggles - however, for now, the same
285 // PictureInPictureToggleChild might be re-used for different documents.
286 // We keep the state stashed inside of this WeakMap, keyed on the document
288 this.weakDocStates = new WeakMap();
290 Services.prefs.getBoolPref(TOGGLE_ENABLED_PREF) &&
291 Services.prefs.getBoolPref(PIP_ENABLED_PREF);
292 this.toggleTesting = Services.prefs.getBoolPref(TOGGLE_TESTING_PREF, false);
294 // Bug 1570744 - JSWindowActorChild's cannot be used as nsIObserver's
295 // directly, so we create a new function here instead to act as our
296 // nsIObserver, which forwards the notification to the observe method.
297 this.observerFunction = (subject, topic, data) => {
298 this.observe(subject, topic, data);
300 Services.prefs.addObserver(TOGGLE_ENABLED_PREF, this.observerFunction);
301 Services.prefs.addObserver(PIP_ENABLED_PREF, this.observerFunction);
302 Services.prefs.addObserver(TOGGLE_FIRST_SEEN_PREF, this.observerFunction);
303 Services.cpmm.sharedData.addEventListener("change", this);
305 this.eligiblePipVideos = new WeakSet();
306 this.trackingVideos = new WeakSet();
309 receiveMessage(message) {
310 switch (message.name) {
311 case "PictureInPicture:UrlbarToggle": {
312 this.urlbarToggle(message.data);
320 this.stopTrackingMouseOverVideos();
321 Services.prefs.removeObserver(TOGGLE_ENABLED_PREF, this.observerFunction);
322 Services.prefs.removeObserver(PIP_ENABLED_PREF, this.observerFunction);
323 Services.prefs.removeObserver(
324 TOGGLE_FIRST_SEEN_PREF,
325 this.observerFunction
327 Services.cpmm.sharedData.removeEventListener("change", this);
329 // remove the observer on the <video> element
330 let state = this.docState;
331 if (state?.intersectionObserver) {
332 state.intersectionObserver.disconnect();
335 // ensure the sandbox created by the video is destroyed
336 this.videoWrapper?.destroy();
337 this.videoWrapper = null;
339 for (let video of ChromeUtils.nondeterministicGetWeakSetKeys(
340 this.eligiblePipVideos
342 video.removeEventListener("emptied", this);
343 video.removeEventListener("loadedmetadata", this);
344 video.removeEventListener("durationchange", this);
347 for (let video of ChromeUtils.nondeterministicGetWeakSetKeys(
350 video.removeEventListener("emptied", this);
351 video.removeEventListener("loadedmetadata", this);
352 video.removeEventListener("durationchange", this);
355 // ensure we don't access the state
356 this.isDestroyed = true;
359 observe(subject, topic, data) {
360 if (topic != "nsPref:changed") {
365 Services.prefs.getBoolPref(TOGGLE_ENABLED_PREF) &&
366 Services.prefs.getBoolPref(PIP_ENABLED_PREF);
368 if (this.toggleEnabled) {
369 // We have enabled the Picture-in-Picture toggle, so we need to make
370 // sure we register all of the videos that might already be on the page.
371 this.contentWindow.requestIdleCallback(() => {
372 let videos = this.document.querySelectorAll("video");
373 for (let video of videos) {
374 this.registerVideo(video);
380 case TOGGLE_FIRST_SEEN_PREF:
381 const firstSeenSeconds = Services.prefs.getIntPref(
382 TOGGLE_FIRST_SEEN_PREF
384 if (!firstSeenSeconds || firstSeenSeconds < 0) {
387 this.changeToIconIfDurationEnd(firstSeenSeconds);
393 * Returns the state for the current document referred to via
394 * this.document. If no such state exists, creates it, stores it
398 if (this.isDestroyed || !this.document) {
402 let state = this.weakDocStates.get(this.document);
404 let visibilityThresholdPref = Services.prefs.getFloatPref(
405 TOGGLE_VISIBILITY_THRESHOLD_PREF,
411 // A reference to the IntersectionObserver that's monitoring for videos
412 // to become visible.
413 intersectionObserver: null,
414 // A WeakSet of videos that are supposedly visible, according to the
415 // IntersectionObserver.
416 weakVisibleVideos: new WeakSet(),
417 // The number of videos that are supposedly visible, according to the
418 // IntersectionObserver
419 visibleVideosCount: 0,
420 // The DeferredTask that we'll arm every time a mousemove event occurs
421 // on a page where we have one or more visible videos.
422 mousemoveDeferredTask: null,
423 // A weak reference to the last video we displayed the toggle over.
425 // True if the user is in the midst of clicking the toggle.
426 isClickingToggle: false,
427 // Set to the original target element on pointerdown if the user is clicking
428 // the toggle - this way, we can determine if a "click" event will need to be
429 // suppressed ("click" events don't fire if a "mouseup" occurs on a different
430 // element from the "pointerdown" / "mousedown" event).
431 clickedElement: null,
432 // This is a DeferredTask to hide the toggle after a period of mouse
434 hideToggleDeferredTask: null,
435 // If we reach a point where we're tracking videos for mouse movements,
436 // then this will be true. If there are no videos worth tracking, then
438 isTrackingVideos: false,
439 togglePolicy: lazy.TOGGLE_POLICIES.DEFAULT,
440 toggleVisibilityThreshold: visibilityThresholdPref,
441 // The documentURI that has been checked with toggle policies and
442 // visibility thresholds for this document. Note that the documentURI
443 // might change for a document via the history API, so we remember
444 // the last checked documentURI to determine if we need to check again.
445 checkedPolicyDocumentURI: null,
447 this.weakDocStates.set(this.document, state);
454 * Returns the video that the user was last hovering with the mouse if it
457 * @return {Element} the <video> element that the user was last hovering,
458 * or null if there was no such <video>, or the <video> no longer exists.
461 let { weakOverVideo } = this.docState;
463 // Bug 800957 - Accessing weakrefs at the wrong time can cause us to
464 // throw NS_ERROR_XPC_BAD_CONVERT_NATIVE
466 return weakOverVideo.get();
475 if (!event.isTrusted) {
476 // We don't care about synthesized events that might be coming from
481 // Don't capture events from Picture-in-Picture content windows
482 if (gPlayerContents.has(this.contentWindow)) {
486 switch (event.type) {
488 // Even if this is a touch event, there may be subsequent click events.
489 // Suppress those events after selecting the toggle to prevent playback changes
490 // when opening the Picture-in-Picture window.
491 if (this.docState.isClickingToggle) {
492 event.stopImmediatePropagation();
493 event.preventDefault();
498 const { changedKeys } = event;
499 if (changedKeys.includes("PictureInPicture:SiteOverrides")) {
500 // For now we only update our cache if the site overrides change.
501 // the user will need to refresh the page for changes to apply.
503 lazy.gSiteOverrides =
504 PictureInPictureToggleChild.getSiteOverrides();
506 // Ignore resulting TypeError if gSiteOverrides is still unloaded
507 if (!(e instanceof TypeError)) {
514 case "UAWidgetSetupOrChange": {
516 this.toggleEnabled &&
517 this.contentWindow.HTMLVideoElement.isInstance(event.target) &&
518 event.target.ownerDocument == this.document
520 this.registerVideo(event.target);
524 case "contextmenu": {
525 if (this.toggleEnabled) {
526 this.checkContextMenu(event);
531 this.onMouseOut(event);
535 if (event.detail == 0) {
536 let shadowRoot = event.originalTarget.containingShadowRoot;
537 let toggle = this.getToggleElement(shadowRoot);
538 if (event.originalTarget == toggle) {
539 this.startPictureInPicture(event, shadowRoot.host, toggle);
547 this.onMouseButtonEvent(event);
550 case "pointerdown": {
551 this.onPointerDown(event);
555 this.onMouseMove(event);
559 this.onPageShow(event);
563 this.onPageHide(event);
566 case "durationchange":
567 // Intentional fall-through
569 // Intentional fall-through
570 case "loadedmetadata": {
571 this.updatePipVideoEligibility(event.target);
578 * Adds a <video> to the IntersectionObserver so that we know when it becomes
581 * @param {Element} video The <video> element to register.
583 registerVideo(video) {
584 let state = this.docState;
585 if (!state.intersectionObserver) {
586 let fn = this.onIntersection.bind(this);
587 state.intersectionObserver = new this.contentWindow.IntersectionObserver(
590 threshold: [0.0, 0.5],
595 state.intersectionObserver.observe(video);
597 if (!lazy.PIP_URLBAR_BUTTON) {
601 video.addEventListener("emptied", this);
602 video.addEventListener("loadedmetadata", this);
603 video.addEventListener("durationchange", this);
605 this.trackingVideos.add(video);
607 this.updatePipVideoEligibility(video);
610 updatePipVideoEligibility(video) {
611 let isEligible = this.isVideoPiPEligible(video);
613 if (!this.eligiblePipVideos.has(video)) {
614 this.eligiblePipVideos.add(video);
616 let mutationObserver = new this.contentWindow.MutationObserver(
618 this.handleEligiblePipVideoMutation(mutationList);
621 mutationObserver.observe(video.parentElement, { childList: true });
623 } else if (this.eligiblePipVideos.has(video)) {
624 this.eligiblePipVideos.delete(video);
627 let videos = ChromeUtils.nondeterministicGetWeakSetKeys(
628 this.eligiblePipVideos
631 this.sendAsyncMessage("PictureInPicture:UpdateEligiblePipVideoCount", {
632 pipCount: videos.length,
633 pipDisabledCount: videos.reduce(
634 (accumulator, currentVal) =>
635 accumulator + (currentVal.disablePictureInPicture ? 1 : 0),
641 handleEligiblePipVideoMutation(mutationList) {
642 for (let mutationRecord of mutationList) {
643 let video = mutationRecord.removedNodes[0];
644 this.eligiblePipVideos.delete(video);
647 let videos = ChromeUtils.nondeterministicGetWeakSetKeys(
648 this.eligiblePipVideos
651 this.sendAsyncMessage("PictureInPicture:UpdateEligiblePipVideoCount", {
652 pipCount: videos.length,
653 pipDisabledCount: videos.reduce(
654 (accumulator, currentVal) =>
655 accumulator + (currentVal.disablePictureInPicture ? 1 : 0),
661 urlbarToggle(eventExtraKeys) {
662 let video = ChromeUtils.nondeterministicGetWeakSetKeys(
663 this.eligiblePipVideos
666 let pipEvent = new this.contentWindow.CustomEvent(
667 "MozTogglePictureInPicture",
670 detail: { reason: "urlBar", eventExtraKeys },
673 video.dispatchEvent(pipEvent);
677 isVideoPiPEligible(video) {
678 if (lazy.PIP_TOGGLE_ALWAYS_SHOW) {
682 if (isNaN(video.duration) || video.duration < lazy.MIN_VIDEO_LENGTH) {
686 const MIN_VIDEO_DIMENSION = 140; // pixels
688 video.clientWidth < MIN_VIDEO_DIMENSION ||
689 video.clientHeight < MIN_VIDEO_DIMENSION
698 * Changes from the first-time toggle to the icon toggle if the Nimbus variable `displayDuration`'s
699 * end date is reached when hovering over a video. The end date is calculated according to the timestamp
700 * indicating when the PiP toggle was first seen.
701 * @param {Number} firstSeenStartSeconds the timestamp in seconds indicating when the PiP toggle was first seen
703 changeToIconIfDurationEnd(firstSeenStartSeconds) {
704 const { displayDuration } =
705 lazy.NimbusFeatures.pictureinpicture.getAllVariables({
707 displayDuration: TOGGLE_FIRST_TIME_DURATION_DAYS,
710 if (!displayDuration || displayDuration < 0) {
714 let daysInSeconds = displayDuration * 24 * 60 * 60;
715 let firstSeenEndSeconds = daysInSeconds + firstSeenStartSeconds;
716 let currentDateSeconds = Math.round(Date.now() / 1000);
718 lazy.logConsole.debug(
719 "Toggle duration experiment - first time toggle seen on:",
720 new Date(firstSeenStartSeconds * 1000).toLocaleDateString()
722 lazy.logConsole.debug(
723 "Toggle duration experiment - first time toggle will change on:",
724 new Date(firstSeenEndSeconds * 1000).toLocaleDateString()
726 lazy.logConsole.debug(
727 "Toggle duration experiment - current date:",
728 new Date(currentDateSeconds * 1000).toLocaleDateString()
731 if (currentDateSeconds >= firstSeenEndSeconds) {
732 this.sendAsyncMessage("PictureInPicture:SetHasUsed", {
739 * Called by the IntersectionObserver callback once a video becomes visible.
740 * This adds some fine-grained checking to ensure that a sufficient amount of
741 * the video is visible before we consider showing the toggles on it. For now,
742 * that means that the entirety of the video must be in the viewport.
744 * @param {IntersectionEntry} intersectionEntry An IntersectionEntry passed to
745 * the IntersectionObserver callback.
746 * @return bool Whether or not we should start tracking mousemove events for
747 * this registered video.
749 worthTracking(intersectionEntry) {
750 return intersectionEntry.isIntersecting;
754 * Called by the IntersectionObserver once a video crosses one of the
755 * thresholds dictated by the IntersectionObserver configuration.
757 * @param {Array<IntersectionEntry>} A collection of one or more
758 * IntersectionEntry's for <video> elements that might have entered or exited
761 onIntersection(entries) {
762 // The IntersectionObserver will also fire when a previously intersecting
763 // element is removed from the DOM. We know, however, that the node is
764 // still alive and referrable from the WeakSet because the
765 // IntersectionObserverEntry holds a strong reference to the video.
766 let state = this.docState;
770 let oldVisibleVideosCount = state.visibleVideosCount;
771 for (let entry of entries) {
772 let video = entry.target;
773 if (this.worthTracking(entry)) {
774 if (!state.weakVisibleVideos.has(video)) {
775 state.weakVisibleVideos.add(video);
776 state.visibleVideosCount++;
777 if (this.toggleTesting) {
778 gWeakIntersectingVideosForTesting.add(video);
781 } else if (state.weakVisibleVideos.has(video)) {
782 state.weakVisibleVideos.delete(video);
783 state.visibleVideosCount--;
784 if (this.toggleTesting) {
785 gWeakIntersectingVideosForTesting.delete(video);
790 // For testing, especially in debug or asan builds, we might not
791 // run this idle callback within an acceptable time. While we're
792 // testing, we'll bypass the idle callback performance optimization
793 // and run our callbacks as soon as possible during the next idle
795 if (!oldVisibleVideosCount && state.visibleVideosCount) {
796 if (this.toggleTesting || !this.contentWindow) {
797 this.beginTrackingMouseOverVideos();
799 this.contentWindow.requestIdleCallback(() => {
800 this.beginTrackingMouseOverVideos();
803 } else if (oldVisibleVideosCount && !state.visibleVideosCount) {
804 if (this.toggleTesting || !this.contentWindow) {
805 this.stopTrackingMouseOverVideos();
807 this.contentWindow.requestIdleCallback(() => {
808 this.stopTrackingMouseOverVideos();
814 addMouseButtonListeners() {
815 // We want to try to cancel the mouse events from continuing
816 // on into content if the user has clicked on the toggle, so
817 // we don't use the mozSystemGroup here, and add the listener
818 // to the parent target of the window, which in this case,
819 // is the windowRoot. Since this event listener is attached to
820 // part of the outer window, we need to also remove it in a
821 // pagehide event listener in the event that the page unloads
822 // before stopTrackingMouseOverVideos fires.
823 this.contentWindow.windowRoot.addEventListener("pointerdown", this, {
826 this.contentWindow.windowRoot.addEventListener("mousedown", this, {
829 this.contentWindow.windowRoot.addEventListener("mouseup", this, {
832 this.contentWindow.windowRoot.addEventListener("pointerup", this, {
835 this.contentWindow.windowRoot.addEventListener("click", this, {
838 this.contentWindow.windowRoot.addEventListener("mouseout", this, {
841 this.contentWindow.windowRoot.addEventListener("touchstart", this, {
846 removeMouseButtonListeners() {
847 // This can be null when closing the tab, but the event
848 // listeners should be removed in that case already.
849 if (!this.contentWindow || !this.contentWindow.windowRoot) {
853 this.contentWindow.windowRoot.removeEventListener("pointerdown", this, {
856 this.contentWindow.windowRoot.removeEventListener("mousedown", this, {
859 this.contentWindow.windowRoot.removeEventListener("mouseup", this, {
862 this.contentWindow.windowRoot.removeEventListener("pointerup", this, {
865 this.contentWindow.windowRoot.removeEventListener("click", this, {
868 this.contentWindow.windowRoot.removeEventListener("mouseout", this, {
871 this.contentWindow.windowRoot.removeEventListener("touchstart", this, {
877 * One of the challenges of displaying this toggle is that many sites put
878 * things over top of <video> elements, like custom controls, or images, or
879 * all manner of things that might intercept mouseevents that would normally
880 * fire directly on the <video>. In order to properly detect when the mouse
881 * is over top of one of the <video> elements in this situation, we currently
882 * add a mousemove event handler to the entire document, and stash the most
883 * recent mousemove that fires. At periodic intervals, that stashed mousemove
884 * event is checked to see if it's hovering over one of our registered
887 * This sort of thing will not be necessary once bug 1539652 is fixed.
889 beginTrackingMouseOverVideos() {
890 let state = this.docState;
891 if (!state.mousemoveDeferredTask) {
892 state.mousemoveDeferredTask = new lazy.DeferredTask(() => {
893 this.checkLastMouseMove();
894 }, MOUSEMOVE_PROCESSING_DELAY_MS);
896 this.document.addEventListener("mousemove", this, {
897 mozSystemGroup: true,
900 this.contentWindow.addEventListener("pageshow", this, {
901 mozSystemGroup: true,
903 this.contentWindow.addEventListener("pagehide", this, {
904 mozSystemGroup: true,
906 this.addMouseButtonListeners();
907 state.isTrackingVideos = true;
911 * If we no longer have any interesting videos in the viewport, we deregister
912 * the mousemove and click listeners, and also remove any toggles that might
913 * be on the page still.
915 stopTrackingMouseOverVideos() {
916 let state = this.docState;
917 // We initialize `mousemoveDeferredTask` in `beginTrackingMouseOverVideos`.
918 // If it doesn't exist, that can't have happened. Nothing else ever sets
919 // this value (though we arm/disarm in various places). So we don't need
920 // to do anything else here and can return early.
921 if (!state.mousemoveDeferredTask) {
924 state.mousemoveDeferredTask.disarm();
925 this.document.removeEventListener("mousemove", this, {
926 mozSystemGroup: true,
929 if (this.contentWindow) {
930 this.contentWindow.removeEventListener("pageshow", this, {
931 mozSystemGroup: true,
933 this.contentWindow.removeEventListener("pagehide", this, {
934 mozSystemGroup: true,
937 this.removeMouseButtonListeners();
938 let oldOverVideo = this.getWeakOverVideo();
940 this.onMouseLeaveVideo(oldOverVideo);
942 state.isTrackingVideos = false;
946 * This pageshow event handler will get called if and when we complete a tab
947 * tear out or in. If we happened to be tracking videos before the tear
948 * occurred, we re-add the mouse event listeners so that they're attached to
949 * the right WindowRoot.
951 * @param {Event} event The pageshow event fired when completing a tab tear
955 let state = this.docState;
956 if (state.isTrackingVideos) {
957 this.addMouseButtonListeners();
962 * This pagehide event handler will get called if and when we start a tab
963 * tear out or in. If we happened to be tracking videos before the tear
964 * occurred, we remove the mouse event listeners. We'll re-add them when the
965 * pageshow event fires.
967 * @param {Event} event The pagehide event fired when starting a tab tear
971 let state = this.docState;
972 if (state.isTrackingVideos) {
973 this.removeMouseButtonListeners();
978 * If we're tracking <video> elements, this pointerdown event handler is run anytime
979 * a pointerdown occurs on the document. This function is responsible for checking
980 * if the user clicked on the Picture-in-Picture toggle. It does this by first
981 * checking if the video is visible beneath the point that was clicked. Then
982 * it tests whether or not the pointerdown occurred within the rectangle of the
983 * toggle. If so, the event's propagation is stopped, and Picture-in-Picture is
986 * @param {Event} event The mousemove event.
988 onPointerDown(event) {
989 // The toggle ignores non-primary mouse clicks.
990 if (event.button != 0) {
994 let video = this.getWeakOverVideo();
999 let shadowRoot = video.openOrClosedShadowRoot;
1004 let state = this.docState;
1006 let overVideo = (() => {
1007 let { clientX, clientY } = event;
1008 let winUtils = this.contentWindow.windowUtils;
1009 // We use winUtils.nodesFromRect instead of document.elementsFromPoint,
1010 // since document.elementsFromPoint always flushes layout. The 1's in that
1011 // function call are for the size of the rect that we want, which is 1x1.
1013 // We pass the aOnlyVisible boolean argument to check that the video isn't
1014 // occluded by anything visible at the point of mousedown. If it is, we'll
1015 // ignore the mousedown.
1016 let elements = winUtils.nodesFromRect(
1025 /* aOnlyVisible = */ true,
1026 state.toggleVisibilityThreshold
1029 for (let element of elements) {
1030 if (element == video || element.containingShadowRoot == shadowRoot) {
1042 let toggle = this.getToggleElement(shadowRoot);
1043 if (this.isMouseOverToggle(toggle, event)) {
1044 state.isClickingToggle = true;
1045 state.clickedElement = Cu.getWeakReference(event.originalTarget);
1046 event.stopImmediatePropagation();
1048 this.startPictureInPicture(event, video, toggle);
1052 startPictureInPicture(event, video, toggle) {
1053 Services.telemetry.keyedScalarAdd(
1054 "pictureinpicture.opened_method",
1059 let pipEvent = new this.contentWindow.CustomEvent(
1060 "MozTogglePictureInPicture",
1063 detail: { reason: "toggle" },
1066 video.dispatchEvent(pipEvent);
1068 // Since we've initiated Picture-in-Picture, we can go ahead and
1069 // hide the toggle now.
1070 this.onMouseLeaveVideo(video);
1074 * Called for mousedown, pointerup, mouseup and click events. If we
1075 * detected that the user is clicking on the Picture-in-Picture toggle,
1076 * these events are cancelled in the capture-phase before they reach
1077 * content. The state for suppressing these events is cleared on the
1078 * click event (unless the mouseup occurs on a different element from
1079 * the mousedown, in which case, the state is cleared on mouseup).
1081 * @param {Event} event A mousedown, pointerup, mouseup or click event.
1083 onMouseButtonEvent(event) {
1084 // The toggle ignores non-primary mouse clicks.
1085 if (event.button != 0) {
1089 let state = this.docState;
1090 if (state.isClickingToggle) {
1091 event.stopImmediatePropagation();
1093 // If this is a mouseup event, check to see if we have a record of what
1094 // the original target was on pointerdown. If so, and if it doesn't match
1095 // the mouseup original target, that means we won't get a click event, and
1096 // we can clear the "clicking the toggle" state right away.
1098 // Otherwise, we wait for the click event to do that.
1099 let isMouseUpOnOtherElement =
1100 event.type == "mouseup" &&
1101 (!state.clickedElement ||
1102 state.clickedElement.get() != event.originalTarget);
1105 isMouseUpOnOtherElement ||
1106 event.type == "click" ||
1107 // pointerup event still triggers after a touchstart event. We just need to detect
1108 // the pointer type and determine if we got to this part of the code through a touch event.
1109 event.pointerType == "touch"
1111 // The click is complete, so now we reset the state so that
1112 // we stop suppressing these events.
1113 state.isClickingToggle = false;
1114 state.clickedElement = null;
1120 * Called on mouseout events to determine whether or not the mouse has
1121 * exited the window.
1123 * @param {Event} event The mouseout event.
1126 if (!event.relatedTarget) {
1127 // For mouseout events, if there's no relatedTarget (which normally
1128 // maps to the element that the mouse entered into) then this means that
1129 // we left the window.
1130 let video = this.getWeakOverVideo();
1135 this.onMouseLeaveVideo(video);
1140 * Called for each mousemove event when we're tracking those events to
1141 * determine if the cursor is hovering over a <video>.
1143 * @param {Event} event The mousemove event.
1145 onMouseMove(event) {
1146 let state = this.docState;
1148 if (state.hideToggleDeferredTask) {
1149 state.hideToggleDeferredTask.disarm();
1150 state.hideToggleDeferredTask.arm();
1153 state.lastMouseMoveEvent = event;
1154 state.mousemoveDeferredTask.arm();
1158 * Called by the DeferredTask after MOUSEMOVE_PROCESSING_DELAY_MS
1159 * milliseconds. Checked to see if that mousemove happens to be overtop of
1160 * any interesting <video> elements that we want to display the toggle
1161 * on. If so, puts the toggle on that video.
1163 checkLastMouseMove() {
1164 let state = this.docState;
1165 let event = state.lastMouseMoveEvent;
1166 let { clientX, clientY } = event;
1167 lazy.logConsole.debug("Visible videos count:", state.visibleVideosCount);
1168 lazy.logConsole.debug("Tracking videos:", state.isTrackingVideos);
1169 let winUtils = this.contentWindow.windowUtils;
1170 // We use winUtils.nodesFromRect instead of document.elementsFromPoint,
1171 // since document.elementsFromPoint always flushes layout. The 1's in that
1172 // function call are for the size of the rect that we want, which is 1x1.
1173 let elements = winUtils.nodesFromRect(
1182 /* aOnlyVisible = */ true
1185 for (let element of elements) {
1186 lazy.logConsole.debug("Element id under cursor:", element.id);
1187 lazy.logConsole.debug(
1188 "Node name of an element under cursor:",
1191 lazy.logConsole.debug(
1192 "Supported <video> element:",
1193 state.weakVisibleVideos.has(element)
1195 lazy.logConsole.debug(
1196 "PiP window is open:",
1197 element.isCloningElementVisually
1200 // Check for hovering over the video controls or so too, not only
1201 // directly over the video.
1202 for (let el = element; el; el = el.containingShadowRoot?.host) {
1203 if (state.weakVisibleVideos.has(el) && !el.isCloningElementVisually) {
1204 lazy.logConsole.debug("Found supported element");
1205 this.onMouseOverVideo(el, event);
1211 let oldOverVideo = this.getWeakOverVideo();
1213 this.onMouseLeaveVideo(oldOverVideo);
1218 * Called once it has been determined that the mouse is overtop of a video
1219 * that is in the viewport.
1221 * @param {Element} video The video the mouse is over.
1223 onMouseOverVideo(video, event) {
1224 let oldOverVideo = this.getWeakOverVideo();
1225 let shadowRoot = video.openOrClosedShadowRoot;
1227 if (shadowRoot.firstChild && video != oldOverVideo) {
1228 if (video.getTransformToViewport().a == -1) {
1229 shadowRoot.firstChild.setAttribute("flipped", true);
1231 shadowRoot.firstChild.removeAttribute("flipped");
1235 // It seems from automated testing that if it's still very early on in the
1236 // lifecycle of a <video> element, it might not yet have a shadowRoot,
1237 // in which case, we can bail out here early.
1240 // We also clear the hover state on the old video we were hovering,
1241 // if there was one.
1242 this.onMouseLeaveVideo(oldOverVideo);
1248 let state = this.docState;
1249 let toggle = this.getToggleElement(shadowRoot);
1250 let controlsOverlay = shadowRoot.querySelector(".controlsOverlay");
1252 if (state.checkedPolicyDocumentURI != this.document.documentURI) {
1253 state.togglePolicy = lazy.TOGGLE_POLICIES.DEFAULT;
1254 // We cache the matchers process-wide. We'll skip this while running tests to make that
1256 let siteOverrides = this.toggleTesting
1257 ? PictureInPictureToggleChild.getSiteOverrides()
1258 : lazy.gSiteOverrides;
1260 let visibilityThresholdPref = Services.prefs.getFloatPref(
1261 TOGGLE_VISIBILITY_THRESHOLD_PREF,
1265 if (!this.videoWrapper) {
1266 this.videoWrapper = applyWrapper(this, video);
1269 // Do we have any toggle overrides? If so, try to apply them.
1270 for (let [override, { policy, visibilityThreshold }] of siteOverrides) {
1272 (policy || visibilityThreshold) &&
1273 override.matches(this.document.documentURI)
1275 state.togglePolicy = this.videoWrapper?.shouldHideToggle(video)
1276 ? lazy.TOGGLE_POLICIES.HIDDEN
1277 : policy || lazy.TOGGLE_POLICIES.DEFAULT;
1278 state.toggleVisibilityThreshold =
1279 visibilityThreshold || visibilityThresholdPref;
1284 state.checkedPolicyDocumentURI = this.document.documentURI;
1287 // The built-in <video> controls are along the bottom, which would overlap the
1288 // toggle if the override is set to BOTTOM, so we ignore overrides that set
1289 // a policy of BOTTOM for <video> elements with controls.
1291 state.togglePolicy != lazy.TOGGLE_POLICIES.DEFAULT &&
1292 !(state.togglePolicy == lazy.TOGGLE_POLICIES.BOTTOM && video.controls)
1294 toggle.setAttribute(
1296 lazy.TOGGLE_POLICY_STRINGS[state.togglePolicy]
1299 toggle.removeAttribute("policy");
1302 // nimbusExperimentVariables will be defaultValues when the experiment is disabled
1303 const nimbusExperimentVariables =
1304 lazy.NimbusFeatures.pictureinpicture.getAllVariables({
1309 showIconOnly: false,
1310 displayDuration: TOGGLE_FIRST_TIME_DURATION_DAYS,
1315 * If a Nimbus variable exists for the first-time PiP toggle design,
1316 * override the old design via a classname "experiment".
1318 if (!nimbusExperimentVariables.oldToggle) {
1319 let controlsContainer = shadowRoot.querySelector(".controlsContainer");
1320 let pipWrapper = shadowRoot.querySelector(".pip-wrapper");
1322 controlsContainer.classList.add("experiment");
1323 pipWrapper.classList.add("experiment");
1325 let controlsContainer = shadowRoot.querySelector(".controlsContainer");
1326 let pipWrapper = shadowRoot.querySelector(".pip-wrapper");
1328 controlsContainer.classList.remove("experiment");
1329 pipWrapper.classList.remove("experiment");
1332 if (nimbusExperimentVariables.title) {
1333 let pipExplainer = shadowRoot.querySelector(".pip-explainer");
1334 let pipLabel = shadowRoot.querySelector(".pip-label");
1336 if (pipExplainer && nimbusExperimentVariables.message) {
1337 pipExplainer.innerText = nimbusExperimentVariables.message;
1339 pipLabel.innerText = nimbusExperimentVariables.title;
1340 } else if (nimbusExperimentVariables.showIconOnly) {
1341 // We only want to show the PiP icon in this experiment scenario
1342 let pipExpanded = shadowRoot.querySelector(".pip-expanded");
1343 pipExpanded.style.display = "none";
1344 let pipSmall = shadowRoot.querySelector(".pip-small");
1345 pipSmall.style.opacity = "1";
1347 let pipIcon = shadowRoot.querySelectorAll(".pip-icon")[1];
1348 pipIcon.style.display = "block";
1351 controlsOverlay.removeAttribute("hidetoggle");
1353 // The hideToggleDeferredTask we create here is for automatically hiding
1354 // the toggle after a period of no mousemove activity for
1355 // TOGGLE_HIDING_TIMEOUT_MS. If the mouse moves, then the DeferredTask
1358 // We disable the toggle hiding timeout during testing to reduce
1359 // non-determinism from timers when testing the toggle.
1360 if (!state.hideToggleDeferredTask && !this.toggleTesting) {
1361 state.hideToggleDeferredTask = new lazy.DeferredTask(() => {
1362 controlsOverlay.setAttribute("hidetoggle", true);
1363 }, TOGGLE_HIDING_TIMEOUT_MS);
1367 if (oldOverVideo == video) {
1368 // If we're still hovering the old video, we might have entered or
1369 // exited the toggle region.
1370 this.checkHoverToggle(toggle, event);
1374 // We had an old video that we were hovering, and we're not hovering
1375 // it anymore. Let's leave it.
1376 this.onMouseLeaveVideo(oldOverVideo);
1379 state.weakOverVideo = Cu.getWeakReference(video);
1380 controlsOverlay.classList.add("hovering");
1383 state.togglePolicy != lazy.TOGGLE_POLICIES.HIDDEN &&
1384 !toggle.hasAttribute("hidden")
1386 Services.telemetry.scalarAdd("pictureinpicture.saw_toggle", 1);
1387 const hasUsedPiP = Services.prefs.getBoolPref(TOGGLE_HAS_USED_PREF);
1389 firstTime: (!hasUsedPiP).toString(),
1391 Services.telemetry.recordEvent(
1398 // only record if this is the first time seeing the toggle
1400 lazy.NimbusFeatures.pictureinpicture.recordExposureEvent();
1402 const firstSeenSeconds = Services.prefs.getIntPref(
1403 TOGGLE_FIRST_SEEN_PREF,
1407 if (!firstSeenSeconds || firstSeenSeconds < 0) {
1408 let firstTimePiPStartDate = Math.round(Date.now() / 1000);
1409 this.sendAsyncMessage("PictureInPicture:SetFirstSeen", {
1410 dateSeconds: firstTimePiPStartDate,
1412 } else if (nimbusExperimentVariables.displayDuration) {
1413 this.changeToIconIfDurationEnd(firstSeenSeconds);
1418 // Now that we're hovering the video, we'll check to see if we're
1419 // hovering the toggle too.
1420 this.checkHoverToggle(toggle, event);
1424 * Checks if a mouse event is happening over a toggle element. If it is,
1425 * sets the hovering class on it. Otherwise, it clears the hovering
1428 * @param {Element} toggle The Picture-in-Picture toggle to check.
1429 * @param {MouseEvent} event A MouseEvent to test.
1431 checkHoverToggle(toggle, event) {
1432 toggle.classList.toggle("hovering", this.isMouseOverToggle(toggle, event));
1436 * Called once it has been determined that the mouse is no longer overlapping
1437 * a video that we'd previously called onMouseOverVideo with.
1439 * @param {Element} video The video that the mouse left.
1441 onMouseLeaveVideo(video) {
1442 let state = this.docState;
1443 let shadowRoot = video.openOrClosedShadowRoot;
1446 let controlsOverlay = shadowRoot.querySelector(".controlsOverlay");
1447 let toggle = this.getToggleElement(shadowRoot);
1448 controlsOverlay.classList.remove("hovering");
1449 toggle.classList.remove("hovering");
1452 state.weakOverVideo = null;
1454 if (!this.toggleTesting) {
1455 state.hideToggleDeferredTask.disarm();
1456 state.mousemoveDeferredTask.disarm();
1459 state.hideToggleDeferredTask = null;
1463 * Given a reference to a Picture-in-Picture toggle element, determines
1464 * if a MouseEvent event is occurring within its bounds.
1466 * @param {Element} toggle The Picture-in-Picture toggle.
1467 * @param {MouseEvent} event A MouseEvent to test.
1471 isMouseOverToggle(toggle, event) {
1473 toggle.ownerGlobal.windowUtils.getBoundsWithoutFlushing(toggle);
1475 // The way the toggle is currently implemented with
1476 // absolute positioning, the root toggle element bounds don't actually
1477 // contain all of the toggle child element bounds. Until we find a way to
1478 // sort that out, we workaround the issue by having each clickable child
1479 // elements of the toggle have a clicklable class, and then compute the
1480 // smallest rect that contains all of their bounding rects and use that
1482 toggleRect = lazy.Rect.fromRect(toggleRect);
1483 let clickableChildren = toggle.querySelectorAll(".clickable");
1484 for (let child of clickableChildren) {
1485 let childRect = lazy.Rect.fromRect(
1486 child.ownerGlobal.windowUtils.getBoundsWithoutFlushing(child)
1488 toggleRect.expandToContain(childRect);
1491 // If the toggle has no dimensions, we're definitely not over it.
1492 if (!toggleRect.width || !toggleRect.height) {
1496 let { clientX, clientY } = event;
1499 clientX >= toggleRect.left &&
1500 clientX <= toggleRect.right &&
1501 clientY >= toggleRect.top &&
1502 clientY <= toggleRect.bottom
1507 * Checks a contextmenu event to see if the mouse is currently over the
1508 * Picture-in-Picture toggle. If so, sends a message to the parent process
1509 * to open up the Picture-in-Picture toggle context menu.
1511 * @param {MouseEvent} event A contextmenu event.
1513 checkContextMenu(event) {
1514 let video = this.getWeakOverVideo();
1519 let shadowRoot = video.openOrClosedShadowRoot;
1524 let toggle = this.getToggleElement(shadowRoot);
1525 if (this.isMouseOverToggle(toggle, event)) {
1526 let devicePixelRatio = toggle.ownerGlobal.devicePixelRatio;
1527 this.sendAsyncMessage("PictureInPicture:OpenToggleContextMenu", {
1528 screenXDevPx: event.screenX * devicePixelRatio,
1529 screenYDevPx: event.screenY * devicePixelRatio,
1530 inputSource: event.inputSource,
1532 event.stopImmediatePropagation();
1533 event.preventDefault();
1538 * Returns the appropriate root element for the Picture-in-Picture toggle,
1539 * depending on whether or not we're using the experimental toggle preference.
1541 * @param {Element} shadowRoot The shadowRoot of the video element.
1542 * @returns {Element} The toggle element.
1544 getToggleElement(shadowRoot) {
1545 return shadowRoot.getElementById("pictureInPictureToggle");
1549 * This is a test-only function that returns true if a video is being tracked
1550 * for mouseover events after having intersected the viewport.
1552 static isTracking(video) {
1553 return gWeakIntersectingVideosForTesting.has(video);
1557 * Gets any Picture-in-Picture site-specific overrides stored in the
1558 * sharedData struct, and returns them as an Array of two-element Arrays,
1559 * where the first element is a MatchPattern and the second element is an
1560 * object of the form { policy, disabledKeyboardControls } (where each property
1561 * may be missing or undefined).
1563 * @returns {Array<Array<2>>} Array of 2-element Arrays where the first element
1564 * is a MatchPattern and the second element is an object with optional policy
1565 * and/or disabledKeyboardControls properties.
1567 static getSiteOverrides() {
1569 let patterns = Services.cpmm.sharedData.get(
1570 "PictureInPicture:SiteOverrides"
1572 for (let pattern in patterns) {
1573 let matcher = new MatchPattern(pattern);
1574 result.push([matcher, patterns[pattern]]);
1580 export class PictureInPictureChild extends JSWindowActorChild {
1581 #subtitlesEnabled = false;
1582 // A weak reference to this PiP window's video element
1585 // A weak reference to this PiP window's content window
1586 weakPlayerContent = null;
1588 // A reference to current WebVTT track currently displayed on the content window
1589 _currentWebVTTTrack = null;
1591 observerFunction = null;
1593 observe(subject, topic, data) {
1594 if (topic != "nsPref:changed") {
1599 case "media.videocontrols.picture-in-picture.display-text-tracks.enabled": {
1600 const originatingVideo = this.getWeakVideo();
1601 let isTextTrackPrefEnabled = Services.prefs.getBoolPref(
1602 "media.videocontrols.picture-in-picture.display-text-tracks.enabled"
1605 // Enable or disable text track support
1606 if (isTextTrackPrefEnabled) {
1607 this.setupTextTracks(originatingVideo);
1609 this.removeTextTracks(originatingVideo);
1617 * Creates a link element with a reference to the css stylesheet needed
1618 * for text tracks responsive styling.
1619 * @returns {Element} the link element containing text tracks stylesheet.
1621 createTextTracksStyleSheet() {
1622 let headStyleElement = this.document.createElement("link");
1623 headStyleElement.setAttribute("rel", "stylesheet");
1624 headStyleElement.setAttribute(
1626 "chrome://global/skin/pictureinpicture/texttracks.css"
1628 headStyleElement.setAttribute("type", "text/css");
1629 return headStyleElement;
1633 * Sets up Picture-in-Picture to support displaying text tracks from WebVTT
1634 * or if WebVTT isn't supported we will register the caption change mutation observer if
1635 * the site wrapper exists.
1637 * If the originating video supports WebVTT, try to read the
1638 * active track and cues. Display any active cues on the pip window
1639 * right away if applicable.
1641 * @param originatingVideo {Element|null}
1642 * The <video> being displayed in Picture-in-Picture mode, or null if that <video> no longer exists.
1644 setupTextTracks(originatingVideo) {
1645 const isWebVTTSupported = !!originatingVideo.textTracks?.length;
1647 if (!isWebVTTSupported) {
1648 this.setUpCaptionChangeListener(originatingVideo);
1652 // Verify active track for originating video
1653 this.setActiveTextTrack(originatingVideo.textTracks);
1655 if (!this._currentWebVTTTrack) {
1656 // If WebVTT track is invalid, try using a video wrapper
1657 this.setUpCaptionChangeListener(originatingVideo);
1661 // Listen for changes in tracks and active cues
1662 originatingVideo.textTracks.addEventListener("change", this);
1663 this._currentWebVTTTrack.addEventListener("cuechange", this.onCueChange);
1665 const cues = this._currentWebVTTTrack.activeCues;
1666 this.updateWebVTTTextTracksDisplay(cues);
1670 * Toggle the visibility of the subtitles in the PiP window
1672 toggleTextTracks() {
1673 let textTracks = this.document.getElementById("texttracks");
1674 textTracks.style.display =
1675 textTracks.style.display === "none" ? "" : "none";
1679 * Removes existing text tracks on the Picture in Picture window.
1681 * If the originating video supports WebVTT, clear references to active
1682 * tracks and cues. No longer listen for any track or cue changes.
1684 * @param originatingVideo {Element|null}
1685 * The <video> being displayed in Picture-in-Picture mode, or null if that <video> no longer exists.
1687 removeTextTracks(originatingVideo) {
1688 const isWebVTTSupported = !!originatingVideo.textTracks;
1690 if (!isWebVTTSupported) {
1694 // No longer listen for changes to tracks and active cues
1695 originatingVideo.textTracks.removeEventListener("change", this);
1696 this._currentWebVTTTrack?.removeEventListener(
1700 this._currentWebVTTTrack = null;
1701 this.updateWebVTTTextTracksDisplay(null);
1705 * Moves the text tracks container position above the pip window's video controls
1706 * if their positions visually overlap. Since pip controls are within the parent
1707 * process, we determine if pip video controls and text tracks visually overlap by
1708 * comparing their relative positions with DOMRect.
1710 * If overlap is found, set attribute "overlap-video-controls" to move text tracks
1711 * and define a new relative bottom position according to pip window size and the
1712 * position of video controls.
1713 * @param {Object} data args needed to determine if text tracks must be moved
1715 moveTextTracks(data) {
1718 isVideoControlsShowing,
1719 playerBottomControlsDOMRect,
1722 let textTracks = this.document.getElementById("texttracks");
1723 const originatingWindow = this.getWeakVideo().ownerGlobal;
1724 const isReducedMotionEnabled = originatingWindow.matchMedia(
1725 "(prefers-reduced-motion: reduce)"
1727 const textTracksFontScale = this.document
1728 .querySelector(":root")
1729 .style.getPropertyValue("--font-scale");
1731 if (isFullscreen || isReducedMotionEnabled) {
1732 textTracks.removeAttribute("overlap-video-controls");
1736 if (isVideoControlsShowing) {
1737 let playerVideoRect = textTracks.parentElement.getBoundingClientRect();
1739 playerVideoRect.bottom - textTracksFontScale * playerVideoRect.height >
1740 playerBottomControlsDOMRect.top;
1743 const root = this.document.querySelector(":root");
1744 if (isScrubberShowing) {
1745 root.style.setProperty("--player-controls-scrubber-height", "30px");
1747 root.style.setProperty("--player-controls-scrubber-height", "0px");
1749 textTracks.setAttribute("overlap-video-controls", true);
1751 textTracks.removeAttribute("overlap-video-controls");
1754 textTracks.removeAttribute("overlap-video-controls");
1759 * Updates the text content for the container that holds and displays text tracks
1760 * on the pip window.
1761 * @param textTrackCues {TextTrackCueList|null}
1762 * Collection of TextTrackCue objects containing text displayed, or null if there is no cue to display.
1764 updateWebVTTTextTracksDisplay(textTrackCues) {
1765 let pipWindowTracksContainer = this.document.getElementById("texttracks");
1766 let playerVideo = this.document.getElementById("playervideo");
1767 let playerVideoWindow = playerVideo.ownerGlobal;
1769 // To prevent overlap with previous cues, clear all text from the pip window
1770 pipWindowTracksContainer.replaceChildren();
1772 if (!textTrackCues) {
1776 if (!this.isSubtitlesEnabled) {
1777 this.isSubtitlesEnabled = true;
1778 this.sendAsyncMessage("PictureInPicture:EnableSubtitlesButton");
1781 let allCuesArray = [...textTrackCues];
1783 this.getOrderedWebVTTCues(allCuesArray);
1784 // Parse through WebVTT cue using vtt.js to ensure
1785 // semantic markup like <b> and <i> tags are rendered.
1786 allCuesArray.forEach(cue => {
1787 let text = cue.text;
1788 // Trim extra newlines and whitespaces
1789 const re = /(\s*\n{2,}\s*)/g;
1791 text = text.replace(re, "\n");
1792 let cueTextNode = WebVTT.convertCueToDOMTree(playerVideoWindow, text);
1793 let cueDiv = this.document.createElement("div");
1794 cueDiv.appendChild(cueTextNode);
1795 pipWindowTracksContainer.appendChild(cueDiv);
1800 * Re-orders list of multiple active cues to ensure cues are rendered in the correct order.
1801 * How cues are ordered depends on the VTTCue.line value of the cue.
1803 * If line is string "auto", we want to reverse the order of cues.
1804 * Cues are read from top to bottom in a vtt file, but are inserted into a video from bottom to top.
1805 * Ensure this order is followed.
1807 * If line is an integer or percentage, we want to order cues according to numeric value.
1809 * 1) all active cues are numeric
1810 * 2) all active cues are in range 0..100
1811 * 3) all actives cue are horizontal (no VTTCue.vertical)
1812 * 4) all active cues with VTTCue.line integer have VTTCue.snapToLines = true
1813 * 5) all active cues with VTTCue.line percentage have VTTCue.snapToLines = false
1815 * vtt.jsm currently sets snapToLines to false if line is a percentage value, but
1816 * cues are still ordered by line. In most cases, snapToLines is set to true by default,
1817 * unless intentionally overridden.
1818 * @param allCuesArray {Array<VTTCue>} array of active cues
1820 getOrderedWebVTTCues(allCuesArray) {
1821 if (!allCuesArray || allCuesArray.length <= 1) {
1825 let allCuesHaveNumericLines = allCuesArray.find(cue => cue.line !== "auto");
1827 if (allCuesHaveNumericLines) {
1828 allCuesArray.sort((cue1, cue2) => cue1.line - cue2.line);
1829 } else if (allCuesArray.length >= 2) {
1830 allCuesArray.reverse();
1835 * Returns a reference to the PiP's <video> element being displayed in Picture-in-Picture
1838 * @return {Element} The <video> being displayed in Picture-in-Picture mode, or null
1839 * if that <video> no longer exists.
1842 if (this.weakVideo) {
1843 // Bug 800957 - Accessing weakrefs at the wrong time can cause us to
1844 // throw NS_ERROR_XPC_BAD_CONVERT_NATIVE
1846 return this.weakVideo.get();
1855 * Returns a reference to the inner window of the about:blank document that is
1856 * cloning the originating <video> in the always-on-top player <xul:browser>.
1858 * @return {Window} The inner window of the about:blank player <xul:browser>, or
1859 * null if that window has been closed.
1861 getWeakPlayerContent() {
1862 if (this.weakPlayerContent) {
1863 // Bug 800957 - Accessing weakrefs at the wrong time can cause us to
1864 // throw NS_ERROR_XPC_BAD_CONVERT_NATIVE
1866 return this.weakPlayerContent.get();
1875 * Returns true if the passed video happens to be the one that this
1876 * content process is running in a Picture-in-Picture window.
1878 * @param {Element} video The <video> element to check.
1882 inPictureInPicture(video) {
1883 return this.getWeakVideo() === video;
1886 static videoIsPlaying(video) {
1887 return !!(!video.paused && !video.ended && video.readyState > 2);
1890 static videoIsMuted(video) {
1891 return this.videoWrapper.isMuted(video);
1894 handleEvent(event) {
1895 switch (event.type) {
1896 case "MozStopPictureInPicture": {
1897 if (event.isTrusted && event.target === this.getWeakVideo()) {
1898 const reason = event.detail?.reason || "videoElRemove";
1899 this.closePictureInPicture({ reason });
1904 // The originating video's content document has unloaded,
1905 // so close Picture-in-Picture.
1906 this.closePictureInPicture({ reason: "pagehide" });
1909 case "MozDOMFullscreen:Request": {
1910 this.closePictureInPicture({ reason: "fullscreen" });
1914 this.sendAsyncMessage("PictureInPicture:Playing");
1918 this.sendAsyncMessage("PictureInPicture:Paused");
1921 case "volumechange": {
1922 let video = this.getWeakVideo();
1924 // Just double-checking that we received the event for the right
1926 if (video !== event.target) {
1927 lazy.logConsole.error(
1928 "PictureInPictureChild received volumechange for " +
1934 if (this.constructor.videoIsMuted(video)) {
1935 this.sendAsyncMessage("PictureInPicture:Muting");
1937 this.sendAsyncMessage("PictureInPicture:Unmuting");
1939 this.sendAsyncMessage("PictureInPicture:VolumeChange", {
1940 volume: this.videoWrapper.getVolume(video),
1945 let video = event.target;
1946 if (this.inPictureInPicture(video)) {
1947 this.sendAsyncMessage("PictureInPicture:Resize", {
1948 videoHeight: video.videoHeight,
1949 videoWidth: video.videoWidth,
1952 this.setupTextTracks(video);
1956 this.isSubtitlesEnabled = false;
1957 if (this.emptiedTimeout) {
1958 clearTimeout(this.emptiedTimeout);
1959 this.emptiedTimeout = null;
1961 let video = this.getWeakVideo();
1962 // We may want to keep the pip window open if the video
1963 // is still in DOM. But if video src is no longer defined,
1964 // close Picture-in-Picture.
1965 this.emptiedTimeout = setTimeout(() => {
1966 if (!video || !video.src) {
1967 this.closePictureInPicture({ reason: "videoElEmptied" });
1969 }, EMPTIED_TIMEOUT_MS);
1973 // Clear currently stored track data (webvtt support) before reading
1975 if (this._currentWebVTTTrack) {
1976 this._currentWebVTTTrack.removeEventListener(
1980 this._currentWebVTTTrack = null;
1983 const tracks = event.target;
1984 this.setActiveTextTrack(tracks);
1985 const isCurrentTrackAvailable = this._currentWebVTTTrack;
1987 // If tracks are disabled or invalid while change occurs,
1988 // remove text tracks from the pip window and stop here.
1989 if (!isCurrentTrackAvailable || !tracks.length) {
1990 this.updateWebVTTTextTracksDisplay(null);
1994 this._currentWebVTTTrack.addEventListener(
1998 const cues = this._currentWebVTTTrack.activeCues;
1999 this.updateWebVTTTextTracksDisplay(cues);
2003 case "durationchange": {
2004 let video = this.getWeakVideo();
2005 let currentTime = this.videoWrapper.getCurrentTime(video);
2006 let duration = this.videoWrapper.getDuration(video);
2007 let scrubberPosition = currentTime === 0 ? 0 : currentTime / duration;
2008 let timestamp = this.videoWrapper.formatTimestamp(
2012 // There's no point in sending this message unless we have a
2013 // reasonable timestamp.
2014 if (timestamp !== undefined && lazy.IMPROVED_CONTROLS_ENABLED_PREF) {
2015 this.sendAsyncMessage(
2016 "PictureInPicture:SetTimestampAndScrubberPosition",
2029 * Tells the parent to close a pre-existing Picture-in-Picture
2034 * @resolves {undefined} Once the pre-existing Picture-in-Picture
2035 * window has unloaded.
2037 async closePictureInPicture({ reason }) {
2038 let video = this.getWeakVideo();
2040 this.untrackOriginatingVideo(video);
2042 this.sendAsyncMessage("PictureInPicture:Close", {
2046 let playerContent = this.getWeakPlayerContent();
2047 if (playerContent) {
2048 if (!playerContent.closed) {
2049 await new Promise(resolve => {
2050 playerContent.addEventListener("unload", resolve, {
2055 // Nothing should be holding a reference to the Picture-in-Picture
2056 // player window content at this point, but just in case, we'll
2057 // clear the weak reference directly so nothing else can get a hold
2058 // of it from this angle.
2059 this.weakPlayerContent = null;
2063 receiveMessage(message) {
2064 switch (message.name) {
2065 case "PictureInPicture:SetupPlayer": {
2066 const { videoRef } = message.data;
2067 this.setupPlayer(videoRef);
2070 case "PictureInPicture:Play": {
2074 case "PictureInPicture:Pause": {
2075 if (message.data && message.data.reason == "pip-closed") {
2076 let video = this.getWeakVideo();
2078 // Currently in Firefox srcObjects are MediaStreams. However, by spec a srcObject
2079 // can be either a MediaStream, MediaSource or Blob. In case of future changes
2080 // we do not want to pause MediaStream srcObjects and we want to maintain current
2081 // behavior for non-MediaStream srcObjects.
2082 if (video && MediaStream.isInstance(video.srcObject)) {
2089 case "PictureInPicture:Mute": {
2093 case "PictureInPicture:Unmute": {
2097 case "PictureInPicture:SeekForward":
2098 case "PictureInPicture:SeekBackward": {
2100 let video = this.getWeakVideo();
2101 let currentTime = this.videoWrapper.getCurrentTime(video);
2102 if (message.name == "PictureInPicture:SeekBackward") {
2103 selectedTime = currentTime - SEEK_TIME_SECS;
2104 selectedTime = selectedTime >= 0 ? selectedTime : 0;
2106 const maxtime = this.videoWrapper.getDuration(video);
2107 selectedTime = currentTime + SEEK_TIME_SECS;
2108 selectedTime = selectedTime <= maxtime ? selectedTime : maxtime;
2110 this.videoWrapper.setCurrentTime(video, selectedTime);
2113 case "PictureInPicture:KeyDown": {
2114 this.keyDown(message.data);
2117 case "PictureInPicture:EnterFullscreen":
2118 case "PictureInPicture:ExitFullscreen": {
2119 let textTracks = this.document.getElementById("texttracks");
2121 this.moveTextTracks(message.data);
2125 case "PictureInPicture:ShowVideoControls":
2126 case "PictureInPicture:HideVideoControls": {
2127 let textTracks = this.document.getElementById("texttracks");
2129 this.moveTextTracks(message.data);
2133 case "PictureInPicture:ToggleTextTracks": {
2134 this.toggleTextTracks();
2137 case "PictureInPicture:ChangeFontSizeTextTracks": {
2138 this.setTextTrackFontSize();
2141 case "PictureInPicture:SetVideoTime": {
2142 const { scrubberPosition, wasPlaying } = message.data;
2143 this.setVideoTime(scrubberPosition, wasPlaying);
2146 case "PictureInPicture:SetVolume": {
2147 const { volume } = message.data;
2148 let video = this.getWeakVideo();
2149 this.videoWrapper.setVolume(video, volume);
2156 * Set the current time of the video based of the position of the scrubber
2157 * @param {Number} scrubberPosition A number between 0 and 1 representing the position of the scrubber
2159 setVideoTime(scrubberPosition, wasPlaying) {
2160 const video = this.getWeakVideo();
2161 let duration = this.videoWrapper.getDuration(video);
2162 let currentTime = scrubberPosition * duration;
2163 this.videoWrapper.setCurrentTime(video, currentTime, wasPlaying);
2167 * @returns {boolean} true if a textTrack with mode "hidden" should be treated as "showing"
2169 shouldShowHiddenTextTracks() {
2170 const video = this.getWeakVideo();
2174 const { documentURI } = video.ownerDocument;
2178 for (let [override, { showHiddenTextTracks }] of lazy.gSiteOverrides) {
2179 if (override.matches(documentURI) && showHiddenTextTracks) {
2187 * Updates this._currentWebVTTTrack if an active track is found
2188 * for the originating video.
2189 * @param {TextTrackList} textTrackList list of text tracks
2191 setActiveTextTrack(textTrackList) {
2192 this._currentWebVTTTrack = null;
2194 for (let i = 0; i < textTrackList.length; i++) {
2195 let track = textTrackList[i];
2196 let isCCText = track.kind === "subtitles" || track.kind === "captions";
2197 let shouldShowTrack =
2198 track.mode === "showing" ||
2199 (track.mode === "hidden" && this.shouldShowHiddenTextTracks());
2200 if (isCCText && shouldShowTrack && track.cues) {
2201 this._currentWebVTTTrack = track;
2208 * Set the font size on the PiP window using the current font size value from
2209 * the "media.videocontrols.picture-in-picture.display-text-tracks.size" pref
2211 setTextTrackFontSize() {
2212 const fontSize = Services.prefs.getStringPref(
2213 TEXT_TRACK_FONT_SIZE,
2216 const root = this.document.querySelector(":root");
2217 if (fontSize === "small") {
2218 root.style.setProperty("--font-scale", "0.03");
2219 } else if (fontSize === "large") {
2220 root.style.setProperty("--font-scale", "0.09");
2222 root.style.setProperty("--font-scale", "0.06");
2227 * Keeps an eye on the originating video's document. If it ever
2228 * goes away, this will cause the Picture-in-Picture window for any
2229 * of its content to go away as well.
2231 trackOriginatingVideo(originatingVideo) {
2232 this.observerFunction = (subject, topic, data) => {
2233 this.observe(subject, topic, data);
2235 Services.prefs.addObserver(
2236 "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
2237 this.observerFunction
2240 let originatingWindow = originatingVideo.ownerGlobal;
2241 if (originatingWindow) {
2242 originatingWindow.addEventListener("pagehide", this);
2243 originatingVideo.addEventListener("play", this);
2244 originatingVideo.addEventListener("pause", this);
2245 originatingVideo.addEventListener("volumechange", this);
2246 originatingVideo.addEventListener("resize", this);
2247 originatingVideo.addEventListener("emptied", this);
2248 originatingVideo.addEventListener("timeupdate", this);
2250 if (lazy.DISPLAY_TEXT_TRACKS_PREF) {
2251 this.setupTextTracks(originatingVideo);
2254 let chromeEventHandler = originatingWindow.docShell.chromeEventHandler;
2255 chromeEventHandler.addEventListener(
2256 "MozDOMFullscreen:Request",
2260 chromeEventHandler.addEventListener(
2261 "MozStopPictureInPicture",
2268 setUpCaptionChangeListener(originatingVideo) {
2269 if (this.videoWrapper) {
2270 this.videoWrapper.setCaptionContainerObserver(originatingVideo, this);
2275 * Stops tracking the originating video's document. This should
2276 * happen once the Picture-in-Picture window goes away (or is about
2277 * to go away), and we no longer care about hearing when the originating
2278 * window's document unloads.
2280 untrackOriginatingVideo(originatingVideo) {
2281 Services.prefs.removeObserver(
2282 "media.videocontrols.picture-in-picture.display-text-tracks.enabled",
2283 this.observerFunction
2286 let originatingWindow = originatingVideo.ownerGlobal;
2287 if (originatingWindow) {
2288 originatingWindow.removeEventListener("pagehide", this);
2289 originatingVideo.removeEventListener("play", this);
2290 originatingVideo.removeEventListener("pause", this);
2291 originatingVideo.removeEventListener("volumechange", this);
2292 originatingVideo.removeEventListener("resize", this);
2293 originatingVideo.removeEventListener("emptied", this);
2294 originatingVideo.removeEventListener("timeupdate", this);
2296 if (lazy.DISPLAY_TEXT_TRACKS_PREF) {
2297 this.removeTextTracks(originatingVideo);
2300 let chromeEventHandler = originatingWindow.docShell.chromeEventHandler;
2301 chromeEventHandler.removeEventListener(
2302 "MozDOMFullscreen:Request",
2306 chromeEventHandler.removeEventListener(
2307 "MozStopPictureInPicture",
2315 * Runs in an instance of PictureInPictureChild for the
2316 * player window's content, and not the originating video
2317 * content. Sets up the player so that it clones the originating
2318 * video. If anything goes wrong during set up, a message is
2319 * sent to the parent to close the Picture-in-Picture window.
2321 * @param videoRef {ContentDOMReference}
2322 * A reference to the video element that a Picture-in-Picture window
2323 * is being created for
2325 * @resolves {undefined} Once the player window has been set up
2326 * properly, or a pre-existing Picture-in-Picture window has gone
2327 * away due to an unexpected error.
2329 async setupPlayer(videoRef) {
2330 const video = await lazy.ContentDOMReference.resolve(videoRef);
2332 this.weakVideo = Cu.getWeakReference(video);
2333 let originatingVideo = this.getWeakVideo();
2334 if (!originatingVideo) {
2335 // If the video element has gone away before we've had a chance to set up
2336 // Picture-in-Picture for it, tell the parent to close the Picture-in-Picture
2338 await this.closePictureInPicture({ reason: "setupFailure" });
2342 this.videoWrapper = applyWrapper(this, originatingVideo);
2344 let loadPromise = new Promise(resolve => {
2345 this.contentWindow.addEventListener("load", resolve, {
2347 mozSystemGroup: true,
2351 this.contentWindow.location.reload();
2354 // We're committed to adding the video to this window now. Ensure we track
2355 // the content window before we do so, so that the toggle actor can
2356 // distinguish this new video we're creating from web-controlled ones.
2357 this.weakPlayerContent = Cu.getWeakReference(this.contentWindow);
2358 gPlayerContents.add(this.contentWindow);
2360 let doc = this.document;
2361 let playerVideo = doc.createElement("video");
2362 playerVideo.id = "playervideo";
2363 let textTracks = doc.createElement("div");
2365 doc.body.style.overflow = "hidden";
2366 doc.body.style.margin = "0";
2368 // Force the player video to assume maximum height and width of the
2369 // containing window
2370 playerVideo.style.height = "100vh";
2371 playerVideo.style.width = "100vw";
2372 playerVideo.style.backgroundColor = "#000";
2374 // Load text tracks container in the content process so that
2375 // we can load text tracks without having to constantly
2376 // access the parent process.
2377 textTracks.id = "texttracks";
2378 // When starting pip, player controls are expected to appear.
2379 textTracks.setAttribute("overlap-video-controls", true);
2380 doc.body.appendChild(playerVideo);
2381 doc.body.appendChild(textTracks);
2382 // Load text tracks stylesheet
2383 let textTracksStyleSheet = this.createTextTracksStyleSheet();
2384 doc.head.appendChild(textTracksStyleSheet);
2386 this.setTextTrackFontSize();
2388 originatingVideo.cloneElementVisually(playerVideo);
2390 let shadowRoot = originatingVideo.openOrClosedShadowRoot;
2391 if (originatingVideo.getTransformToViewport().a == -1) {
2392 shadowRoot.firstChild.setAttribute("flipped", true);
2393 playerVideo.style.transform = "scaleX(-1)";
2396 this.onCueChange = this.onCueChange.bind(this);
2397 this.trackOriginatingVideo(originatingVideo);
2399 // A request to open PIP implies that the user intends to be interacting
2400 // with the page, even if they open PIP by some means outside of the page
2401 // itself (e.g., the keyboard shortcut or the page action button). So we
2402 // manually record that the document has been activated via user gesture
2403 // to make sure the video can be played regardless of autoplay permissions.
2404 originatingVideo.ownerDocument.notifyUserGestureActivation();
2406 this.contentWindow.addEventListener(
2409 let video = this.getWeakVideo();
2411 this.untrackOriginatingVideo(video);
2412 video.stopCloningElementVisually();
2414 this.weakVideo = null;
2421 let video = this.getWeakVideo();
2422 if (video && this.videoWrapper) {
2423 this.videoWrapper.play(video);
2428 let video = this.getWeakVideo();
2429 if (video && this.videoWrapper) {
2430 this.videoWrapper.pause(video);
2435 let video = this.getWeakVideo();
2436 if (video && this.videoWrapper) {
2437 this.videoWrapper.setMuted(video, true);
2442 let video = this.getWeakVideo();
2443 if (video && this.videoWrapper) {
2444 this.videoWrapper.setMuted(video, false);
2449 if (!lazy.DISPLAY_TEXT_TRACKS_PREF) {
2450 this.updateWebVTTTextTracksDisplay(null);
2452 const cues = this._currentWebVTTTrack.activeCues;
2453 this.updateWebVTTTextTracksDisplay(cues);
2458 * This checks if a given keybinding has been disabled for the specific site
2459 * currently being viewed.
2461 isKeyDisabled(key) {
2462 const video = this.getWeakVideo();
2466 const { documentURI } = video.ownerDocument;
2470 for (let [override, { disabledKeyboardControls }] of lazy.gSiteOverrides) {
2472 disabledKeyboardControls !== undefined &&
2473 override.matches(documentURI)
2475 if (disabledKeyboardControls === lazy.KEYBOARD_CONTROLS.ALL) {
2478 return !!(disabledKeyboardControls & key);
2485 * This reuses the keyHandler logic in the VideoControlsWidget
2486 * https://searchfox.org/mozilla-central/rev/cfd1cc461f1efe0d66c2fdc17c024a203d5a2fd8/toolkit/content/widgets/videocontrols.js#1687-1810.
2487 * There are future plans to eventually combine the two implementations.
2489 /* eslint-disable complexity */
2490 keyDown({ altKey, shiftKey, metaKey, ctrlKey, keyCode }) {
2491 let video = this.getWeakVideo();
2498 keystroke += "alt-";
2501 keystroke += "shift-";
2503 if (this.contentWindow.navigator.platform.startsWith("Mac")) {
2505 keystroke += "accel-";
2508 keystroke += "control-";
2512 keystroke += "meta-";
2515 keystroke += "accel-";
2520 case this.contentWindow.KeyEvent.DOM_VK_UP:
2521 keystroke += "upArrow";
2523 case this.contentWindow.KeyEvent.DOM_VK_DOWN:
2524 keystroke += "downArrow";
2526 case this.contentWindow.KeyEvent.DOM_VK_LEFT:
2527 keystroke += "leftArrow";
2529 case this.contentWindow.KeyEvent.DOM_VK_RIGHT:
2530 keystroke += "rightArrow";
2532 case this.contentWindow.KeyEvent.DOM_VK_HOME:
2533 keystroke += "home";
2535 case this.contentWindow.KeyEvent.DOM_VK_END:
2538 case this.contentWindow.KeyEvent.DOM_VK_SPACE:
2539 keystroke += "space";
2541 case this.contentWindow.KeyEvent.DOM_VK_W:
2546 const isVideoStreaming = this.videoWrapper.isLive(video);
2550 switch (keystroke) {
2551 case "space" /* Toggle Play / Pause */:
2552 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.PLAY_PAUSE)) {
2557 this.videoWrapper.getPaused(video) ||
2558 this.videoWrapper.getEnded(video)
2560 this.videoWrapper.play(video);
2562 this.videoWrapper.pause(video);
2566 case "accel-w" /* Close video */:
2567 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.CLOSE)) {
2571 this.closePictureInPicture({ reason: "closePlayerShortcut" });
2573 case "downArrow" /* Volume decrease */:
2575 this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.VOLUME) ||
2576 this.videoWrapper.isMuted(video)
2580 oldval = this.videoWrapper.getVolume(video);
2581 newval = oldval < 0.1 ? 0 : oldval - 0.1;
2582 this.videoWrapper.setVolume(video, newval);
2583 this.videoWrapper.setMuted(video, newval === 0);
2585 case "upArrow" /* Volume increase */:
2586 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.VOLUME)) {
2589 oldval = this.videoWrapper.getVolume(video);
2590 this.videoWrapper.setVolume(video, oldval > 0.9 ? 1 : oldval + 0.1);
2591 this.videoWrapper.setMuted(video, false);
2593 case "accel-downArrow" /* Mute */:
2594 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.MUTE_UNMUTE)) {
2597 this.videoWrapper.setMuted(video, true);
2599 case "accel-upArrow" /* Unmute */:
2600 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.MUTE_UNMUTE)) {
2603 this.videoWrapper.setMuted(video, false);
2605 case "leftArrow": /* Seek back 5 seconds */
2606 case "accel-leftArrow" /* Seek back 10% */:
2608 this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.SEEK) ||
2609 (isVideoStreaming &&
2610 this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.LIVE_SEEK))
2615 oldval = this.videoWrapper.getCurrentTime(video);
2616 if (keystroke == "leftArrow") {
2617 newval = oldval - SEEK_TIME_SECS;
2619 newval = oldval - this.videoWrapper.getDuration(video) / 10;
2621 this.videoWrapper.setCurrentTime(video, newval >= 0 ? newval : 0);
2623 case "rightArrow": /* Seek forward 5 seconds */
2624 case "accel-rightArrow" /* Seek forward 10% */:
2626 this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.SEEK) ||
2627 (isVideoStreaming &&
2628 this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.LIVE_SEEK))
2633 oldval = this.videoWrapper.getCurrentTime(video);
2634 var maxtime = this.videoWrapper.getDuration(video);
2635 if (keystroke == "rightArrow") {
2636 newval = oldval + SEEK_TIME_SECS;
2638 newval = oldval + maxtime / 10;
2640 let selectedTime = newval <= maxtime ? newval : maxtime;
2641 this.videoWrapper.setCurrentTime(video, selectedTime);
2643 case "home" /* Seek to beginning */:
2644 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.SEEK)) {
2647 if (!isVideoStreaming) {
2648 this.videoWrapper.setCurrentTime(video, 0);
2651 case "end" /* Seek to end */:
2652 if (this.isKeyDisabled(lazy.KEYBOARD_CONTROLS.SEEK)) {
2656 let duration = this.videoWrapper.getDuration(video);
2658 !isVideoStreaming &&
2659 this.videoWrapper.getCurrentTime(video) != duration
2661 this.videoWrapper.setCurrentTime(video, duration);
2667 /* ignore any exception from setting video.currentTime */
2671 get isSubtitlesEnabled() {
2672 return this.#subtitlesEnabled;
2675 set isSubtitlesEnabled(val) {
2677 Services.telemetry.recordEvent(
2683 webVTTSubtitles: (!!this.getWeakVideo().textTracks
2684 ?.length).toString(),
2688 this.sendAsyncMessage("PictureInPicture:DisableSubtitlesButton");
2690 this.#subtitlesEnabled = val;
2695 * The PictureInPictureChildVideoWrapper class handles providing a path to a script that
2696 * defines a "site wrapper" for the original <video> (or other controls API provided
2697 * by the site) to command it.
2699 * This "site wrapper" provided to PictureInPictureChildVideoWrapper is a script file that
2700 * defines a class called `PictureInPictureVideoWrapper` and exports it. These scripts can
2701 * be found under "browser/extensions/pictureinpicture/video-wrappers" as part of the
2702 * Picture-In-Picture addon.
2704 * Site wrappers need to adhere to a specific interface to work properly with
2705 * PictureInPictureChildVideoWrapper:
2707 * - The "site wrapper" script must export a class called "PictureInPictureVideoWrapper"
2708 * - Method names on a site wrapper class should match its caller's name
2709 * (i.e: PictureInPictureChildVideoWrapper.play will only call `play` on a site-wrapper, if available)
2711 class PictureInPictureChildVideoWrapper {
2714 #PictureInPictureChild;
2717 * Create a wrapper for the original <video>
2719 * @param {String|null} videoWrapperScriptPath
2720 * Path to a wrapper script from the Picture-in-Picture addon. If a wrapper isn't
2721 * provided to the class, then we fallback on a default implementation for
2722 * commanding the original <video>.
2723 * @param {HTMLVideoElement} video
2724 * The original <video> we want to create a wrapper class for.
2725 * @param {Object} pipChild
2726 * Reference to PictureInPictureChild class calling this function.
2728 constructor(videoWrapperScriptPath, video, pipChild) {
2729 this.#sandbox = videoWrapperScriptPath
2730 ? this.#createSandbox(videoWrapperScriptPath, video)
2732 this.#PictureInPictureChild = pipChild;
2736 * Handles calling methods defined on the site wrapper class to perform video
2737 * controls operations on the source video. If the method doesn't exist,
2738 * or if an error is thrown while calling it, use a fallback implementation.
2740 * @param {String} methodInfo.name
2741 * The method name to call.
2742 * @param {Array} methodInfo.args
2743 * Arguments to pass to the site wrapper method being called.
2744 * @param {Function} methodInfo.fallback
2745 * A fallback function that's invoked when a method doesn't exist on the site
2746 * wrapper class or an error is thrown while calling a method
2747 * @param {Function} methodInfo.validateReturnVal
2748 * Validates whether or not the return value of the wrapper method is correct.
2749 * If this isn't provided or if it evaluates false for a return value, then
2752 * @returns The expected output of the wrapper function.
2754 #callWrapperMethod({ name, args = [], fallback = () => {}, validateRetVal }) {
2756 const wrappedMethod = this.#siteWrapper?.[name];
2757 if (typeof wrappedMethod === "function") {
2758 let retVal = wrappedMethod.call(this.#siteWrapper, ...args);
2760 if (!validateRetVal) {
2761 lazy.logConsole.error(
2762 `No return value validator was provided for method ${name}(). Returning null.`
2767 if (!validateRetVal(retVal)) {
2768 lazy.logConsole.error(
2769 `Calling method ${name}() returned an unexpected value: ${retVal}. Returning null.`
2777 lazy.logConsole.error(
2778 `There was an error while calling ${name}(): `,
2787 * Creates a sandbox with Xray vision to execute content code in an unprivileged
2788 * context. This way, privileged code (PictureInPictureChild) can call into the
2789 * sandbox to perform video controls operations on the originating video
2790 * (content code) and still be protected from direct access by it.
2792 * @param {String} videoWrapperScriptPath
2793 * Path to a wrapper script from the Picture-in-Picture addon.
2794 * @param {HTMLVideoElement} video
2795 * The source video element whose window to create a sandbox for.
2797 #createSandbox(videoWrapperScriptPath, video) {
2798 const addonPolicy = WebExtensionPolicy.getByID(
2799 "pictureinpicture@mozilla.org"
2801 let wrapperScriptUrl = addonPolicy.getURL(videoWrapperScriptPath);
2802 let originatingWin = video.ownerGlobal;
2803 let originatingDoc = video.ownerDocument;
2805 let sandbox = Cu.Sandbox([originatingDoc.nodePrincipal], {
2806 sandboxName: "Picture-in-Picture video wrapper sandbox",
2807 sandboxPrototype: originatingWin,
2808 sameZoneAs: originatingWin,
2813 Services.scriptloader.loadSubScript(wrapperScriptUrl, sandbox);
2815 Cu.nukeSandbox(sandbox);
2816 lazy.logConsole.error(
2817 "Error loading wrapper script for Picture-in-Picture",
2823 // The prototype of the wrapper class instantiated from the sandbox with Xray
2824 // vision is `Object` and not actually `PictureInPictureVideoWrapper`. But we
2825 // need to be able to access methods defined on this class to perform site-specific
2826 // video control operations otherwise we fallback to a default implementation.
2827 // Because of this, we need to "waive Xray vision" by adding `.wrappedObject` to the
2829 this.#siteWrapper = new sandbox.PictureInPictureVideoWrapper(
2837 return typeof val === "boolean";
2841 return typeof val === "number";
2845 * Destroys the sandbox for the site wrapper class
2848 if (this.#sandbox) {
2849 Cu.nukeSandbox(this.#sandbox);
2854 * Function to display the captions on the PiP window
2855 * @param text The captions to be shown on the PiP window
2857 updatePiPTextTracks(text) {
2858 if (!this.#PictureInPictureChild.isSubtitlesEnabled && text) {
2859 this.#PictureInPictureChild.isSubtitlesEnabled = true;
2860 this.#PictureInPictureChild.sendAsyncMessage(
2861 "PictureInPicture:EnableSubtitlesButton"
2864 let pipWindowTracksContainer =
2865 this.#PictureInPictureChild.document.getElementById("texttracks");
2866 pipWindowTracksContainer.textContent = text;
2869 /* Video methods to be used for video controls from the PiP window. */
2872 * OVERRIDABLE - calls the play() method defined in the site wrapper script. Runs a fallback implementation
2873 * if the method does not exist or if an error is thrown while calling it. This method is meant to handle video
2874 * behaviour when a video is played.
2875 * @param {HTMLVideoElement} video
2876 * The originating video source element
2879 return this.#callWrapperMethod({
2882 fallback: () => video.play(),
2883 validateRetVal: retVal => retVal == null,
2888 * OVERRIDABLE - calls the pause() method defined in the site wrapper script. Runs a fallback implementation
2889 * if the method does not exist or if an error is thrown while calling it. This method is meant to handle video
2890 * behaviour when a video is paused.
2891 * @param {HTMLVideoElement} video
2892 * The originating video source element
2895 return this.#callWrapperMethod({
2898 fallback: () => video.pause(),
2899 validateRetVal: retVal => retVal == null,
2904 * OVERRIDABLE - calls the getPaused() method defined in the site wrapper script. Runs a fallback implementation
2905 * if the method does not exist or if an error is thrown while calling it. This method is meant to determine if
2906 * a video is paused or not.
2907 * @param {HTMLVideoElement} video
2908 * The originating video source element
2909 * @returns {Boolean} Boolean value true if paused, or false if video is still playing
2912 return this.#callWrapperMethod({
2915 fallback: () => video.paused,
2916 validateRetVal: retVal => this.#isBoolean(retVal),
2921 * OVERRIDABLE - calls the getEnded() method defined in the site wrapper script. Runs a fallback implementation
2922 * if the method does not exist or if an error is thrown while calling it. This method is meant to determine if
2923 * video playback or streaming has stopped.
2924 * @param {HTMLVideoElement} video
2925 * The originating video source element
2926 * @returns {Boolean} Boolean value true if the video has ended, or false if still playing
2929 return this.#callWrapperMethod({
2932 fallback: () => video.ended,
2933 validateRetVal: retVal => this.#isBoolean(retVal),
2938 * OVERRIDABLE - calls the getDuration() method defined in the site wrapper script. Runs a fallback implementation
2939 * if the method does not exist or if an error is thrown while calling it. This method is meant to get the current
2940 * duration of a video in seconds.
2941 * @param {HTMLVideoElement} video
2942 * The originating video source element
2943 * @returns {Number} Duration of the video in seconds
2945 getDuration(video) {
2946 return this.#callWrapperMethod({
2947 name: "getDuration",
2949 fallback: () => video.duration,
2950 validateRetVal: retVal => this.#isNumber(retVal),
2955 * OVERRIDABLE - calls the getCurrentTime() method defined in the site wrapper script. Runs a fallback implementation
2956 * if the method does not exist or if an error is thrown while calling it. This method is meant to get the current
2957 * time of a video in seconds.
2958 * @param {HTMLVideoElement} video
2959 * The originating video source element
2960 * @returns {Number} Current time of the video in seconds
2962 getCurrentTime(video) {
2963 return this.#callWrapperMethod({
2964 name: "getCurrentTime",
2966 fallback: () => video.currentTime,
2967 validateRetVal: retVal => this.#isNumber(retVal),
2972 * OVERRIDABLE - calls the setCurrentTime() method defined in the site wrapper script. Runs a fallback implementation
2973 * if the method does not exist or if an error is thrown while calling it. This method is meant to set the current
2975 * @param {HTMLVideoElement} video
2976 * The originating video source element
2977 * @param {Number} position
2978 * The current playback time of the video
2979 * @param {Boolean} wasPlaying
2980 * True if the video was playing before seeking else false
2982 setCurrentTime(video, position, wasPlaying) {
2983 return this.#callWrapperMethod({
2984 name: "setCurrentTime",
2985 args: [video, position, wasPlaying],
2987 video.currentTime = position;
2989 validateRetVal: retVal => retVal == null,
2994 * Return hours, minutes, and seconds from seconds
2995 * @param {Number} aSeconds
2996 * The time in seconds
2997 * @returns {String} Timestamp string
2999 timeFromSeconds(aSeconds) {
3000 aSeconds = isNaN(aSeconds) ? 0 : Math.round(aSeconds);
3001 let seconds = Math.floor(aSeconds % 60),
3002 minutes = Math.floor((aSeconds / 60) % 60),
3003 hours = Math.floor(aSeconds / 3600);
3004 seconds = seconds < 10 ? "0" + seconds : seconds;
3005 minutes = hours > 0 && minutes < 10 ? "0" + minutes : minutes;
3006 return aSeconds < 3600
3007 ? `${minutes}:${seconds}`
3008 : `${hours}:${minutes}:${seconds}`;
3012 * Format a timestamp from current time and total duration,
3013 * output as a string in the form '0:00 / 0:00'
3014 * @param {Number} aCurrentTime
3015 * The current time in seconds
3016 * @param {Number} aDuration
3017 * The total duration in seconds
3018 * @returns {String} Formatted timestamp
3020 formatTimestamp(aCurrentTime, aDuration) {
3021 // We can't format numbers that can't be represented as decimal digits.
3022 if (!Number.isFinite(aCurrentTime) || !Number.isFinite(aDuration)) {
3026 return `${this.timeFromSeconds(aCurrentTime)} / ${this.timeFromSeconds(
3032 * OVERRIDABLE - calls the getVolume() method defined in the site wrapper script. Runs a fallback implementation
3033 * if the method does not exist or if an error is thrown while calling it. This method is meant to get the volume
3035 * @param {HTMLVideoElement} video
3036 * The originating video source element
3037 * @returns {Number} Volume of the video between 0 (muted) and 1 (loudest)
3040 return this.#callWrapperMethod({
3043 fallback: () => video.volume,
3044 validateRetVal: retVal => this.#isNumber(retVal),
3049 * OVERRIDABLE - calls the setVolume() method defined in the site wrapper script. Runs a fallback implementation
3050 * if the method does not exist or if an error is thrown while calling it. This method is meant to set the volume
3052 * @param {HTMLVideoElement} video
3053 * The originating video source element
3054 * @param {Number} volume
3055 * Value between 0 (muted) and 1 (loudest)
3057 setVolume(video, volume) {
3058 return this.#callWrapperMethod({
3060 args: [video, volume],
3062 video.volume = volume;
3064 validateRetVal: retVal => retVal == null,
3069 * OVERRIDABLE - calls the isMuted() method defined in the site wrapper script. Runs a fallback implementation
3070 * if the method does not exist or if an error is thrown while calling it. This method is meant to get the mute
3072 * @param {HTMLVideoElement} video
3073 * The originating video source element
3074 * @param {Boolean} shouldMute
3075 * Boolean value true to mute the video, or false to unmute the video
3078 return this.#callWrapperMethod({
3081 fallback: () => video.muted,
3082 validateRetVal: retVal => this.#isBoolean(retVal),
3087 * OVERRIDABLE - calls the setMuted() method defined in the site wrapper script. Runs a fallback implementation
3088 * if the method does not exist or if an error is thrown while calling it. This method is meant to mute or unmute
3090 * @param {HTMLVideoElement} video
3091 * The originating video source element
3092 * @param {Boolean} shouldMute
3093 * Boolean value true to mute the video, or false to unmute the video
3095 setMuted(video, shouldMute) {
3096 return this.#callWrapperMethod({
3098 args: [video, shouldMute],
3100 video.muted = shouldMute;
3102 validateRetVal: retVal => retVal == null,
3107 * OVERRIDABLE - calls the setCaptionContainerObserver() method defined in the site wrapper script. Runs a fallback implementation
3108 * if the method does not exist or if an error is thrown while calling it. This method is meant to listen for any cue changes in a
3109 * video's caption container and execute a callback function responsible for updating the pip window's text tracks container whenever
3110 * a cue change is triggered {@see updatePiPTextTracks()}.
3111 * @param {HTMLVideoElement} video
3112 * The originating video source element
3113 * @param {Function} callback
3114 * The callback function to be executed when cue changes are detected
3116 setCaptionContainerObserver(video, callback) {
3117 return this.#callWrapperMethod({
3118 name: "setCaptionContainerObserver",
3122 this.updatePiPTextTracks(text);
3126 validateRetVal: retVal => retVal == null,
3131 * OVERRIDABLE - calls the shouldHideToggle() method defined in the site wrapper script. Runs a fallback implementation
3132 * if the method does not exist or if an error is thrown while calling it. This method is meant to determine if the pip toggle
3133 * for a video should be hidden by the site wrapper.
3134 * @param {HTMLVideoElement} video
3135 * The originating video source element
3136 * @returns {Boolean} Boolean value true if the pip toggle should be hidden by the site wrapper, or false if it should not
3138 shouldHideToggle(video) {
3139 return this.#callWrapperMethod({
3140 name: "shouldHideToggle",
3142 fallback: () => false,
3143 validateRetVal: retVal => this.#isBoolean(retVal),
3148 * OVERRIDABLE - calls the isLive() method defined in the site wrapper script. Runs a fallback implementation
3149 * if the method does not exist or if an error is thrown while calling it. This method is meant to get if the
3150 * video is a live stream.
3151 * @param {HTMLVideoElement} video
3152 * The originating video source element
3155 return this.#callWrapperMethod({
3158 fallback: () => video.duration === Infinity,
3159 validateRetVal: retVal => this.#isBoolean(retVal),