1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 r
"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime).
7 Browsertime is a harness for running performance tests, similar to
8 Mozilla's Raptor testing framework. Browsertime is written in Node.js
9 and uses Selenium WebDriver to drive multiple browsers including
10 Chrome, Chrome for Android, Firefox, and (pending the resolution of
11 [Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126)
12 and similar tickets) Firefox for Android and GeckoView-based vehicles.
14 Right now a custom version of browsertime and the underlying
15 geckodriver binary are needed to support GeckoView-based vehicles;
16 this module accommodates those in-progress custom versions.
20 ./mach browsertime --setup [--clobber]
22 This will populate `tools/browsertime/node_modules`.
24 To invoke browsertime, run
26 ./mach browsertime [ARGS]
28 All arguments are passed through to browsertime.
44 import mozpack
.path
as mozpath
45 from mach
.decorators
import Command
, CommandArgument
46 from mozbuild
.base
import BinaryNotFoundException
, MachCommandBase
47 from mozbuild
.dirutils
import mkdir
48 from six
import StringIO
50 AUTOMATION
= "MOZ_AUTOMATION" in os
.environ
51 BROWSERTIME_ROOT
= os
.path
.dirname(__file__
)
53 PILLOW_VERSION
= "8.4.0" # version 8.4.0 currently supports python 3.6 to 3.10
54 PYSSIM_VERSION
= "0.4"
55 SCIPY_VERSION
= "1.2.3"
56 NUMPY_VERSION
= "1.16.1"
57 OPENCV_VERSION
= "4.5.4.60"
59 py3_minor
= sys
.version_info
.minor
61 SCIPY_VERSION
= "1.9.3"
62 NUMPY_VERSION
= "1.23.5"
63 PILLOW_VERSION
= "9.2.0"
64 OPENCV_VERSION
= "4.6.0.66"
66 MIN_NODE_VERSION
= "16.0.0"
68 IS_APPLE_SILICON
= sys
.platform
.startswith(
70 ) and platform
.processor().startswith("arm")
73 @contextlib.contextmanager
75 oldout
, olderr
= sys
.stdout
, sys
.stderr
77 sys
.stdout
, sys
.stderr
= StringIO(), StringIO()
80 sys
.stdout
, sys
.stderr
= oldout
, olderr
83 def node_path(command_context
):
86 from mozbuild
.nodeutil
import find_node_executable
87 from packaging
.version
import Version
89 state_dir
= command_context
._mach
_context
.state_dir
90 cache_path
= os
.path
.join(state_dir
, "browsertime", "node-16")
93 "Could not locate a node binary that is at least version {}. ".format(
96 + "Please run `./mach raptor --browsertime -t amazon` to install it "
97 + "from the Taskcluster Toolchain artifacts."
100 # Check standard locations first
101 node_exe
= find_node_executable(min_version
=Version(MIN_NODE_VERSION
))
102 if node_exe
and (node_exe
[0] is not None):
103 return os
.path
.abspath(node_exe
[0])
104 if not os
.path
.exists(cache_path
):
105 raise Exception(NODE_FAILURE_MSG
)
107 # Check the browsertime-specific node location next
109 if platform
.system() == "Windows":
110 node_name
= "node.exe"
111 node_exe_path
= os
.path
.join(
118 node_exe_path
= os
.path
.join(
126 node_exe
= os
.path
.join(node_exe_path
, node_name
)
127 if not os
.path
.exists(node_exe
):
128 raise Exception(NODE_FAILURE_MSG
)
130 return os
.path
.abspath(node_exe
)
134 """The path to the `browsertime` directory.
136 Override the default with the `BROWSERTIME` environment variable."""
137 override
= os
.environ
.get("BROWSERTIME", None)
141 return mozpath
.join(BROWSERTIME_ROOT
, "node_modules", "browsertime")
144 def browsertime_path():
145 """The path to the `browsertime.js` script."""
146 # On Windows, invoking `node_modules/.bin/browsertime{.cmd}`
147 # doesn't work when invoked as an argument to our specific
148 # binary. Since we want our version of node, invoke the
149 # actual script directly.
150 return mozpath
.join(package_path(), "bin", "browsertime.js")
153 def visualmetrics_path():
154 """The path to the `visualmetrics.py` script."""
155 return mozpath
.join(package_path(), "visualmetrics", "visualmetrics-portable.py")
159 is_64bits
= sys
.maxsize
> 2**32
161 if sys
.platform
.startswith("win"):
164 elif sys
.platform
.startswith("linux"):
167 elif sys
.platform
.startswith("darwin"):
170 raise ValueError("sys.platform is not yet supported: {}".format(sys
.platform
))
173 # Map from `host_platform()` to a `fetch`-like syntax.
177 "type": "static-url",
178 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip", # noqa
179 # An extension to `fetch` syntax.
180 "path": "ffmpeg-macos",
185 "type": "static-url",
186 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz", # noqa
187 # An extension to `fetch` syntax.
188 "path": "ffmpeg-4.4.1-i686-static",
193 "type": "static-url",
194 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip", # noqa
195 # An extension to `fetch` syntax.
196 "path": "ffmpeg-4.4.1-full_build",
202 def artifact_cache_path(command_context
):
203 r
"""Downloaded artifacts will be kept here."""
204 # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
205 return mozpath
.join(command_context
._mach
_context
.state_dir
, "cache", "browsertime")
208 def state_path(command_context
):
209 r
"""Unpacked artifacts will be kept here."""
210 # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
211 return mozpath
.join(command_context
._mach
_context
.state_dir
, "browsertime")
214 def setup_prerequisites(command_context
):
215 r
"""Install browsertime and visualmetrics.py prerequisites."""
217 from mozbuild
.action
.tooltool
import unpack_file
218 from mozbuild
.artifact_cache
import ArtifactCache
220 # Download the visualmetrics-portable.py requirements.
221 artifact_cache
= ArtifactCache(
222 artifact_cache_path(command_context
),
223 log
=command_context
.log
,
227 fetches
= host_fetches
[host_platform()]
228 for tool
, fetch
in sorted(fetches
.items()):
229 archive
= artifact_cache
.fetch(fetch
["url"])
230 # TODO: assert type, verify sha256 (and size?).
232 if fetch
.get("unpack", True):
235 mkdir(state_path(command_context
))
236 os
.chdir(state_path(command_context
))
241 "Unpacking temporary location {path}",
246 # Make sure the expected path exists after extraction
247 path
= os
.path
.join(state_path(command_context
), fetch
.get("path"))
248 if not os
.path
.exists(path
):
249 raise Exception("Cannot find an extracted directory: %s" % path
)
252 # Some archives provide binaries that don't have the
253 # executable bit set so we need to set it here
254 for root
, dirs
, files
in os
.walk(path
):
256 loc_to_change
= os
.path
.join(root
, edir
)
257 st
= os
.stat(loc_to_change
)
258 os
.chmod(loc_to_change
, st
.st_mode | stat
.S_IEXEC
)
260 loc_to_change
= os
.path
.join(root
, efile
)
261 st
= os
.stat(loc_to_change
)
262 os
.chmod(loc_to_change
, st
.st_mode | stat
.S_IEXEC
)
263 except Exception as e
:
265 "Could not set executable bit in %s, error: %s" % (path
, str(e
))
271 def setup_browsertime(
273 should_clobber
=False,
275 install_vismet_reqs
=False,
277 r
"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
279 sys
.path
.append(mozpath
.join(command_context
.topsrcdir
, "tools", "lint", "eslint"))
282 if not new_upstream_url
:
283 setup_prerequisites(command_context
)
286 package_json_path
= os
.path
.join(BROWSERTIME_ROOT
, "package.json")
292 "new_upstream_url": new_upstream_url
,
293 "package_json_path": package_json_path
,
295 "Updating browsertime node module version in {package_json_path} "
296 "to {new_upstream_url}",
299 if not re
.search("/tarball/[a-f0-9]{40}$", new_upstream_url
):
301 "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'"
305 with
open(package_json_path
) as f
:
306 existing_body
= json
.loads(
307 f
.read(), object_pairs_hook
=collections
.OrderedDict
310 existing_body
["devDependencies"]["browsertime"] = new_upstream_url
312 updated_body
= json
.dumps(existing_body
, indent
=2)
314 with
open(package_json_path
, "w") as f
:
315 f
.write(updated_body
)
317 # Install the browsertime Node.js requirements.
318 if not setup_helper
.check_node_executables_valid():
321 # To use a custom `geckodriver`, set
322 # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
323 # to an endpoint with binaries named like
324 # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
326 os
.environ
["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
327 os
.environ
["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
329 if install_vismet_reqs
:
330 # Hide this behind a flag so we don't install them by default in Raptor
332 logging
.INFO
, "browsertime", {}, "Installing python requirements"
334 activate_browsertime_virtualenv(command_context
)
339 {"package_json": mozpath
.join(BROWSERTIME_ROOT
, "package.json")},
340 "Installing browsertime node module from {package_json}",
343 # Add the mozbuild Node binary path to the OS environment in Apple Silicon.
344 # During the browesertime installation, it seems installation of sitespeed.io
345 # sub dependencies look for a global Node rather than the mozbuild Node binary.
346 # Normally `--scripts-prepend-node-path` should prevent this but it seems to still
347 # look for a system Node in the environment. This logic ensures the same Node is used.
348 node_dir
= os
.path
.dirname(node_path(command_context
))
349 if IS_APPLE_SILICON
and node_dir
not in os
.environ
["PATH"]:
350 os
.environ
["PATH"] += os
.pathsep
+ node_dir
352 status
= setup_helper
.package_setup(
355 should_update
=new_upstream_url
!= "",
356 should_clobber
=should_clobber
,
357 no_optional
=new_upstream_url
or AUTOMATION
,
362 if new_upstream_url
or AUTOMATION
:
364 if install_vismet_reqs
:
365 return check(command_context
)
370 def node(command_context
, args
):
371 r
"""Invoke node (interactively) with the given arguments."""
372 return command_context
.run_process(
373 [node_path(command_context
)] + args
,
374 append_env
=append_env(command_context
),
375 pass_thru
=True, # Allow user to run Node interactively.
376 ensure_exit_code
=False, # Don't throw on non-zero exit code.
377 cwd
=mozpath
.join(command_context
.topsrcdir
),
381 def append_env(command_context
, append_path
=True):
382 fetches
= host_fetches
[host_platform()]
384 # Ensure that `ffmpeg` is found and added to the environment
385 path
= os
.environ
.get("PATH", "").split(os
.pathsep
) if append_path
else []
386 path_to_ffmpeg
= mozpath
.join(
387 state_path(command_context
), fetches
["ffmpeg"]["path"]
393 if host_platform().startswith("linux")
394 else mozpath
.join(path_to_ffmpeg
, "bin"),
397 # Ensure that bare `node` and `npm` in scripts, including post-install
398 # scripts, finds the binary we're invoking with. Without this, it's
399 # easy for compiled extensions to get mismatched versions of the Node.js
401 node_dir
= os
.path
.dirname(node_path(command_context
))
402 path
= [node_dir
] + path
405 "PATH": os
.pathsep
.join(path
),
406 # Bug 1560193: The JS library browsertime uses to execute commands
407 # (execa) will muck up the PATH variable and put the directory that
408 # node is in first in path. If this is globally-installed node,
409 # that means `/usr/bin` will be inserted first which means that we
410 # will get `/usr/bin/python` for `python`.
412 # Our fork of browsertime supports a `PYTHON` environment variable
413 # that points to the exact python executable to use.
414 "PYTHON": command_context
.virtualenv_manager
.python_path
,
420 def _need_install(command_context
, package
):
421 from pip
._internal
.req
.constructors
import install_req_from_line
423 req
= install_req_from_line(package
)
424 req
.check_if_exists(use_user_site
=False)
425 if req
.satisfied_by
is None:
427 venv_site_lib
= os
.path
.abspath(
428 os
.path
.join(command_context
.virtualenv_manager
.bin_path
, "..", "lib")
430 site_packages
= os
.path
.abspath(req
.satisfied_by
.location
)
431 return not site_packages
.startswith(venv_site_lib
)
434 def activate_browsertime_virtualenv(command_context
, *args
, **kwargs
):
435 r
"""Activates virtualenv.
437 This function will also install Pillow and pyssim if needed.
438 It will raise an error in case the install failed.
440 # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990
441 MachCommandBase
.activate_virtualenv(command_context
, *args
, **kwargs
)
443 # installing Python deps on the fly
445 "Pillow==%s" % PILLOW_VERSION
,
446 "pyssim==%s" % PYSSIM_VERSION
,
447 "scipy==%s" % SCIPY_VERSION
,
448 "numpy==%s" % NUMPY_VERSION
,
449 "opencv-python==%s" % OPENCV_VERSION
,
451 if _need_install(command_context
, dep
):
452 subprocess
.check_call(
454 command_context
.virtualenv_manager
.python_path
,
463 def check(command_context
):
464 r
"""Run `visualmetrics.py --check`."""
465 command_context
.activate_virtualenv()
468 status
= command_context
.run_process(
469 [command_context
.virtualenv_manager
.python_path
, visualmetrics_path()] + args
,
470 # For --check, don't allow user's path to interfere with path testing except on Linux
471 append_env
=append_env(
472 command_context
, append_path
=host_platform().startswith("linux")
475 ensure_exit_code
=False, # Don't throw on non-zero exit code.
476 cwd
=mozpath
.join(command_context
.topsrcdir
),
485 # Avoid logging the command (and, on Windows, the environment).
486 command_context
.log_manager
.terminal_handler
.setLevel(logging
.CRITICAL
)
487 print("browsertime version:", end
=" ")
492 return node(command_context
, [browsertime_path()] + ["--version"])
495 def extra_default_args(command_context
, args
=[]):
496 # Add Mozilla-specific default arguments. This is tricky because browsertime is quite
497 # loose about arguments; repeat arguments are generally accepted but then produce
498 # difficult to interpret type errors.
500 def extract_browser_name(args
):
501 "Extracts the browser name if any"
502 # These are BT arguments, it's BT job to check them
503 # here we just want to extract the browser name
504 res
= re
.findall(r
"(--browser|-b)[= ]([\w]+)", " ".join(args
))
509 def matches(args
, *flags
):
510 "Return True if any argument matches any of the given flags (maybe with an argument)."
512 if flag
in args
or any(arg
.startswith(flag
+ "=") for arg
in args
):
518 # Default to Firefox. Override with `-b ...` or `--browser=...`.
519 specifies_browser
= matches(args
, "-b", "--browser")
520 if not specifies_browser
:
521 extra_args
.extend(("-b", "firefox"))
523 # Default to not collect HAR. Override with `--skipHar=false`.
524 specifies_har
= matches(args
, "--har", "--skipHar", "--gzipHar")
525 if not specifies_har
:
526 extra_args
.append("--skipHar")
528 if not matches(args
, "--android"):
529 # If --firefox.binaryPath is not specified, default to the objdir binary
530 # Note: --firefox.release is not a real browsertime option, but it will
531 # silently ignore it instead and default to a release installation.
532 specifies_binaryPath
= matches(
534 "--firefox.binaryPath",
538 "--firefox.developer",
541 if not specifies_binaryPath
:
542 specifies_binaryPath
= extract_browser_name(args
) == "chrome"
544 if not specifies_binaryPath
:
547 ("--firefox.binaryPath", command_context
.get_binary_path())
549 except BinaryNotFoundException
as e
:
560 "Please run |./mach build| "
561 "or specify a Firefox binary with --firefox.binaryPath.",
569 {"extra_args": extra_args
},
570 "Running browsertime with extra default arguments: {extra_args}",
576 def _verify_node_install(command_context
):
577 # check if Node is installed
578 sys
.path
.append(mozpath
.join(command_context
.topsrcdir
, "tools", "lint", "eslint"))
582 node_valid
= setup_helper
.check_node_executables_valid()
584 print("Can't find Node. did you run ./mach bootstrap ?")
587 # check if the browsertime package has been deployed correctly
588 # for this we just check for the browsertime directory presence
589 if not os
.path
.exists(browsertime_path()):
590 print("Could not find browsertime.js, try ./mach browsertime --setup")
591 print("If that still fails, try ./mach browsertime --setup --clobber")
600 description
="Run [browsertime](https://github.com/sitespeedio/browsertime) "
601 "performance tests.",
606 help="Verbose output for what commands the build is running.",
608 @CommandArgument("--update-upstream-url", default
="")
609 @CommandArgument("--setup", default
=False, action
="store_true")
610 @CommandArgument("--clobber", default
=False, action
="store_true")
615 help="Skip all local caches to force re-fetching remote artifacts.",
617 @CommandArgument("--check-browsertime", default
=False, action
="store_true")
619 "--install-vismet-reqs",
622 help="Add this flag to get the visual metrics requirements installed.",
625 "--browsertime-help",
628 help="Show the browsertime help message.",
630 @CommandArgument("args", nargs
=argparse
.REMAINDER
)
635 update_upstream_url
="",
639 check_browsertime
=False,
640 browsertime_help
=False,
641 install_vismet_reqs
=False,
643 command_context
._set
_log
_level
(verbose
)
645 # Output a message before going further to make sure the
646 # user knows that this tool is unsupported by the perftest
647 # team and point them to our supported tools. Pause a bit to
648 # make sure the user sees this message.
653 "[INFO] This command should be used for browsertime setup only.\n"
654 "If you are looking to run performance tests on your patch, use "
655 "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics "
656 "by using the --browsertime-video and --browsertime-visualmetrics. "
657 "Here is a sample command for raptor-browsertime: \n\t`./mach raptor "
658 "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n"
659 "See this wiki page for more information if needed: "
660 "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n",
664 if update_upstream_url
:
665 return setup_browsertime(
667 new_upstream_url
=update_upstream_url
,
668 install_vismet_reqs
=install_vismet_reqs
,
671 return setup_browsertime(
673 should_clobber
=clobber
,
674 install_vismet_reqs
=install_vismet_reqs
,
677 if not _verify_node_install(command_context
):
680 if check_browsertime
:
681 return check(command_context
)
684 args
.append("--help")
686 activate_browsertime_virtualenv(command_context
)
687 default_args
= extra_default_args(command_context
, args
)
688 if default_args
== 1:
690 return node(command_context
, [browsertime_path()] + default_args
+ args
)