Backed out changeset 2bbc01486e2f (bug 1910796) for causing multiple failures. CLOSED...
[gecko.git] / tools / browsertime / mach_commands.py
bloba75a4d9b5d07cbfdeb59b4c6b2e61c4ca85155ff
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 r"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime).
7 Browsertime is a harness for running performance tests, similar to
8 Mozilla's Raptor testing framework. Browsertime is written in Node.js
9 and uses Selenium WebDriver to drive multiple browsers including
10 Chrome, Chrome for Android, Firefox, and (pending the resolution of
11 [Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126)
12 and similar tickets) Firefox for Android and GeckoView-based vehicles.
14 Right now a custom version of browsertime and the underlying
15 geckodriver binary are needed to support GeckoView-based vehicles;
16 this module accommodates those in-progress custom versions.
18 To get started, run
19 ```
20 ./mach browsertime --setup [--clobber]
21 ```
22 This will populate `tools/browsertime/node_modules`.
24 To invoke browsertime, run
25 ```
26 ./mach browsertime [ARGS]
27 ```
28 All arguments are passed through to browsertime.
29 """
31 import argparse
32 import collections
33 import contextlib
34 import json
35 import logging
36 import os
37 import platform
38 import re
39 import stat
40 import subprocess
41 import sys
42 import time
44 import mozpack.path as mozpath
45 from mach.decorators import Command, CommandArgument
46 from mozbuild.base import BinaryNotFoundException, MachCommandBase
47 from mozbuild.dirutils import mkdir
48 from six import StringIO
50 AUTOMATION = "MOZ_AUTOMATION" in os.environ
51 BROWSERTIME_ROOT = os.path.dirname(__file__)
53 PILLOW_VERSION = "8.4.0" # version 8.4.0 currently supports python 3.6 to 3.10
54 PYSSIM_VERSION = "0.4"
55 SCIPY_VERSION = "1.2.3"
56 NUMPY_VERSION = "1.16.1"
57 OPENCV_VERSION = "4.5.4.60"
59 py3_minor = sys.version_info.minor
60 if py3_minor > 7:
61 SCIPY_VERSION = "1.9.3"
62 NUMPY_VERSION = "1.23.5"
63 PILLOW_VERSION = "9.2.0"
64 OPENCV_VERSION = "4.6.0.66"
66 MIN_NODE_VERSION = "16.0.0"
68 IS_APPLE_SILICON = sys.platform.startswith(
69 "darwin"
70 ) and platform.processor().startswith("arm")
73 @contextlib.contextmanager
74 def silence():
75 oldout, olderr = sys.stdout, sys.stderr
76 try:
77 sys.stdout, sys.stderr = StringIO(), StringIO()
78 yield
79 finally:
80 sys.stdout, sys.stderr = oldout, olderr
83 def node_path(command_context):
84 import platform
86 from mozbuild.nodeutil import find_node_executable
87 from packaging.version import Version
89 state_dir = command_context._mach_context.state_dir
90 cache_path = os.path.join(state_dir, "browsertime", "node-16")
92 NODE_FAILURE_MSG = (
93 "Could not locate a node binary that is at least version {}. ".format(
94 MIN_NODE_VERSION
96 + "Please run `./mach raptor --browsertime -t amazon` to install it "
97 + "from the Taskcluster Toolchain artifacts."
100 # Check standard locations first
101 node_exe = find_node_executable(min_version=Version(MIN_NODE_VERSION))
102 if node_exe and (node_exe[0] is not None):
103 return os.path.abspath(node_exe[0])
104 if not os.path.exists(cache_path):
105 raise Exception(NODE_FAILURE_MSG)
107 # Check the browsertime-specific node location next
108 node_name = "node"
109 if platform.system() == "Windows":
110 node_name = "node.exe"
111 node_exe_path = os.path.join(
112 state_dir,
113 "browsertime",
114 "node-16",
115 "node",
117 else:
118 node_exe_path = os.path.join(
119 state_dir,
120 "browsertime",
121 "node-16",
122 "node",
123 "bin",
126 node_exe = os.path.join(node_exe_path, node_name)
127 if not os.path.exists(node_exe):
128 raise Exception(NODE_FAILURE_MSG)
130 return os.path.abspath(node_exe)
133 def package_path():
134 """The path to the `browsertime` directory.
136 Override the default with the `BROWSERTIME` environment variable."""
137 override = os.environ.get("BROWSERTIME", None)
138 if override:
139 return override
141 return mozpath.join(BROWSERTIME_ROOT, "node_modules", "browsertime")
144 def browsertime_path():
145 """The path to the `browsertime.js` script."""
146 # On Windows, invoking `node_modules/.bin/browsertime{.cmd}`
147 # doesn't work when invoked as an argument to our specific
148 # binary. Since we want our version of node, invoke the
149 # actual script directly.
150 return mozpath.join(package_path(), "bin", "browsertime.js")
153 def visualmetrics_path():
154 """The path to the `visualmetrics.py` script."""
155 return mozpath.join(package_path(), "visualmetrics", "visualmetrics-portable.py")
158 def host_platform():
159 is_64bits = sys.maxsize > 2**32
161 if sys.platform.startswith("win"):
162 if is_64bits:
163 return "win64"
164 elif sys.platform.startswith("linux"):
165 if is_64bits:
166 return "linux64"
167 elif sys.platform.startswith("darwin"):
168 return "darwin"
170 raise ValueError("sys.platform is not yet supported: {}".format(sys.platform))
173 # Map from `host_platform()` to a `fetch`-like syntax.
174 host_fetches = {
175 "darwin": {
176 "ffmpeg": {
177 "type": "static-url",
178 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip", # noqa
179 # An extension to `fetch` syntax.
180 "path": "ffmpeg-macos",
183 "linux64": {
184 "ffmpeg": {
185 "type": "static-url",
186 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz", # noqa
187 # An extension to `fetch` syntax.
188 "path": "ffmpeg-4.4.1-i686-static",
191 "win64": {
192 "ffmpeg": {
193 "type": "static-url",
194 "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip", # noqa
195 # An extension to `fetch` syntax.
196 "path": "ffmpeg-4.4.1-full_build",
202 def artifact_cache_path(command_context):
203 r"""Downloaded artifacts will be kept here."""
204 # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
205 return mozpath.join(command_context._mach_context.state_dir, "cache", "browsertime")
208 def state_path(command_context):
209 r"""Unpacked artifacts will be kept here."""
210 # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
211 return mozpath.join(command_context._mach_context.state_dir, "browsertime")
214 def setup_prerequisites(command_context):
215 r"""Install browsertime and visualmetrics.py prerequisites."""
217 from mozbuild.action.tooltool import unpack_file
218 from mozbuild.artifact_cache import ArtifactCache
220 # Download the visualmetrics-portable.py requirements.
221 artifact_cache = ArtifactCache(
222 artifact_cache_path(command_context),
223 log=command_context.log,
224 skip_cache=False,
227 fetches = host_fetches[host_platform()]
228 for tool, fetch in sorted(fetches.items()):
229 archive = artifact_cache.fetch(fetch["url"])
230 # TODO: assert type, verify sha256 (and size?).
232 if fetch.get("unpack", True):
233 cwd = os.getcwd()
234 try:
235 mkdir(state_path(command_context))
236 os.chdir(state_path(command_context))
237 command_context.log(
238 logging.INFO,
239 "browsertime",
240 {"path": archive},
241 "Unpacking temporary location {path}",
244 unpack_file(archive)
246 # Make sure the expected path exists after extraction
247 path = os.path.join(state_path(command_context), fetch.get("path"))
248 if not os.path.exists(path):
249 raise Exception("Cannot find an extracted directory: %s" % path)
251 try:
252 # Some archives provide binaries that don't have the
253 # executable bit set so we need to set it here
254 for root, dirs, files in os.walk(path):
255 for edir in dirs:
256 loc_to_change = os.path.join(root, edir)
257 st = os.stat(loc_to_change)
258 os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
259 for efile in files:
260 loc_to_change = os.path.join(root, efile)
261 st = os.stat(loc_to_change)
262 os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC)
263 except Exception as e:
264 raise Exception(
265 "Could not set executable bit in %s, error: %s" % (path, str(e))
267 finally:
268 os.chdir(cwd)
271 def setup_browsertime(
272 command_context,
273 should_clobber=False,
274 new_upstream_url="",
275 install_vismet_reqs=False,
277 r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
279 sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
280 import setup_helper
282 if not new_upstream_url:
283 setup_prerequisites(command_context)
285 if new_upstream_url:
286 package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json")
288 command_context.log(
289 logging.INFO,
290 "browsertime",
292 "new_upstream_url": new_upstream_url,
293 "package_json_path": package_json_path,
295 "Updating browsertime node module version in {package_json_path} "
296 "to {new_upstream_url}",
299 if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url):
300 raise ValueError(
301 "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'"
302 % new_upstream_url
305 with open(package_json_path) as f:
306 existing_body = json.loads(
307 f.read(), object_pairs_hook=collections.OrderedDict
310 existing_body["devDependencies"]["browsertime"] = new_upstream_url
312 updated_body = json.dumps(existing_body, indent=2)
314 with open(package_json_path, "w") as f:
315 f.write(updated_body)
317 # Install the browsertime Node.js requirements.
318 if not setup_helper.check_node_executables_valid():
319 return 1
321 # To use a custom `geckodriver`, set
322 # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
323 # to an endpoint with binaries named like
324 # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
325 if AUTOMATION:
326 os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
327 os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
329 if install_vismet_reqs:
330 # Hide this behind a flag so we don't install them by default in Raptor
331 command_context.log(
332 logging.INFO, "browsertime", {}, "Installing python requirements"
334 activate_browsertime_virtualenv(command_context)
336 command_context.log(
337 logging.INFO,
338 "browsertime",
339 {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")},
340 "Installing browsertime node module from {package_json}",
343 # Add the mozbuild Node binary path to the OS environment in Apple Silicon.
344 # During the browesertime installation, it seems installation of sitespeed.io
345 # sub dependencies look for a global Node rather than the mozbuild Node binary.
346 # Normally `--scripts-prepend-node-path` should prevent this but it seems to still
347 # look for a system Node in the environment. This logic ensures the same Node is used.
348 node_dir = os.path.dirname(node_path(command_context))
349 if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]:
350 os.environ["PATH"] += os.pathsep + node_dir
352 status = setup_helper.package_setup(
353 BROWSERTIME_ROOT,
354 "browsertime",
355 should_update=new_upstream_url != "",
356 should_clobber=should_clobber,
357 no_optional=new_upstream_url or AUTOMATION,
360 if status:
361 return status
362 if new_upstream_url or AUTOMATION:
363 return 0
364 if install_vismet_reqs:
365 return check(command_context)
367 return 0
370 def node(command_context, args):
371 r"""Invoke node (interactively) with the given arguments."""
372 return command_context.run_process(
373 [node_path(command_context)] + args,
374 append_env=append_env(command_context),
375 pass_thru=True, # Allow user to run Node interactively.
376 ensure_exit_code=False, # Don't throw on non-zero exit code.
377 cwd=mozpath.join(command_context.topsrcdir),
381 def append_env(command_context, append_path=True):
382 fetches = host_fetches[host_platform()]
384 # Ensure that `ffmpeg` is found and added to the environment
385 path = os.environ.get("PATH", "").split(os.pathsep) if append_path else []
386 path_to_ffmpeg = mozpath.join(
387 state_path(command_context), fetches["ffmpeg"]["path"]
390 path.insert(
392 path_to_ffmpeg
393 if host_platform().startswith("linux")
394 else mozpath.join(path_to_ffmpeg, "bin"),
395 ) # noqa
397 # Ensure that bare `node` and `npm` in scripts, including post-install
398 # scripts, finds the binary we're invoking with. Without this, it's
399 # easy for compiled extensions to get mismatched versions of the Node.js
400 # extension API.
401 node_dir = os.path.dirname(node_path(command_context))
402 path = [node_dir] + path
404 append_env = {
405 "PATH": os.pathsep.join(path),
406 # Bug 1560193: The JS library browsertime uses to execute commands
407 # (execa) will muck up the PATH variable and put the directory that
408 # node is in first in path. If this is globally-installed node,
409 # that means `/usr/bin` will be inserted first which means that we
410 # will get `/usr/bin/python` for `python`.
412 # Our fork of browsertime supports a `PYTHON` environment variable
413 # that points to the exact python executable to use.
414 "PYTHON": command_context.virtualenv_manager.python_path,
417 return append_env
420 def _need_install(command_context, package):
421 from pip._internal.req.constructors import install_req_from_line
423 req = install_req_from_line(package)
424 req.check_if_exists(use_user_site=False)
425 if req.satisfied_by is None:
426 return True
427 venv_site_lib = os.path.abspath(
428 os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib")
430 site_packages = os.path.abspath(req.satisfied_by.location)
431 return not site_packages.startswith(venv_site_lib)
434 def activate_browsertime_virtualenv(command_context, *args, **kwargs):
435 r"""Activates virtualenv.
437 This function will also install Pillow and pyssim if needed.
438 It will raise an error in case the install failed.
440 # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990
441 MachCommandBase.activate_virtualenv(command_context, *args, **kwargs)
443 # installing Python deps on the fly
444 for dep in (
445 "Pillow==%s" % PILLOW_VERSION,
446 "pyssim==%s" % PYSSIM_VERSION,
447 "scipy==%s" % SCIPY_VERSION,
448 "numpy==%s" % NUMPY_VERSION,
449 "opencv-python==%s" % OPENCV_VERSION,
451 if _need_install(command_context, dep):
452 subprocess.check_call(
454 command_context.virtualenv_manager.python_path,
455 "-m",
456 "pip",
457 "install",
458 dep,
463 def check(command_context):
464 r"""Run `visualmetrics.py --check`."""
465 command_context.activate_virtualenv()
467 args = ["--check"]
468 status = command_context.run_process(
469 [command_context.virtualenv_manager.python_path, visualmetrics_path()] + args,
470 # For --check, don't allow user's path to interfere with path testing except on Linux
471 append_env=append_env(
472 command_context, append_path=host_platform().startswith("linux")
474 pass_thru=True,
475 ensure_exit_code=False, # Don't throw on non-zero exit code.
476 cwd=mozpath.join(command_context.topsrcdir),
479 sys.stdout.flush()
480 sys.stderr.flush()
482 if status:
483 return status
485 # Avoid logging the command (and, on Windows, the environment).
486 command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
487 print("browsertime version:", end=" ")
489 sys.stdout.flush()
490 sys.stderr.flush()
492 return node(command_context, [browsertime_path()] + ["--version"])
495 def extra_default_args(command_context, args=[]):
496 # Add Mozilla-specific default arguments. This is tricky because browsertime is quite
497 # loose about arguments; repeat arguments are generally accepted but then produce
498 # difficult to interpret type errors.
500 def extract_browser_name(args):
501 "Extracts the browser name if any"
502 # These are BT arguments, it's BT job to check them
503 # here we just want to extract the browser name
504 res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args))
505 if res == []:
506 return None
507 return res[0][-1]
509 def matches(args, *flags):
510 "Return True if any argument matches any of the given flags (maybe with an argument)."
511 for flag in flags:
512 if flag in args or any(arg.startswith(flag + "=") for arg in args):
513 return True
514 return False
516 extra_args = []
518 # Default to Firefox. Override with `-b ...` or `--browser=...`.
519 specifies_browser = matches(args, "-b", "--browser")
520 if not specifies_browser:
521 extra_args.extend(("-b", "firefox"))
523 # Default to not collect HAR. Override with `--skipHar=false`.
524 specifies_har = matches(args, "--har", "--skipHar", "--gzipHar")
525 if not specifies_har:
526 extra_args.append("--skipHar")
528 if not matches(args, "--android"):
529 # If --firefox.binaryPath is not specified, default to the objdir binary
530 # Note: --firefox.release is not a real browsertime option, but it will
531 # silently ignore it instead and default to a release installation.
532 specifies_binaryPath = matches(
533 args,
534 "--firefox.binaryPath",
535 "--firefox.release",
536 "--firefox.nightly",
537 "--firefox.beta",
538 "--firefox.developer",
541 if not specifies_binaryPath:
542 specifies_binaryPath = extract_browser_name(args) == "chrome"
544 if not specifies_binaryPath:
545 try:
546 extra_args.extend(
547 ("--firefox.binaryPath", command_context.get_binary_path())
549 except BinaryNotFoundException as e:
550 command_context.log(
551 logging.ERROR,
552 "browsertime",
553 {"error": str(e)},
554 "ERROR: {error}",
556 command_context.log(
557 logging.INFO,
558 "browsertime",
560 "Please run |./mach build| "
561 "or specify a Firefox binary with --firefox.binaryPath.",
563 return 1
565 if extra_args:
566 command_context.log(
567 logging.DEBUG,
568 "browsertime",
569 {"extra_args": extra_args},
570 "Running browsertime with extra default arguments: {extra_args}",
573 return extra_args
576 def _verify_node_install(command_context):
577 # check if Node is installed
578 sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
579 import setup_helper
581 with silence():
582 node_valid = setup_helper.check_node_executables_valid()
583 if not node_valid:
584 print("Can't find Node. did you run ./mach bootstrap ?")
585 return False
587 # check if the browsertime package has been deployed correctly
588 # for this we just check for the browsertime directory presence
589 if not os.path.exists(browsertime_path()):
590 print("Could not find browsertime.js, try ./mach browsertime --setup")
591 print("If that still fails, try ./mach browsertime --setup --clobber")
592 return False
594 return True
597 @Command(
598 "browsertime",
599 category="testing",
600 description="Run [browsertime](https://github.com/sitespeedio/browsertime) "
601 "performance tests.",
603 @CommandArgument(
604 "--verbose",
605 action="store_true",
606 help="Verbose output for what commands the build is running.",
608 @CommandArgument("--update-upstream-url", default="")
609 @CommandArgument("--setup", default=False, action="store_true")
610 @CommandArgument("--clobber", default=False, action="store_true")
611 @CommandArgument(
612 "--skip-cache",
613 default=False,
614 action="store_true",
615 help="Skip all local caches to force re-fetching remote artifacts.",
617 @CommandArgument("--check-browsertime", default=False, action="store_true")
618 @CommandArgument(
619 "--install-vismet-reqs",
620 default=False,
621 action="store_true",
622 help="Add this flag to get the visual metrics requirements installed.",
624 @CommandArgument(
625 "--browsertime-help",
626 default=False,
627 action="store_true",
628 help="Show the browsertime help message.",
630 @CommandArgument("args", nargs=argparse.REMAINDER)
631 def browsertime(
632 command_context,
633 args,
634 verbose=False,
635 update_upstream_url="",
636 setup=False,
637 clobber=False,
638 skip_cache=False,
639 check_browsertime=False,
640 browsertime_help=False,
641 install_vismet_reqs=False,
643 command_context._set_log_level(verbose)
645 # Output a message before going further to make sure the
646 # user knows that this tool is unsupported by the perftest
647 # team and point them to our supported tools. Pause a bit to
648 # make sure the user sees this message.
649 command_context.log(
650 logging.INFO,
651 "browsertime",
653 "[INFO] This command should be used for browsertime setup only.\n"
654 "If you are looking to run performance tests on your patch, use "
655 "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics "
656 "by using the --browsertime-video and --browsertime-visualmetrics. "
657 "Here is a sample command for raptor-browsertime: \n\t`./mach raptor "
658 "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n"
659 "See this wiki page for more information if needed: "
660 "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n",
662 time.sleep(5)
664 if update_upstream_url:
665 return setup_browsertime(
666 command_context,
667 new_upstream_url=update_upstream_url,
668 install_vismet_reqs=install_vismet_reqs,
670 elif setup:
671 return setup_browsertime(
672 command_context,
673 should_clobber=clobber,
674 install_vismet_reqs=install_vismet_reqs,
676 else:
677 if not _verify_node_install(command_context):
678 return 1
680 if check_browsertime:
681 return check(command_context)
683 if browsertime_help:
684 args.append("--help")
686 activate_browsertime_virtualenv(command_context)
687 default_args = extra_default_args(command_context, args)
688 if default_args == 1:
689 return 1
690 return node(command_context, [browsertime_path()] + default_args + args)