testing/raptor/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Originally taken from /talos/mach_commands.py
   6
   7 # Integrates raptor mozharness with mach
   8
   9 import json
  10 import logging
  11 import os
  12 import socket
  13 import subprocess
  14 import sys
  15
  16 from mach.decorators import Command
  17 from mach.util import get_state_dir
  18 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  19 from mozbuild.base import MachCommandConditions as Conditions
  20
  21 HERE = os.path.dirname(os.path.realpath(__file__))
  22
  23 ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m", "cstm-car-m"]
  24
  25
  26 class RaptorRunner(MozbuildObject):
  27     def run_test(self, raptor_args, kwargs):
  28         """Setup and run mozharness.
  29
  30         We want to do a few things before running Raptor:
  31
  32         1. Clone mozharness
  33         2. Make the config for Raptor mozharness
  34         3. Run mozharness
  35         """
  36         # Validate that the user is using a supported python version before doing anything else
  37         max_py_major, max_py_minor = 3, 11
  38         sys_maj, sys_min = sys.version_info.major, sys.version_info.minor
  39         if sys_min > max_py_minor:
  40             raise PythonVersionException(
  41                 print(
  42                     f"\tPlease downgrade your Python version as Raptor does not yet support Python "
  43                     f"versions greater than {max_py_major}.{max_py_minor}."
  44                     f"\n\tYou seem to currently be using Python {sys_maj}.{sys_min}."
  45                     f"\n\tSee here for a possible solution in debugging your python environment: "
  46                     f"https://firefox-source-docs.mozilla.org/testing/perfdocs/"
  47                     f"debugging.html#debugging-local-python-environment"
  48                 )
  49             )
  50         self.init_variables(raptor_args, kwargs)
  51         self.make_config()
  52         self.write_config()
  53         self.make_args()
  54         return self.run_mozharness()
  55
  56     def init_variables(self, raptor_args, kwargs):
  57         self.raptor_args = raptor_args
  58
  59         if kwargs.get("host") == "HOST_IP":
  60             kwargs["host"] = os.environ["HOST_IP"]
  61         self.host = kwargs["host"]
  62         self.is_release_build = kwargs["is_release_build"]
  63         self.live_sites = kwargs["live_sites"]
  64         self.disable_perf_tuning = kwargs["disable_perf_tuning"]
  65         self.conditioned_profile = kwargs["conditioned_profile"]
  66         self.device_name = kwargs["device_name"]
  67         self.enable_marionette_trace = kwargs["enable_marionette_trace"]
  68         self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
  69         self.browsertime_node = kwargs["browsertime_node"]
  70         self.clean = kwargs["clean"]
  71         self.screenshot_on_failure = kwargs["screenshot_on_failure"]
  72
  73         if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
  74             self.binary_path = None
  75         else:
  76             self.binary_path = kwargs.get("binary") or self.get_binary_path()
  77
  78         self.python = sys.executable
  79
  80         self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor")
  81         self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness")
  82         self.config_file_path = os.path.join(
  83             self._topobjdir, "testing", "raptor-in_tree_conf.json"
  84         )
  85
  86         self.virtualenv_script = os.path.join(
  87             self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
  88         )
  89         self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv")
  90
  91     def make_config(self):
  92         default_actions = [
  93             "populate-webroot",
  94             "create-virtualenv",
  95             "install-chromium-distribution",
  96             "run-tests",
  97         ]
  98         self.config = {
  99             "run_local": True,
 100             "binary_path": self.binary_path,
 101             "repo_path": self.topsrcdir,
 102             "raptor_path": self.raptor_dir,
 103             "obj_path": self.topobjdir,
 104             "log_name": "raptor",
 105             "virtualenv_path": self.virtualenv_path,
 106             "pypi_url": "http://pypi.org/simple",
 107             "base_work_dir": self.mozharness_dir,
 108             "exes": {
 109                 "python": self.python,
 110                 "virtualenv": [self.python, self.virtualenv_script],
 111             },
 112             "title": socket.gethostname(),
 113             "default_actions": default_actions,
 114             "raptor_cmd_line_args": self.raptor_args,
 115             "host": self.host,
 116             "live_sites": self.live_sites,
 117             "disable_perf_tuning": self.disable_perf_tuning,
 118             "conditioned_profile": self.conditioned_profile,
 119             "is_release_build": self.is_release_build,
 120             "device_name": self.device_name,
 121             "enable_marionette_trace": self.enable_marionette_trace,
 122             "browsertime_visualmetrics": self.browsertime_visualmetrics,
 123             "browsertime_node": self.browsertime_node,
 124             "mozbuild_path": get_state_dir(),
 125             "clean": self.clean,
 126             "screenshot_on_failure": self.screenshot_on_failure,
 127         }
 128
 129         sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime"))
 130         try:
 131             import platform
 132
 133             import mach_commands as browsertime
 134
 135             # We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by
 136             # browsertime in its `node_modules` directory, which is appropriate for local builds.
 137             # We don't set `browsertime_ffmpeg` yet: it will need to be on the path.  There is code
 138             # to configure the environment including the path in
 139             # `tools/browsertime/mach_commands.py` but integrating it here will take more effort.
 140             self.config.update(
 141                 {
 142                     "browsertime_browsertimejs": browsertime.browsertime_path(),
 143                     "browsertime_vismet_script": browsertime.visualmetrics_path(),
 144                 }
 145             )
 146
 147             def _get_browsertime_package():
 148                 with open(
 149                     os.path.join(
 150                         self.topsrcdir,
 151                         "tools",
 152                         "browsertime",
 153                         "node_modules",
 154                         "browsertime",
 155                         "package.json",
 156                     )
 157                 ) as package:
 158                     return json.load(package)
 159
 160             def _get_browsertime_resolved():
 161                 try:
 162                     with open(
 163                         os.path.join(
 164                             self.topsrcdir,
 165                             "tools",
 166                             "browsertime",
 167                             "node_modules",
 168                             ".package-lock.json",
 169                         )
 170                     ) as package_lock:
 171                         return json.load(package_lock)["packages"][
 172                             "node_modules/browsertime"
 173                         ]["resolved"]
 174                 except FileNotFoundError:
 175                     # Older versions of node/npm add this metadata to package.json
 176                     return _get_browsertime_package()["_from"]
 177
 178             def _should_install():
 179                 # If ffmpeg doesn't exist in the .mozbuild directory,
 180                 # then we should install
 181                 btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime")
 182                 if not os.path.exists(btime_cache) or not any(
 183                     ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)]
 184                 ):
 185                     return True
 186
 187                 # If browsertime doesn't exist, install it
 188                 if not os.path.exists(
 189                     self.config["browsertime_browsertimejs"]
 190                 ) or not os.path.exists(self.config["browsertime_vismet_script"]):
 191                     return True
 192
 193                 # Browsertime exists, check if it's outdated
 194                 with open(
 195                     os.path.join(self.topsrcdir, "tools", "browsertime", "package.json")
 196                 ) as new:
 197                     new_pkg = json.load(new)
 198
 199                 return not _get_browsertime_resolved().endswith(
 200                     new_pkg["devDependencies"]["browsertime"]
 201                 )
 202
 203             def _get_browsertime_version():
 204                 # Returns the (version number, current commit) used
 205                 return (
 206                     _get_browsertime_package()["version"],
 207                     _get_browsertime_resolved(),
 208                 )
 209
 210             # Check if browsertime scripts exist and try to install them if
 211             # they aren't
 212             if _should_install():
 213                 # TODO: Make this "integration" nicer in the near future
 214                 print("Missing browsertime files...attempting to install")
 215                 subprocess.check_output(
 216                     [
 217                         os.path.join(self.topsrcdir, "mach"),
 218                         "browsertime",
 219                         "--setup",
 220                         "--clobber",
 221                     ],
 222                     shell="windows" in platform.system().lower(),
 223                 )
 224                 if _should_install():
 225                     raise Exception(
 226                         "Failed installation attempt. Cannot find browsertime scripts. "
 227                         "Run `./mach browsertime --setup --clobber` to set it up."
 228                     )
 229
 230                 # Bug 1766112 - For the time being, we need to trigger a
 231                 # clean build to upgrade browsertime. This should be disabled
 232                 # after some time.
 233                 print(
 234                     "Setting --clean to True to rebuild Python "
 235                     "environment for Browsertime upgrade..."
 236                 )
 237                 self.config["clean"] = True
 238
 239             print("Using browsertime version %s from %s" % _get_browsertime_version())
 240
 241         finally:
 242             sys.path = sys.path[1:]
 243
 244     def make_args(self):
 245         self.args = {
 246             "config": {},
 247             "initial_config_file": self.config_file_path,
 248         }
 249
 250     def write_config(self):
 251         try:
 252             config_file = open(self.config_file_path, "w")
 253             config_file.write(json.dumps(self.config))
 254             config_file.close()
 255         except IOError as e:
 256             err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
 257             print(err_str.format(self.config_file_path, str(e)))
 258             raise e
 259
 260     def run_mozharness(self):
 261         sys.path.insert(0, self.mozharness_dir)
 262         from mozharness.mozilla.testing.raptor import Raptor
 263
 264         raptor_mh = Raptor(
 265             config=self.args["config"],
 266             initial_config_file=self.args["initial_config_file"],
 267         )
 268         return raptor_mh.run()
 269
 270
 271 def setup_node(command_context):
 272     """Fetch the latest node-18 binary and install it into the .mozbuild directory."""
 273     import platform
 274
 275     from mozbuild.artifact_commands import artifact_toolchain
 276     from mozbuild.nodeutil import find_node_executable
 277     from packaging.version import Version
 278
 279     print("Setting up node for browsertime...")
 280     state_dir = get_state_dir()
 281     cache_path = os.path.join(state_dir, "browsertime", "node-18")
 282
 283     def __check_for_node():
 284         # Check standard locations first
 285         node_exe = find_node_executable(min_version=Version("18.0.0"))
 286         if node_exe and (node_exe[0] is not None):
 287             return node_exe[0]
 288         if not os.path.exists(cache_path):
 289             return None
 290
 291         # Check the browsertime-specific node location next
 292         node_name = "node"
 293         if platform.system() == "Windows":
 294             node_name = "node.exe"
 295             node_exe_path = os.path.join(
 296                 state_dir,
 297                 "browsertime",
 298                 "node-18",
 299                 "node",
 300             )
 301         else:
 302             node_exe_path = os.path.join(
 303                 state_dir,
 304                 "browsertime",
 305                 "node-18",
 306                 "node",
 307                 "bin",
 308             )
 309
 310         node_exe = os.path.join(node_exe_path, node_name)
 311         if not os.path.exists(node_exe):
 312             return None
 313
 314         return node_exe
 315
 316     node_exe = __check_for_node()
 317     if node_exe is None:
 318         toolchain_job = "{}-node-18"
 319         plat = platform.system()
 320         if plat == "Windows":
 321             toolchain_job = toolchain_job.format("win64")
 322         elif plat == "Darwin":
 323             if platform.processor() == "arm":
 324                 toolchain_job = toolchain_job.format("macosx64-aarch64")
 325             else:
 326                 toolchain_job = toolchain_job.format("macosx64")
 327         else:
 328             toolchain_job = toolchain_job.format("linux64")
 329
 330         print(
 331             "Downloading Node v18 from Taskcluster toolchain {}...".format(
 332                 toolchain_job
 333             )
 334         )
 335
 336         if not os.path.exists(cache_path):
 337             os.makedirs(cache_path, exist_ok=True)
 338
 339         # Change directories to where node should be installed
 340         # before installing. Otherwise, it gets installed in the
 341         # top level of the repo (or the current working directory).
 342         cur_dir = os.getcwd()
 343         os.chdir(cache_path)
 344         artifact_toolchain(
 345             command_context,
 346             verbose=False,
 347             from_build=[toolchain_job],
 348             no_unpack=False,
 349             retry=0,
 350             cache_dir=cache_path,
 351         )
 352         os.chdir(cur_dir)
 353
 354         node_exe = __check_for_node()
 355         if node_exe is None:
 356             raise Exception("Could not find Node v18 binary for Raptor-Browsertime")
 357
 358         print("Finished downloading Node v18 from Taskcluster")
 359
 360     print("Node v18+ found at: %s" % node_exe)
 361     return node_exe
 362
 363
 364 def create_parser():
 365     sys.path.insert(0, HERE)  # allow to import the raptor package
 366     from raptor.cmdline import create_parser
 367
 368     return create_parser(mach_interface=True)
 369
 370
 371 @Command(
 372     "raptor",
 373     category="testing",
 374     description="Run Raptor performance tests.",
 375     parser=create_parser,
 376 )
 377 def run_raptor(command_context, **kwargs):
 378     build_obj = command_context
 379
 380     # Setup node for browsertime
 381     kwargs["browsertime_node"] = setup_node(command_context)
 382
 383     is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS
 384
 385     if is_android:
 386         from mozrunner.devices.android_device import (
 387             InstallIntent,
 388             verify_android_device,
 389         )
 390
 391         install = (
 392             InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES
 393         )
 394         verbose = False
 395         if (
 396             kwargs.get("log_mach_verbose")
 397             or kwargs.get("log_tbpl_level") == "debug"
 398             or kwargs.get("log_mach_level") == "debug"
 399             or kwargs.get("log_raw_level") == "debug"
 400         ):
 401             verbose = True
 402         if not verify_android_device(
 403             build_obj,
 404             install=install,
 405             app=kwargs["binary"],
 406             verbose=verbose,
 407             xre=True,
 408         ):  # Equivalent to 'run_local' = True.
 409             print(
 410                 "****************************************************************************"
 411             )
 412             print(
 413                 "Unable to verify device, please check your attached/connected android device"
 414             )
 415             print(
 416                 "****************************************************************************"
 417             )
 418             return 1
 419         # Disable fission until geckoview supports fission by default.
 420         # Need fission on Android? Use '--setpref fission.autostart=true'
 421         kwargs["fission"] = False
 422
 423     # Remove mach global arguments from sys.argv to prevent them
 424     # from being consumed by raptor. Treat any item in sys.argv
 425     # occuring before "raptor" as a mach global argument.
 426     argv = []
 427     in_mach = True
 428     for arg in sys.argv:
 429         if not in_mach:
 430             argv.append(arg)
 431         if arg.startswith("raptor"):
 432             in_mach = False
 433
 434     raptor = command_context._spawn(RaptorRunner)
 435
 436     try:
 437         return raptor.run_test(argv, kwargs)
 438     except BinaryNotFoundException as e:
 439         command_context.log(
 440             logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}"
 441         )
 442         command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
 443         return 1
 444     except Exception as e:
 445         print(repr(e))
 446         return 1
 447
 448
 449 @Command(
 450     "raptor-test",
 451     category="testing",
 452     description="Run Raptor performance tests.",
 453     parser=create_parser,
 454 )
 455 def run_raptor_test(command_context, **kwargs):
 456     return run_raptor(command_context, **kwargs)
 457
 458
 459 class PythonVersionException(Exception):
 460     pass