testing/raptor/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Originally taken from /talos/mach_commands.py
   6
   7 # Integrates raptor mozharness with mach
   8
   9 import json
  10 import logging
  11 import os
  12 import socket
  13 import subprocess
  14 import sys
  15
  16 from mach.decorators import Command
  17 from mach.util import get_state_dir
  18 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  19 from mozbuild.base import MachCommandConditions as Conditions
  20
  21 HERE = os.path.dirname(os.path.realpath(__file__))
  22
  23 ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m", "cstm-car-m"]
  24
  25
  26 class RaptorRunner(MozbuildObject):
  27     def run_test(self, raptor_args, kwargs):
  28         """Setup and run mozharness.
  29
  30         We want to do a few things before running Raptor:
  31
  32         1. Clone mozharness
  33         2. Make the config for Raptor mozharness
  34         3. Run mozharness
  35         """
  36         # Validate that the user is using a supported python version before doing anything else
  37         max_py_major, max_py_minor = 3, 11
  38         sys_maj, sys_min = sys.version_info.major, sys.version_info.minor
  39         if sys_min > max_py_minor:
  40             raise PythonVersionException(
  41                 print(
  42                     f"\tPlease downgrade your Python version as Raptor does not yet support Python "
  43                     f"versions greater than {max_py_major}.{max_py_minor}."
  44                     f"\n\tYou seem to currently be using Python {sys_maj}.{sys_min}."
  45                     f"\n\tSee here for a possible solution in debugging your python environment: "
  46                     f"https://firefox-source-docs.mozilla.org/testing/perfdocs/"
  47                     f"debugging.html#debugging-local-python-environment"
  48                 )
  49             )
  50         self.init_variables(raptor_args, kwargs)
  51         self.make_config()
  52         self.write_config()
  53         self.make_args()
  54         return self.run_mozharness()
  55
  56     def init_variables(self, raptor_args, kwargs):
  57         self.raptor_args = raptor_args
  58
  59         if kwargs.get("host") == "HOST_IP":
  60             kwargs["host"] = os.environ["HOST_IP"]
  61         self.host = kwargs["host"]
  62         self.is_release_build = kwargs["is_release_build"]
  63         self.live_sites = kwargs["live_sites"]
  64         self.disable_perf_tuning = kwargs["disable_perf_tuning"]
  65         self.conditioned_profile = kwargs["conditioned_profile"]
  66         self.device_name = kwargs["device_name"]
  67         self.enable_marionette_trace = kwargs["enable_marionette_trace"]
  68         self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
  69         self.browsertime_node = kwargs["browsertime_node"]
  70         self.clean = kwargs["clean"]
  71
  72         if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
  73             self.binary_path = None
  74         else:
  75             self.binary_path = kwargs.get("binary") or self.get_binary_path()
  76
  77         self.python = sys.executable
  78
  79         self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor")
  80         self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness")
  81         self.config_file_path = os.path.join(
  82             self._topobjdir, "testing", "raptor-in_tree_conf.json"
  83         )
  84
  85         self.virtualenv_script = os.path.join(
  86             self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
  87         )
  88         self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv")
  89
  90     def make_config(self):
  91         default_actions = [
  92             "populate-webroot",
  93             "create-virtualenv",
  94             "install-chromium-distribution",
  95             "run-tests",
  96         ]
  97         self.config = {
  98             "run_local": True,
  99             "binary_path": self.binary_path,
 100             "repo_path": self.topsrcdir,
 101             "raptor_path": self.raptor_dir,
 102             "obj_path": self.topobjdir,
 103             "log_name": "raptor",
 104             "virtualenv_path": self.virtualenv_path,
 105             "pypi_url": "http://pypi.org/simple",
 106             "base_work_dir": self.mozharness_dir,
 107             "exes": {
 108                 "python": self.python,
 109                 "virtualenv": [self.python, self.virtualenv_script],
 110             },
 111             "title": socket.gethostname(),
 112             "default_actions": default_actions,
 113             "raptor_cmd_line_args": self.raptor_args,
 114             "host": self.host,
 115             "live_sites": self.live_sites,
 116             "disable_perf_tuning": self.disable_perf_tuning,
 117             "conditioned_profile": self.conditioned_profile,
 118             "is_release_build": self.is_release_build,
 119             "device_name": self.device_name,
 120             "enable_marionette_trace": self.enable_marionette_trace,
 121             "browsertime_visualmetrics": self.browsertime_visualmetrics,
 122             "browsertime_node": self.browsertime_node,
 123             "mozbuild_path": get_state_dir(),
 124             "clean": self.clean,
 125         }
 126
 127         sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime"))
 128         try:
 129             import platform
 130
 131             import mach_commands as browsertime
 132
 133             # We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by
 134             # browsertime in its `node_modules` directory, which is appropriate for local builds.
 135             # We don't set `browsertime_ffmpeg` yet: it will need to be on the path.  There is code
 136             # to configure the environment including the path in
 137             # `tools/browsertime/mach_commands.py` but integrating it here will take more effort.
 138             self.config.update(
 139                 {
 140                     "browsertime_browsertimejs": browsertime.browsertime_path(),
 141                     "browsertime_vismet_script": browsertime.visualmetrics_path(),
 142                 }
 143             )
 144
 145             def _get_browsertime_package():
 146                 with open(
 147                     os.path.join(
 148                         self.topsrcdir,
 149                         "tools",
 150                         "browsertime",
 151                         "node_modules",
 152                         "browsertime",
 153                         "package.json",
 154                     )
 155                 ) as package:
 156                     return json.load(package)
 157
 158             def _get_browsertime_resolved():
 159                 try:
 160                     with open(
 161                         os.path.join(
 162                             self.topsrcdir,
 163                             "tools",
 164                             "browsertime",
 165                             "node_modules",
 166                             ".package-lock.json",
 167                         )
 168                     ) as package_lock:
 169                         return json.load(package_lock)["packages"][
 170                             "node_modules/browsertime"
 171                         ]["resolved"]
 172                 except FileNotFoundError:
 173                     # Older versions of node/npm add this metadata to package.json
 174                     return _get_browsertime_package()["_from"]
 175
 176             def _should_install():
 177                 # If ffmpeg doesn't exist in the .mozbuild directory,
 178                 # then we should install
 179                 btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime")
 180                 if not os.path.exists(btime_cache) or not any(
 181                     ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)]
 182                 ):
 183                     return True
 184
 185                 # If browsertime doesn't exist, install it
 186                 if not os.path.exists(
 187                     self.config["browsertime_browsertimejs"]
 188                 ) or not os.path.exists(self.config["browsertime_vismet_script"]):
 189                     return True
 190
 191                 # Browsertime exists, check if it's outdated
 192                 with open(
 193                     os.path.join(self.topsrcdir, "tools", "browsertime", "package.json")
 194                 ) as new:
 195                     new_pkg = json.load(new)
 196
 197                 return not _get_browsertime_resolved().endswith(
 198                     new_pkg["devDependencies"]["browsertime"]
 199                 )
 200
 201             def _get_browsertime_version():
 202                 # Returns the (version number, current commit) used
 203                 return (
 204                     _get_browsertime_package()["version"],
 205                     _get_browsertime_resolved(),
 206                 )
 207
 208             # Check if browsertime scripts exist and try to install them if
 209             # they aren't
 210             if _should_install():
 211                 # TODO: Make this "integration" nicer in the near future
 212                 print("Missing browsertime files...attempting to install")
 213                 subprocess.check_output(
 214                     [
 215                         os.path.join(self.topsrcdir, "mach"),
 216                         "browsertime",
 217                         "--setup",
 218                         "--clobber",
 219                     ],
 220                     shell="windows" in platform.system().lower(),
 221                 )
 222                 if _should_install():
 223                     raise Exception(
 224                         "Failed installation attempt. Cannot find browsertime scripts. "
 225                         "Run `./mach browsertime --setup --clobber` to set it up."
 226                     )
 227
 228                 # Bug 1766112 - For the time being, we need to trigger a
 229                 # clean build to upgrade browsertime. This should be disabled
 230                 # after some time.
 231                 print(
 232                     "Setting --clean to True to rebuild Python "
 233                     "environment for Browsertime upgrade..."
 234                 )
 235                 self.config["clean"] = True
 236
 237             print("Using browsertime version %s from %s" % _get_browsertime_version())
 238
 239         finally:
 240             sys.path = sys.path[1:]
 241
 242     def make_args(self):
 243         self.args = {
 244             "config": {},
 245             "initial_config_file": self.config_file_path,
 246         }
 247
 248     def write_config(self):
 249         try:
 250             config_file = open(self.config_file_path, "w")
 251             config_file.write(json.dumps(self.config))
 252             config_file.close()
 253         except IOError as e:
 254             err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
 255             print(err_str.format(self.config_file_path, str(e)))
 256             raise e
 257
 258     def run_mozharness(self):
 259         sys.path.insert(0, self.mozharness_dir)
 260         from mozharness.mozilla.testing.raptor import Raptor
 261
 262         raptor_mh = Raptor(
 263             config=self.args["config"],
 264             initial_config_file=self.args["initial_config_file"],
 265         )
 266         return raptor_mh.run()
 267
 268
 269 def setup_node(command_context):
 270     """Fetch the latest node-16 binary and install it into the .mozbuild directory."""
 271     import platform
 272
 273     from mozbuild.artifact_commands import artifact_toolchain
 274     from mozbuild.nodeutil import find_node_executable
 275     from packaging.version import Version
 276
 277     print("Setting up node for browsertime...")
 278     state_dir = get_state_dir()
 279     cache_path = os.path.join(state_dir, "browsertime", "node-16")
 280
 281     def __check_for_node():
 282         # Check standard locations first
 283         node_exe = find_node_executable(min_version=Version("16.0.0"))
 284         if node_exe and (node_exe[0] is not None):
 285             return node_exe[0]
 286         if not os.path.exists(cache_path):
 287             return None
 288
 289         # Check the browsertime-specific node location next
 290         node_name = "node"
 291         if platform.system() == "Windows":
 292             node_name = "node.exe"
 293             node_exe_path = os.path.join(
 294                 state_dir,
 295                 "browsertime",
 296                 "node-16",
 297                 "node",
 298             )
 299         else:
 300             node_exe_path = os.path.join(
 301                 state_dir,
 302                 "browsertime",
 303                 "node-16",
 304                 "node",
 305                 "bin",
 306             )
 307
 308         node_exe = os.path.join(node_exe_path, node_name)
 309         if not os.path.exists(node_exe):
 310             return None
 311
 312         return node_exe
 313
 314     node_exe = __check_for_node()
 315     if node_exe is None:
 316         toolchain_job = "{}-node-16"
 317         plat = platform.system()
 318         if plat == "Windows":
 319             toolchain_job = toolchain_job.format("win64")
 320         elif plat == "Darwin":
 321             if platform.processor() == "arm":
 322                 toolchain_job = toolchain_job.format("macosx64-aarch64")
 323             else:
 324                 toolchain_job = toolchain_job.format("macosx64")
 325         else:
 326             toolchain_job = toolchain_job.format("linux64")
 327
 328         print(
 329             "Downloading Node v16 from Taskcluster toolchain {}...".format(
 330                 toolchain_job
 331             )
 332         )
 333
 334         if not os.path.exists(cache_path):
 335             os.makedirs(cache_path, exist_ok=True)
 336
 337         # Change directories to where node should be installed
 338         # before installing. Otherwise, it gets installed in the
 339         # top level of the repo (or the current working directory).
 340         cur_dir = os.getcwd()
 341         os.chdir(cache_path)
 342         artifact_toolchain(
 343             command_context,
 344             verbose=False,
 345             from_build=[toolchain_job],
 346             no_unpack=False,
 347             retry=0,
 348             cache_dir=cache_path,
 349         )
 350         os.chdir(cur_dir)
 351
 352         node_exe = __check_for_node()
 353         if node_exe is None:
 354             raise Exception("Could not find Node v16 binary for Raptor-Browsertime")
 355
 356         print("Finished downloading Node v16 from Taskcluster")
 357
 358     print("Node v16+ found at: %s" % node_exe)
 359     return node_exe
 360
 361
 362 def create_parser():
 363     sys.path.insert(0, HERE)  # allow to import the raptor package
 364     from raptor.cmdline import create_parser
 365
 366     return create_parser(mach_interface=True)
 367
 368
 369 @Command(
 370     "raptor",
 371     category="testing",
 372     description="Run Raptor performance tests.",
 373     parser=create_parser,
 374 )
 375 def run_raptor(command_context, **kwargs):
 376     build_obj = command_context
 377
 378     # Setup node for browsertime
 379     kwargs["browsertime_node"] = setup_node(command_context)
 380
 381     is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS
 382
 383     if is_android:
 384         from mozrunner.devices.android_device import (
 385             InstallIntent,
 386             verify_android_device,
 387         )
 388
 389         install = (
 390             InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES
 391         )
 392         verbose = False
 393         if (
 394             kwargs.get("log_mach_verbose")
 395             or kwargs.get("log_tbpl_level") == "debug"
 396             or kwargs.get("log_mach_level") == "debug"
 397             or kwargs.get("log_raw_level") == "debug"
 398         ):
 399             verbose = True
 400         if not verify_android_device(
 401             build_obj,
 402             install=install,
 403             app=kwargs["binary"],
 404             verbose=verbose,
 405             xre=True,
 406         ):  # Equivalent to 'run_local' = True.
 407             print(
 408                 "****************************************************************************"
 409             )
 410             print(
 411                 "Unable to verify device, please check your attached/connected android device"
 412             )
 413             print(
 414                 "****************************************************************************"
 415             )
 416             return 1
 417         # Disable fission until geckoview supports fission by default.
 418         # Need fission on Android? Use '--setpref fission.autostart=true'
 419         kwargs["fission"] = False
 420
 421     # Remove mach global arguments from sys.argv to prevent them
 422     # from being consumed by raptor. Treat any item in sys.argv
 423     # occuring before "raptor" as a mach global argument.
 424     argv = []
 425     in_mach = True
 426     for arg in sys.argv:
 427         if not in_mach:
 428             argv.append(arg)
 429         if arg.startswith("raptor"):
 430             in_mach = False
 431
 432     raptor = command_context._spawn(RaptorRunner)
 433
 434     try:
 435         return raptor.run_test(argv, kwargs)
 436     except BinaryNotFoundException as e:
 437         command_context.log(
 438             logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}"
 439         )
 440         command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
 441         return 1
 442     except Exception as e:
 443         print(repr(e))
 444         return 1
 445
 446
 447 @Command(
 448     "raptor-test",
 449     category="testing",
 450     description="Run Raptor performance tests.",
 451     parser=create_parser,
 452 )
 453 def run_raptor_test(command_context, **kwargs):
 454     return run_raptor(command_context, **kwargs)
 455
 456
 457 class PythonVersionException(Exception):
 458     pass