testing/raptor/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Originally taken from /talos/mach_commands.py
   6
   7 # Integrates raptor mozharness with mach
   8
   9 import json
  10 import logging
  11 import os
  12 import socket
  13 import subprocess
  14 import sys
  15
  16 from mach.decorators import Command
  17 from mach.util import get_state_dir
  18 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  19 from mozbuild.base import MachCommandConditions as Conditions
  20
  21 HERE = os.path.dirname(os.path.realpath(__file__))
  22
  23 ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m"]
  24
  25
  26 class RaptorRunner(MozbuildObject):
  27     def run_test(self, raptor_args, kwargs):
  28         """Setup and run mozharness.
  29
  30         We want to do a few things before running Raptor:
  31
  32         1. Clone mozharness
  33         2. Make the config for Raptor mozharness
  34         3. Run mozharness
  35         """
  36         # Validate that the user is using a supported python version before doing anything else
  37         max_py_major, max_py_minor = 3, 10
  38         sys_maj, sys_min = sys.version_info.major, sys.version_info.minor
  39         if sys_min > max_py_minor:
  40             raise PythonVersionException(
  41                 f"Please downgrade your Python version as Raptor does not yet support Python versions greater than {max_py_major}.{max_py_minor}. "
  42                 f"You seem to currently be using Python {sys_maj}.{sys_min}"
  43             )
  44         self.init_variables(raptor_args, kwargs)
  45         self.make_config()
  46         self.write_config()
  47         self.make_args()
  48         return self.run_mozharness()
  49
  50     def init_variables(self, raptor_args, kwargs):
  51         self.raptor_args = raptor_args
  52
  53         if kwargs.get("host") == "HOST_IP":
  54             kwargs["host"] = os.environ["HOST_IP"]
  55         self.host = kwargs["host"]
  56         self.is_release_build = kwargs["is_release_build"]
  57         self.live_sites = kwargs["live_sites"]
  58         self.disable_perf_tuning = kwargs["disable_perf_tuning"]
  59         self.conditioned_profile = kwargs["conditioned_profile"]
  60         self.device_name = kwargs["device_name"]
  61         self.enable_marionette_trace = kwargs["enable_marionette_trace"]
  62         self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
  63         self.browsertime_node = kwargs["browsertime_node"]
  64         self.clean = kwargs["clean"]
  65
  66         if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
  67             self.binary_path = None
  68         else:
  69             self.binary_path = kwargs.get("binary") or self.get_binary_path()
  70
  71         self.python = sys.executable
  72
  73         self.raptor_dir = os.path.join(self.topsrcdir, "testing", "raptor")
  74         self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness")
  75         self.config_file_path = os.path.join(
  76             self._topobjdir, "testing", "raptor-in_tree_conf.json"
  77         )
  78
  79         self.virtualenv_script = os.path.join(
  80             self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
  81         )
  82         self.virtualenv_path = os.path.join(self._topobjdir, "testing", "raptor-venv")
  83
  84     def make_config(self):
  85         default_actions = [
  86             "populate-webroot",
  87             "create-virtualenv",
  88             "install-chromium-distribution",
  89             "run-tests",
  90         ]
  91         self.config = {
  92             "run_local": True,
  93             "binary_path": self.binary_path,
  94             "repo_path": self.topsrcdir,
  95             "raptor_path": self.raptor_dir,
  96             "obj_path": self.topobjdir,
  97             "log_name": "raptor",
  98             "virtualenv_path": self.virtualenv_path,
  99             "pypi_url": "http://pypi.org/simple",
 100             "base_work_dir": self.mozharness_dir,
 101             "exes": {
 102                 "python": self.python,
 103                 "virtualenv": [self.python, self.virtualenv_script],
 104             },
 105             "title": socket.gethostname(),
 106             "default_actions": default_actions,
 107             "raptor_cmd_line_args": self.raptor_args,
 108             "host": self.host,
 109             "live_sites": self.live_sites,
 110             "disable_perf_tuning": self.disable_perf_tuning,
 111             "conditioned_profile": self.conditioned_profile,
 112             "is_release_build": self.is_release_build,
 113             "device_name": self.device_name,
 114             "enable_marionette_trace": self.enable_marionette_trace,
 115             "browsertime_visualmetrics": self.browsertime_visualmetrics,
 116             "browsertime_node": self.browsertime_node,
 117             "mozbuild_path": get_state_dir(),
 118             "clean": self.clean,
 119         }
 120
 121         sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime"))
 122         try:
 123             import platform
 124
 125             import mach_commands as browsertime
 126
 127             # We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by
 128             # browsertime in its `node_modules` directory, which is appropriate for local builds.
 129             # We don't set `browsertime_ffmpeg` yet: it will need to be on the path.  There is code
 130             # to configure the environment including the path in
 131             # `tools/browsertime/mach_commands.py` but integrating it here will take more effort.
 132             self.config.update(
 133                 {
 134                     "browsertime_browsertimejs": browsertime.browsertime_path(),
 135                     "browsertime_vismet_script": browsertime.visualmetrics_path(),
 136                 }
 137             )
 138
 139             def _get_browsertime_package():
 140                 with open(
 141                     os.path.join(
 142                         self.topsrcdir,
 143                         "tools",
 144                         "browsertime",
 145                         "node_modules",
 146                         "browsertime",
 147                         "package.json",
 148                     )
 149                 ) as package:
 150                     return json.load(package)
 151
 152             def _get_browsertime_resolved():
 153                 try:
 154                     with open(
 155                         os.path.join(
 156                             self.topsrcdir,
 157                             "tools",
 158                             "browsertime",
 159                             "node_modules",
 160                             ".package-lock.json",
 161                         )
 162                     ) as package_lock:
 163                         return json.load(package_lock)["packages"][
 164                             "node_modules/browsertime"
 165                         ]["resolved"]
 166                 except FileNotFoundError:
 167                     # Older versions of node/npm add this metadata to package.json
 168                     return _get_browsertime_package()["_from"]
 169
 170             def _should_install():
 171                 # If ffmpeg doesn't exist in the .mozbuild directory,
 172                 # then we should install
 173                 btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime")
 174                 if not os.path.exists(btime_cache) or not any(
 175                     ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)]
 176                 ):
 177                     return True
 178
 179                 # If browsertime doesn't exist, install it
 180                 if not os.path.exists(
 181                     self.config["browsertime_browsertimejs"]
 182                 ) or not os.path.exists(self.config["browsertime_vismet_script"]):
 183                     return True
 184
 185                 # Browsertime exists, check if it's outdated
 186                 with open(
 187                     os.path.join(self.topsrcdir, "tools", "browsertime", "package.json")
 188                 ) as new:
 189                     new_pkg = json.load(new)
 190
 191                 return not _get_browsertime_resolved().endswith(
 192                     new_pkg["devDependencies"]["browsertime"]
 193                 )
 194
 195             def _get_browsertime_version():
 196                 # Returns the (version number, current commit) used
 197                 return (
 198                     _get_browsertime_package()["version"],
 199                     _get_browsertime_resolved(),
 200                 )
 201
 202             # Check if browsertime scripts exist and try to install them if
 203             # they aren't
 204             if _should_install():
 205                 # TODO: Make this "integration" nicer in the near future
 206                 print("Missing browsertime files...attempting to install")
 207                 subprocess.check_output(
 208                     [
 209                         os.path.join(self.topsrcdir, "mach"),
 210                         "browsertime",
 211                         "--setup",
 212                         "--clobber",
 213                     ],
 214                     shell="windows" in platform.system().lower(),
 215                 )
 216                 if _should_install():
 217                     raise Exception(
 218                         "Failed installation attempt. Cannot find browsertime scripts. "
 219                         "Run `./mach browsertime --setup --clobber` to set it up."
 220                     )
 221
 222                 # Bug 1766112 - For the time being, we need to trigger a
 223                 # clean build to upgrade browsertime. This should be disabled
 224                 # after some time.
 225                 print(
 226                     "Setting --clean to True to rebuild Python "
 227                     "environment for Browsertime upgrade..."
 228                 )
 229                 self.config["clean"] = True
 230
 231             print("Using browsertime version %s from %s" % _get_browsertime_version())
 232
 233         finally:
 234             sys.path = sys.path[1:]
 235
 236     def make_args(self):
 237         self.args = {
 238             "config": {},
 239             "initial_config_file": self.config_file_path,
 240         }
 241
 242     def write_config(self):
 243         try:
 244             config_file = open(self.config_file_path, "w")
 245             config_file.write(json.dumps(self.config))
 246             config_file.close()
 247         except IOError as e:
 248             err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
 249             print(err_str.format(self.config_file_path, str(e)))
 250             raise e
 251
 252     def run_mozharness(self):
 253         sys.path.insert(0, self.mozharness_dir)
 254         from mozharness.mozilla.testing.raptor import Raptor
 255
 256         raptor_mh = Raptor(
 257             config=self.args["config"],
 258             initial_config_file=self.args["initial_config_file"],
 259         )
 260         return raptor_mh.run()
 261
 262
 263 def setup_node(command_context):
 264     """Fetch the latest node-16 binary and install it into the .mozbuild directory."""
 265     import platform
 266     from distutils.version import StrictVersion
 267
 268     from mozbuild.artifact_commands import artifact_toolchain
 269     from mozbuild.nodeutil import find_node_executable
 270
 271     print("Setting up node for browsertime...")
 272     state_dir = get_state_dir()
 273     cache_path = os.path.join(state_dir, "browsertime", "node-16")
 274
 275     def __check_for_node():
 276         # Check standard locations first
 277         node_exe = find_node_executable(min_version=StrictVersion("16.0.0"))
 278         if node_exe and (node_exe[0] is not None):
 279             return node_exe[0]
 280         if not os.path.exists(cache_path):
 281             return None
 282
 283         # Check the browsertime-specific node location next
 284         node_name = "node"
 285         if platform.system() == "Windows":
 286             node_name = "node.exe"
 287             node_exe_path = os.path.join(
 288                 state_dir,
 289                 "browsertime",
 290                 "node-16",
 291                 "node",
 292             )
 293         else:
 294             node_exe_path = os.path.join(
 295                 state_dir,
 296                 "browsertime",
 297                 "node-16",
 298                 "node",
 299                 "bin",
 300             )
 301
 302         node_exe = os.path.join(node_exe_path, node_name)
 303         if not os.path.exists(node_exe):
 304             return None
 305
 306         return node_exe
 307
 308     node_exe = __check_for_node()
 309     if node_exe is None:
 310         toolchain_job = "{}-node-16"
 311         plat = platform.system()
 312         if plat == "Windows":
 313             toolchain_job = toolchain_job.format("win64")
 314         elif plat == "Darwin":
 315             toolchain_job = toolchain_job.format("macosx64")
 316         else:
 317             toolchain_job = toolchain_job.format("linux64")
 318
 319         print(
 320             "Downloading Node v16 from Taskcluster toolchain {}...".format(
 321                 toolchain_job
 322             )
 323         )
 324
 325         if not os.path.exists(cache_path):
 326             os.makedirs(cache_path, exist_ok=True)
 327
 328         # Change directories to where node should be installed
 329         # before installing. Otherwise, it gets installed in the
 330         # top level of the repo (or the current working directory).
 331         cur_dir = os.getcwd()
 332         os.chdir(cache_path)
 333         artifact_toolchain(
 334             command_context,
 335             verbose=False,
 336             from_build=[toolchain_job],
 337             no_unpack=False,
 338             retry=0,
 339             cache_dir=cache_path,
 340         )
 341         os.chdir(cur_dir)
 342
 343         node_exe = __check_for_node()
 344         if node_exe is None:
 345             raise Exception("Could not find Node v16 binary for Raptor-Browsertime")
 346
 347         print("Finished downloading Node v16 from Taskcluster")
 348
 349     print("Node v16+ found at: %s" % node_exe)
 350     return node_exe
 351
 352
 353 def create_parser():
 354     sys.path.insert(0, HERE)  # allow to import the raptor package
 355     from raptor.cmdline import create_parser
 356
 357     return create_parser(mach_interface=True)
 358
 359
 360 @Command(
 361     "raptor",
 362     category="testing",
 363     description="Run Raptor performance tests.",
 364     parser=create_parser,
 365 )
 366 def run_raptor(command_context, **kwargs):
 367     build_obj = command_context
 368
 369     # Setup node for browsertime
 370     kwargs["browsertime_node"] = setup_node(command_context)
 371
 372     is_android = Conditions.is_android(build_obj) or kwargs["app"] in ANDROID_BROWSERS
 373
 374     if is_android:
 375         from mozrunner.devices.android_device import (
 376             InstallIntent,
 377             verify_android_device,
 378         )
 379
 380         install = (
 381             InstallIntent.NO if kwargs.pop("noinstall", False) else InstallIntent.YES
 382         )
 383         verbose = False
 384         if (
 385             kwargs.get("log_mach_verbose")
 386             or kwargs.get("log_tbpl_level") == "debug"
 387             or kwargs.get("log_mach_level") == "debug"
 388             or kwargs.get("log_raw_level") == "debug"
 389         ):
 390             verbose = True
 391         if not verify_android_device(
 392             build_obj,
 393             install=install,
 394             app=kwargs["binary"],
 395             verbose=verbose,
 396             xre=True,
 397         ):  # Equivalent to 'run_local' = True.
 398             return 1
 399         # Disable fission until geckoview supports fission by default.
 400         # Need fission on Android? Use '--setpref fission.autostart=true'
 401         kwargs["fission"] = False
 402
 403     # Remove mach global arguments from sys.argv to prevent them
 404     # from being consumed by raptor. Treat any item in sys.argv
 405     # occuring before "raptor" as a mach global argument.
 406     argv = []
 407     in_mach = True
 408     for arg in sys.argv:
 409         if not in_mach:
 410             argv.append(arg)
 411         if arg.startswith("raptor"):
 412             in_mach = False
 413
 414     raptor = command_context._spawn(RaptorRunner)
 415
 416     try:
 417         return raptor.run_test(argv, kwargs)
 418     except BinaryNotFoundException as e:
 419         command_context.log(
 420             logging.ERROR, "raptor", {"error": str(e)}, "ERROR: {error}"
 421         )
 422         command_context.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
 423         return 1
 424     except Exception as e:
 425         print(repr(e))
 426         return 1
 427
 428
 429 @Command(
 430     "raptor-test",
 431     category="testing",
 432     description="Run Raptor performance tests.",
 433     parser=create_parser,
 434 )
 435 def run_raptor_test(command_context, **kwargs):
 436     return run_raptor(command_context, **kwargs)
 437
 438
 439 class PythonVersionException(Exception):
 440     pass