testing/talos/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates Talos mozharness with mach
   6
   7 import json
   8 import logging
   9 import os
  10 import socket
  11 import sys
  12
  13 import six
  14 from mach.decorators import Command
  15 from mach.util import get_state_dir
  16 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  17
  18 HERE = os.path.dirname(os.path.realpath(__file__))
  19
  20
  21 class TalosRunner(MozbuildObject):
  22     def run_test(self, talos_args):
  23         """
  24         We want to do couple of things before running Talos
  25         1. Clone mozharness
  26         2. Make config for Talos Mozharness
  27         3. Run mozharness
  28         """
  29         # Validate that the user is using a supported python version before doing anything else
  30         max_py_major, max_py_minor = 3, 11
  31         sys_maj, sys_min = sys.version_info.major, sys.version_info.minor
  32         if sys_min > max_py_minor:
  33             raise PythonVersionException(
  34                 print(
  35                     f"\tPlease downgrade your Python version as talos does not yet support Python "
  36                     f"versions greater than {max_py_major}.{max_py_minor}."
  37                     f"\n\tYou seem to currently be using Python {sys_maj}.{sys_min}."
  38                     f"\n\tSee here for a possible solution in debugging your python environment: "
  39                     f"https://firefox-source-docs.mozilla.org/testing/perfdocs/"
  40                     f"debugging.html#debugging-local-python-environment"
  41                 )
  42             )
  43         try:
  44             self.init_variables(talos_args)
  45         except BinaryNotFoundException as e:
  46             self.log(logging.ERROR, "talos", {"error": str(e)}, "ERROR: {error}")
  47             self.log(logging.INFO, "raptor", {"help": e.help()}, "{help}")
  48             return 1
  49
  50         self.make_config()
  51         self.write_config()
  52         self.make_args()
  53         return self.run_mozharness()
  54
  55     def init_variables(self, talos_args):
  56         self.talos_dir = os.path.join(self.topsrcdir, "testing", "talos")
  57         self.mozharness_dir = os.path.join(self.topsrcdir, "testing", "mozharness")
  58         self.talos_json = os.path.join(self.talos_dir, "talos.json")
  59         self.config_file_path = os.path.join(
  60             self._topobjdir, "testing", "talos-in_tree_conf.json"
  61         )
  62         self.binary_path = self.get_binary_path()
  63         self.virtualenv_script = os.path.join(
  64             self.topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
  65         )
  66         self.virtualenv_path = os.path.join(self._topobjdir, "testing", "talos-venv")
  67         self.python_interp = sys.executable
  68         self.talos_args = talos_args
  69
  70     def make_config(self):
  71         default_actions = ["populate-webroot"]
  72         default_actions.extend(
  73             [
  74                 "create-virtualenv",
  75                 "run-tests",
  76             ]
  77         )
  78         self.config = {
  79             "run_local": True,
  80             "talos_json": self.talos_json,
  81             "binary_path": self.binary_path,
  82             "repo_path": self.topsrcdir,
  83             "obj_path": self.topobjdir,
  84             "log_name": "talos",
  85             "virtualenv_path": self.virtualenv_path,
  86             "pypi_url": "http://pypi.python.org/simple",
  87             "base_work_dir": self.mozharness_dir,
  88             "exes": {
  89                 "python": self.python_interp,
  90                 "virtualenv": [self.python_interp, self.virtualenv_script],
  91             },
  92             "title": socket.gethostname(),
  93             "default_actions": default_actions,
  94             "talos_extra_options": ["--develop"] + self.talos_args,
  95             "python3_manifest": {
  96                 "win32": "python3.manifest",
  97                 "win64": "python3_x64.manifest",
  98             },
  99             "mozbuild_path": get_state_dir(),
 100         }
 101
 102     def make_args(self):
 103         self.args = {
 104             "config": {},
 105             "initial_config_file": self.config_file_path,
 106         }
 107
 108     def write_config(self):
 109         try:
 110             config_file = open(self.config_file_path, "wb")
 111             config_file.write(six.ensure_binary(json.dumps(self.config)))
 112         except IOError as e:
 113             err_str = "Error writing to Talos Mozharness config file {0}:{1}"
 114             print(err_str.format(self.config_file_path, str(e)))
 115             raise e
 116
 117     def run_mozharness(self):
 118         sys.path.insert(0, self.mozharness_dir)
 119         from mozharness.mozilla.testing.talos import Talos
 120
 121         talos_mh = Talos(
 122             config=self.args["config"],
 123             initial_config_file=self.args["initial_config_file"],
 124         )
 125         return talos_mh.run()
 126
 127
 128 def create_parser():
 129     sys.path.insert(0, HERE)  # allow to import the talos package
 130     from talos.cmdline import create_parser
 131
 132     return create_parser(mach_interface=True)
 133
 134
 135 def setup_toolchain_artifacts(args, command_context):
 136     if not any(arg.lower() == "pdfpaint" for arg in args):
 137         return
 138
 139     from mozbuild.bootstrap import bootstrap_toolchain
 140
 141     print("Setting up pdfpaint PDFs...")
 142     bootstrap_toolchain("talos-pdfs")
 143
 144
 145 @Command(
 146     "talos-test",
 147     category="testing",
 148     description="Run talos tests (performance testing).",
 149     parser=create_parser,
 150 )
 151 def run_talos_test(command_context, **kwargs):
 152     talos = command_context._spawn(TalosRunner)
 153
 154     try:
 155         args = sys.argv[2:]
 156         setup_toolchain_artifacts(args, command_context)
 157         return talos.run_test(args)
 158     except Exception as e:
 159         print(str(e))
 160         return 1
 161
 162
 163 class PythonVersionException(Exception):
 164     pass