tools/tryselect/task_config.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 """
   6 Templates provide a way of modifying the task definition of selected tasks.
   7 They are added to 'try_task_config.json' and processed by the transforms.
   8 """
   9
  10
  11 import json
  12 import os
  13 import pathlib
  14 import subprocess
  15 import sys
  16 from abc import ABCMeta, abstractmethod, abstractproperty
  17 from argparse import SUPPRESS, Action
  18 from contextlib import contextmanager
  19 from textwrap import dedent
  20
  21 import mozpack.path as mozpath
  22 import six
  23 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
  24 from mozversioncontrol import Repository
  25
  26 from .tasks import resolve_tests_by_suite
  27
  28 here = os.path.abspath(os.path.dirname(__file__))
  29 build = MozbuildObject.from_environment(cwd=here)
  30
  31
  32 @contextmanager
  33 def try_config_commit(vcs: Repository, commit_message: str):
  34     """Context manager that creates and removes a try config commit."""
  35     # Add the `try_task_config.json` file if it exists.
  36     try_task_config_path = pathlib.Path(build.topsrcdir) / "try_task_config.json"
  37     if try_task_config_path.exists():
  38         vcs.add_remove_files("try_task_config.json")
  39
  40     try:
  41         # Create a try config commit.
  42         vcs.create_try_commit(commit_message)
  43
  44         yield
  45     finally:
  46         # Revert the try config commit.
  47         vcs.remove_current_commit()
  48
  49
  50 class TryConfig:
  51     __metaclass__ = ABCMeta
  52
  53     def __init__(self):
  54         self.dests = set()
  55
  56     def add_arguments(self, parser):
  57         for cli, kwargs in self.arguments:
  58             action = parser.add_argument(*cli, **kwargs)
  59             self.dests.add(action.dest)
  60
  61     @abstractproperty
  62     def arguments(self):
  63         pass
  64
  65     @abstractmethod
  66     def try_config(self, **kwargs):
  67         pass
  68
  69     def validate(self, **kwargs):
  70         pass
  71
  72
  73 class Artifact(TryConfig):
  74     arguments = [
  75         [
  76             ["--artifact"],
  77             {"action": "store_true", "help": "Force artifact builds where possible."},
  78         ],
  79         [
  80             ["--no-artifact"],
  81             {
  82                 "action": "store_true",
  83                 "help": "Disable artifact builds even if being used locally.",
  84             },
  85         ],
  86     ]
  87
  88     def add_arguments(self, parser):
  89         group = parser.add_mutually_exclusive_group()
  90         return super().add_arguments(group)
  91
  92     @classmethod
  93     def is_artifact_build(cls):
  94         try:
  95             return build.substs.get("MOZ_ARTIFACT_BUILDS", False)
  96         except BuildEnvironmentNotFoundException:
  97             return False
  98
  99     def try_config(self, artifact, no_artifact, **kwargs):
 100         if artifact:
 101             return {"use-artifact-builds": True, "disable-pgo": True}
 102
 103         if no_artifact:
 104             return
 105
 106         if self.is_artifact_build():
 107             print("Artifact builds enabled, pass --no-artifact to disable")
 108             return {"use-artifact-builds": True, "disable-pgo": True}
 109
 110
 111 class Pernosco(TryConfig):
 112     arguments = [
 113         [
 114             ["--pernosco"],
 115             {
 116                 "action": "store_true",
 117                 "default": None,
 118                 "help": "Opt-in to analysis by the Pernosco debugging service.",
 119             },
 120         ],
 121         [
 122             ["--no-pernosco"],
 123             {
 124                 "dest": "pernosco",
 125                 "action": "store_false",
 126                 "default": None,
 127                 "help": "Opt-out of the Pernosco debugging service (if you are on the include list).",
 128             },
 129         ],
 130     ]
 131
 132     def add_arguments(self, parser):
 133         group = parser.add_mutually_exclusive_group()
 134         return super().add_arguments(group)
 135
 136     def try_config(self, pernosco, **kwargs):
 137         if pernosco is None:
 138             return
 139
 140         if pernosco:
 141             try:
 142                 # The Pernosco service currently requires a Mozilla e-mail address to
 143                 # log in. Prevent people with non-Mozilla addresses from using this
 144                 # flag so they don't end up consuming time and resources only to
 145                 # realize they can't actually log in and see the reports.
 146                 cmd = ["ssh", "-G", "hg.mozilla.org"]
 147                 output = subprocess.check_output(
 148                     cmd, universal_newlines=True
 149                 ).splitlines()
 150                 address = [
 151                     l.rsplit(" ", 1)[-1] for l in output if l.startswith("user")
 152                 ][0]
 153                 if not address.endswith("@mozilla.com"):
 154                     print(
 155                         dedent(
 156                             """\
 157                         Pernosco requires a Mozilla e-mail address to view its reports. Please
 158                         push to try with an @mozilla.com address to use --pernosco.
 159
 160                             Current user: {}
 161                     """.format(
 162                                 address
 163                             )
 164                         )
 165                     )
 166                     sys.exit(1)
 167
 168             except (subprocess.CalledProcessError, IndexError):
 169                 print("warning: failed to detect current user for 'hg.mozilla.org'")
 170                 print("Pernosco requires a Mozilla e-mail address to view its reports.")
 171                 while True:
 172                     answer = input(
 173                         "Do you have an @mozilla.com address? [Y/n]: "
 174                     ).lower()
 175                     if answer == "n":
 176                         sys.exit(1)
 177                     elif answer == "y":
 178                         break
 179
 180         return {
 181             "env": {
 182                 "PERNOSCO": str(int(pernosco)),
 183             }
 184         }
 185
 186     def validate(self, **kwargs):
 187         if kwargs["try_config"].get("use-artifact-builds"):
 188             print(
 189                 "Pernosco does not support artifact builds at this time. "
 190                 "Please try again with '--no-artifact'."
 191             )
 192             sys.exit(1)
 193
 194
 195 class Path(TryConfig):
 196     arguments = [
 197         [
 198             ["paths"],
 199             {
 200                 "nargs": "*",
 201                 "default": [],
 202                 "help": "Run tasks containing tests under the specified path(s).",
 203             },
 204         ],
 205     ]
 206
 207     def try_config(self, paths, **kwargs):
 208         if not paths:
 209             return
 210
 211         for p in paths:
 212             if not os.path.exists(p):
 213                 print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
 214                 sys.exit(1)
 215
 216         paths = [
 217             mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir)
 218             for p in paths
 219         ]
 220         return {
 221             "env": {
 222                 "MOZHARNESS_TEST_PATHS": six.ensure_text(
 223                     json.dumps(resolve_tests_by_suite(paths))
 224                 ),
 225             }
 226         }
 227
 228
 229 class Environment(TryConfig):
 230     arguments = [
 231         [
 232             ["--env"],
 233             {
 234                 "action": "append",
 235                 "default": None,
 236                 "help": "Set an environment variable, of the form FOO=BAR. "
 237                 "Can be passed in multiple times.",
 238             },
 239         ],
 240     ]
 241
 242     def try_config(self, env, **kwargs):
 243         if not env:
 244             return
 245         return {
 246             "env": dict(e.split("=", 1) for e in env),
 247         }
 248
 249
 250 class RangeAction(Action):
 251     def __init__(self, min, max, *args, **kwargs):
 252         self.min = min
 253         self.max = max
 254         kwargs["metavar"] = "[{}-{}]".format(self.min, self.max)
 255         super().__init__(*args, **kwargs)
 256
 257     def __call__(self, parser, namespace, values, option_string=None):
 258         name = option_string or self.dest
 259         if values < self.min:
 260             parser.error("{} can not be less than {}".format(name, self.min))
 261         if values > self.max:
 262             parser.error("{} can not be more than {}".format(name, self.max))
 263         setattr(namespace, self.dest, values)
 264
 265
 266 class Rebuild(TryConfig):
 267     arguments = [
 268         [
 269             ["--rebuild"],
 270             {
 271                 "action": RangeAction,
 272                 "min": 2,
 273                 "max": 20,
 274                 "default": None,
 275                 "type": int,
 276                 "help": "Rebuild all selected tasks the specified number of times.",
 277             },
 278         ],
 279     ]
 280
 281     def try_config(self, rebuild, **kwargs):
 282         if not rebuild:
 283             return
 284
 285         if kwargs.get("full") and rebuild > 3:
 286             print(
 287                 "warning: limiting --rebuild to 3 when using --full. "
 288                 "Use custom push actions to add more."
 289             )
 290             rebuild = 3
 291
 292         return {
 293             "rebuild": rebuild,
 294         }
 295
 296
 297 class Routes(TryConfig):
 298     arguments = [
 299         [
 300             ["--route"],
 301             {
 302                 "action": "append",
 303                 "dest": "routes",
 304                 "help": (
 305                     "Additional route to add to the tasks "
 306                     "(note: these will not be added to the decision task)"
 307                 ),
 308             },
 309         ],
 310     ]
 311
 312     def try_config(self, routes, **kwargs):
 313         if routes:
 314             return {
 315                 "routes": routes,
 316             }
 317
 318
 319 class ChemspillPrio(TryConfig):
 320     arguments = [
 321         [
 322             ["--chemspill-prio"],
 323             {
 324                 "action": "store_true",
 325                 "help": "Run at a higher priority than most try jobs (chemspills only).",
 326             },
 327         ],
 328     ]
 329
 330     def try_config(self, chemspill_prio, **kwargs):
 331         if chemspill_prio:
 332             return {"chemspill-prio": {}}
 333
 334
 335 class GeckoProfile(TryConfig):
 336     arguments = [
 337         [
 338             ["--gecko-profile"],
 339             {
 340                 "dest": "profile",
 341                 "action": "store_true",
 342                 "default": False,
 343                 "help": "Create and upload a gecko profile during talos/raptor tasks.",
 344             },
 345         ],
 346         [
 347             ["--gecko-profile-interval"],
 348             {
 349                 "dest": "gecko_profile_interval",
 350                 "type": float,
 351                 "help": "How frequently to take samples (ms)",
 352             },
 353         ],
 354         [
 355             ["--gecko-profile-entries"],
 356             {
 357                 "dest": "gecko_profile_entries",
 358                 "type": int,
 359                 "help": "How many samples to take with the profiler",
 360             },
 361         ],
 362         [
 363             ["--gecko-profile-features"],
 364             {
 365                 "dest": "gecko_profile_features",
 366                 "type": str,
 367                 "default": None,
 368                 "help": "Set the features enabled for the profiler.",
 369             },
 370         ],
 371         [
 372             ["--gecko-profile-threads"],
 373             {
 374                 "dest": "gecko_profile_threads",
 375                 "type": str,
 376                 "help": "Comma-separated list of threads to sample.",
 377             },
 378         ],
 379         # For backwards compatibility
 380         [
 381             ["--talos-profile"],
 382             {
 383                 "dest": "profile",
 384                 "action": "store_true",
 385                 "default": False,
 386                 "help": SUPPRESS,
 387             },
 388         ],
 389         # This is added for consistency with the 'syntax' selector
 390         [
 391             ["--geckoProfile"],
 392             {
 393                 "dest": "profile",
 394                 "action": "store_true",
 395                 "default": False,
 396                 "help": SUPPRESS,
 397             },
 398         ],
 399     ]
 400
 401     def try_config(
 402         self,
 403         profile,
 404         gecko_profile_interval,
 405         gecko_profile_entries,
 406         gecko_profile_features,
 407         gecko_profile_threads,
 408         **kwargs
 409     ):
 410         if profile or not all(
 411             s is None for s in (gecko_profile_features, gecko_profile_threads)
 412         ):
 413             cfg = {
 414                 "gecko-profile": True,
 415                 "gecko-profile-interval": gecko_profile_interval,
 416                 "gecko-profile-entries": gecko_profile_entries,
 417                 "gecko-profile-features": gecko_profile_features,
 418                 "gecko-profile-threads": gecko_profile_threads,
 419             }
 420             return {key: value for key, value in cfg.items() if value is not None}
 421
 422
 423 class Browsertime(TryConfig):
 424     arguments = [
 425         [
 426             ["--browsertime"],
 427             {
 428                 "action": "store_true",
 429                 "help": "Use browsertime during Raptor tasks.",
 430             },
 431         ],
 432     ]
 433
 434     def try_config(self, browsertime, **kwargs):
 435         if browsertime:
 436             return {
 437                 "browsertime": True,
 438             }
 439
 440
 441 class DisablePgo(TryConfig):
 442     arguments = [
 443         [
 444             ["--disable-pgo"],
 445             {
 446                 "action": "store_true",
 447                 "help": "Don't run PGO builds",
 448             },
 449         ],
 450     ]
 451
 452     def try_config(self, disable_pgo, **kwargs):
 453         if disable_pgo:
 454             return {
 455                 "disable-pgo": True,
 456             }
 457
 458
 459 class WorkerOverrides(TryConfig):
 460     arguments = [
 461         [
 462             ["--worker-override"],
 463             {
 464                 "action": "append",
 465                 "dest": "worker_overrides",
 466                 "help": (
 467                     "Override the worker pool used for a given taskgraph worker alias. "
 468                     "The argument should be `<alias>=<worker-pool>`. "
 469                     "Can be specified multiple times."
 470                 ),
 471             },
 472         ],
 473         [
 474             ["--worker-suffix"],
 475             {
 476                 "action": "append",
 477                 "dest": "worker_suffixes",
 478                 "help": (
 479                     "Override the worker pool used for a given taskgraph worker alias, "
 480                     "by appending a suffix to the work-pool. "
 481                     "The argument should be `<alias>=<suffix>`. "
 482                     "Can be specified multiple times."
 483                 ),
 484             },
 485         ],
 486     ]
 487
 488     def try_config(self, worker_overrides, worker_suffixes, **kwargs):
 489         from gecko_taskgraph.util.workertypes import get_worker_type
 490         from taskgraph.config import load_graph_config
 491
 492         overrides = {}
 493         if worker_overrides:
 494             for override in worker_overrides:
 495                 alias, worker_pool = override.split("=", 1)
 496                 if alias in overrides:
 497                     print(
 498                         "Can't override worker alias {alias} more than once. "
 499                         "Already set to use {previous}, but also asked to use {new}.".format(
 500                             alias=alias, previous=overrides[alias], new=worker_pool
 501                         )
 502                     )
 503                     sys.exit(1)
 504                 overrides[alias] = worker_pool
 505
 506         if worker_suffixes:
 507             root = build.topsrcdir
 508             root = os.path.join(root, "taskcluster", "ci")
 509             graph_config = load_graph_config(root)
 510             for worker_suffix in worker_suffixes:
 511                 alias, suffix = worker_suffix.split("=", 1)
 512                 if alias in overrides:
 513                     print(
 514                         "Can't override worker alias {alias} more than once. "
 515                         "Already set to use {previous}, but also asked "
 516                         "to add suffix {suffix}.".format(
 517                             alias=alias, previous=overrides[alias], suffix=suffix
 518                         )
 519                     )
 520                     sys.exit(1)
 521                 provisioner, worker_type = get_worker_type(
 522                     graph_config, worker_type=alias, parameters={"level": "1"}
 523                 )
 524                 overrides[alias] = "{provisioner}/{worker_type}{suffix}".format(
 525                     provisioner=provisioner, worker_type=worker_type, suffix=suffix
 526                 )
 527
 528         if overrides:
 529             return {"worker-overrides": overrides}
 530
 531
 532 all_task_configs = {
 533     "artifact": Artifact,
 534     "browsertime": Browsertime,
 535     "chemspill-prio": ChemspillPrio,
 536     "disable-pgo": DisablePgo,
 537     "env": Environment,
 538     "gecko-profile": GeckoProfile,
 539     "path": Path,
 540     "pernosco": Pernosco,
 541     "rebuild": Rebuild,
 542     "routes": Routes,
 543     "worker-overrides": WorkerOverrides,
 544 }