tools/tryselect/task_config.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 """
   6 Templates provide a way of modifying the task definition of selected tasks.
   7 They are added to 'try_task_config.json' and processed by the transforms.
   8 """
   9
  10 from __future__ import absolute_import, print_function, unicode_literals
  11
  12 import json
  13 import os
  14 import six
  15 import subprocess
  16 import sys
  17 from abc import ABCMeta, abstractmethod, abstractproperty
  18 from argparse import Action, SUPPRESS
  19 from textwrap import dedent
  20
  21 import mozpack.path as mozpath
  22 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
  23 from taskgraph.util.python_path import find_object
  24
  25 from .tasks import resolve_tests_by_suite
  26
  27 here = os.path.abspath(os.path.dirname(__file__))
  28 build = MozbuildObject.from_environment(cwd=here)
  29
  30
  31 class TryConfig(object):
  32     __metaclass__ = ABCMeta
  33
  34     def __init__(self):
  35         self.dests = set()
  36
  37     def add_arguments(self, parser):
  38         for cli, kwargs in self.arguments:
  39             action = parser.add_argument(*cli, **kwargs)
  40             self.dests.add(action.dest)
  41
  42     @abstractproperty
  43     def arguments(self):
  44         pass
  45
  46     @abstractmethod
  47     def try_config(self, **kwargs):
  48         pass
  49
  50     def validate(self, **kwargs):
  51         pass
  52
  53
  54 class Artifact(TryConfig):
  55
  56     arguments = [
  57         [['--artifact'],
  58          {'action': 'store_true',
  59           'help': 'Force artifact builds where possible.'
  60           }],
  61         [['--no-artifact'],
  62          {'action': 'store_true',
  63           'help': 'Disable artifact builds even if being used locally.',
  64           }],
  65     ]
  66
  67     def add_arguments(self, parser):
  68         group = parser.add_mutually_exclusive_group()
  69         return super(Artifact, self).add_arguments(group)
  70
  71     @classmethod
  72     def is_artifact_build(cls):
  73         try:
  74             return build.substs.get("MOZ_ARTIFACT_BUILDS", False)
  75         except BuildEnvironmentNotFoundException:
  76             return False
  77
  78     def try_config(self, artifact, no_artifact, **kwargs):
  79         if artifact:
  80             return {
  81                 'use-artifact-builds': True
  82             }
  83
  84         if no_artifact:
  85             return
  86
  87         if self.is_artifact_build():
  88             print("Artifact builds enabled, pass --no-artifact to disable")
  89             return {
  90                 'use-artifact-builds': True
  91             }
  92
  93
  94 class Pernosco(TryConfig):
  95     arguments = [
  96         [['--pernosco'],
  97          {'action': 'store_true',
  98           'default': None,
  99           'help': 'Opt-in to analysis by the Pernosco debugging service.',
 100           }],
 101         [['--no-pernosco'],
 102          {'dest': 'pernosco',
 103           'action': 'store_false',
 104           'default': None,
 105           'help': 'Opt-out of the Pernosco debugging service (if you are on the whitelist).',
 106           }],
 107     ]
 108
 109     def add_arguments(self, parser):
 110         group = parser.add_mutually_exclusive_group()
 111         return super(Pernosco, self).add_arguments(group)
 112
 113     def try_config(self, pernosco, **kwargs):
 114         if pernosco is None:
 115             return
 116
 117         if pernosco:
 118             try:
 119                 # The Pernosco service currently requires a Mozilla e-mail address to
 120                 # log in. Prevent people with non-Mozilla addresses from using this
 121                 # flag so they don't end up consuming time and resources only to
 122                 # realize they can't actually log in and see the reports.
 123                 cmd = ['ssh', '-G', 'hg.mozilla.org']
 124                 output = subprocess.check_output(cmd, universal_newlines=True).splitlines()
 125                 address = [l.rsplit(' ', 1)[-1] for l in output if l.startswith('user')][0]
 126                 if not address.endswith('@mozilla.com'):
 127                     print(dedent("""\
 128                         Pernosco requires a Mozilla e-mail address to view its reports. Please
 129                         push to try with an @mozilla.com address to use --pernosco.
 130
 131                             Current user: {}
 132                     """.format(address)))
 133                     sys.exit(1)
 134
 135             except (subprocess.CalledProcessError, IndexError):
 136                 print("warning: failed to detect current user for 'hg.mozilla.org'")
 137                 print("Pernosco requires a Mozilla e-mail address to view its reports.")
 138                 while True:
 139                     answer = input("Do you have an @mozilla.com address? [Y/n]: ").lower()
 140                     if answer == 'n':
 141                         sys.exit(1)
 142                     elif answer == 'y':
 143                         break
 144
 145         return {
 146             'env': {
 147                 'PERNOSCO': str(int(pernosco)),
 148             }
 149         }
 150
 151     def validate(self, **kwargs):
 152         if kwargs['try_config'].get('use-artifact-builds'):
 153             print("Pernosco does not support artifact builds at this time. "
 154                   "Please try again with '--no-artifact'.")
 155             sys.exit(1)
 156
 157
 158 class Path(TryConfig):
 159
 160     arguments = [
 161         [['paths'],
 162          {'nargs': '*',
 163           'default': [],
 164           'help': 'Run tasks containing tests under the specified path(s).',
 165           }],
 166     ]
 167
 168     def try_config(self, paths, **kwargs):
 169         if not paths:
 170             return
 171
 172         for p in paths:
 173             if not os.path.exists(p):
 174                 print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
 175                 sys.exit(1)
 176
 177         paths = [mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) for p in paths]
 178         return {
 179             'env': {
 180                 'MOZHARNESS_TEST_PATHS': six.ensure_text(
 181                     json.dumps(resolve_tests_by_suite(paths))),
 182             }
 183         }
 184
 185
 186 class Environment(TryConfig):
 187
 188     arguments = [
 189         [['--env'],
 190          {'action': 'append',
 191           'default': None,
 192           'help': 'Set an environment variable, of the form FOO=BAR. '
 193                   'Can be passed in multiple times.',
 194           }],
 195     ]
 196
 197     def try_config(self, env, **kwargs):
 198         if not env:
 199             return
 200         return {
 201             'env': dict(e.split('=', 1) for e in env),
 202         }
 203
 204
 205 class RangeAction(Action):
 206     def __init__(self, min, max, *args, **kwargs):
 207         self.min = min
 208         self.max = max
 209         kwargs['metavar'] = '[{}-{}]'.format(self.min, self.max)
 210         super(RangeAction, self).__init__(*args, **kwargs)
 211
 212     def __call__(self, parser, namespace, values, option_string=None):
 213         name = option_string or self.dest
 214         if values < self.min:
 215             parser.error('{} can not be less than {}'.format(name, self.min))
 216         if values > self.max:
 217             parser.error('{} can not be more than {}'.format(name, self.max))
 218         setattr(namespace, self.dest, values)
 219
 220
 221 class Rebuild(TryConfig):
 222
 223     arguments = [
 224         [['--rebuild'],
 225          {'action': RangeAction,
 226           'min': 2,
 227           'max': 20,
 228           'default': None,
 229           'type': int,
 230           'help': 'Rebuild all selected tasks the specified number of times.',
 231           }],
 232     ]
 233
 234     def try_config(self, rebuild, **kwargs):
 235         if not rebuild:
 236             return
 237
 238         if kwargs.get('full') and rebuild > 3:
 239             print('warning: limiting --rebuild to 3 when using --full. '
 240                   'Use custom push actions to add more.')
 241             rebuild = 3
 242
 243         return {
 244             'rebuild': rebuild,
 245         }
 246
 247
 248 class Routes(TryConfig):
 249     arguments = [
 250         [
 251             ["--route"],
 252             {
 253                 "action": "append",
 254                 "dest": "routes",
 255                 "help": (
 256                     "Additional route to add to the tasks "
 257                     "(note: these will not be added to the decision task)"
 258                 ),
 259             },
 260         ],
 261     ]
 262
 263     def try_config(self, routes, **kwargs):
 264         if routes:
 265             return {
 266                 'routes': routes,
 267             }
 268
 269
 270 class ChemspillPrio(TryConfig):
 271
 272     arguments = [
 273         [['--chemspill-prio'],
 274          {'action': 'store_true',
 275           'help': 'Run at a higher priority than most try jobs (chemspills only).',
 276           }],
 277     ]
 278
 279     def try_config(self, chemspill_prio, **kwargs):
 280         if chemspill_prio:
 281             return {
 282                 'chemspill-prio': {}
 283             }
 284
 285
 286 class GeckoProfile(TryConfig):
 287     arguments = [
 288         [['--gecko-profile'],
 289          {'dest': 'profile',
 290           'action': 'store_true',
 291           'default': False,
 292           'help': 'Create and upload a gecko profile during talos/raptor tasks.',
 293           }],
 294         # For backwards compatibility
 295         [['--talos-profile'],
 296          {'dest': 'profile',
 297           'action': 'store_true',
 298           'default': False,
 299           'help': SUPPRESS,
 300           }],
 301         # This is added for consistency with the 'syntax' selector
 302         [['--geckoProfile'],
 303          {'dest': 'profile',
 304           'action': 'store_true',
 305           'default': False,
 306           'help': SUPPRESS,
 307           }],
 308     ]
 309
 310     def try_config(self, profile, **kwargs):
 311         if profile:
 312             return {
 313                 'gecko-profile': True,
 314             }
 315
 316
 317 class OptimizeStrategies(TryConfig):
 318
 319     arguments = [
 320         [['--strategy'],
 321          {'default': None,
 322           'help': 'Override the default optimization strategy. Valid values '
 323                   'are the experimental strategies defined at the bottom of '
 324                   '`taskcluster/taskgraph/optimize/__init__.py`.'
 325           }],
 326     ]
 327
 328     def try_config(self, strategy, **kwargs):
 329         if strategy:
 330             if ':' not in strategy:
 331                 strategy = "taskgraph.optimize:tryselect.{}".format(strategy)
 332
 333             try:
 334                 obj = find_object(strategy)
 335             except (ImportError, AttributeError):
 336                 print("error: invalid module path '{}'".format(strategy))
 337                 sys.exit(1)
 338
 339             if not isinstance(obj, dict):
 340                 print("error: object at '{}' must be a dict".format(strategy))
 341                 sys.exit(1)
 342
 343             return {
 344                 'optimize-strategies': strategy,
 345             }
 346
 347
 348 class Browsertime(TryConfig):
 349     arguments = [
 350         [['--browsertime'],
 351          {'action': 'store_true',
 352           'help': 'Use browsertime during Raptor tasks.',
 353           }],
 354     ]
 355
 356     def try_config(self, browsertime, **kwargs):
 357         if browsertime:
 358             return {
 359                 'browsertime': True,
 360             }
 361
 362
 363 class DisablePgo(TryConfig):
 364
 365     arguments = [
 366         [['--disable-pgo'],
 367          {'action': 'store_true',
 368           'help': 'Don\'t run PGO builds',
 369           }],
 370     ]
 371
 372     def try_config(self, disable_pgo, **kwargs):
 373         if disable_pgo:
 374             return {
 375                 'disable-pgo': True,
 376             }
 377
 378
 379 class WorkerOverrides(TryConfig):
 380
 381     arguments = [
 382         [
 383             ["--worker-override"],
 384             {
 385                 "action": "append",
 386                 "dest": "worker_overrides",
 387                 "help": (
 388                     "Override the worker pool used for a given taskgraph worker alias. "
 389                     "The argument should be `<alias>=<worker-pool>`. "
 390                     "Can be specified multiple times."
 391                 ),
 392             },
 393         ],
 394         [
 395             ["--worker-suffix"],
 396             {
 397                 "action": "append",
 398                 "dest": "worker_suffixes",
 399                 "help": (
 400                     "Override the worker pool used for a given taskgraph worker alias, "
 401                     "by appending a suffix to the work-pool. "
 402                     "The argument should be `<alias>=<suffix>`. "
 403                     "Can be specified multiple times."
 404                 ),
 405             },
 406         ],
 407     ]
 408
 409     def try_config(self, worker_overrides, worker_suffixes, **kwargs):
 410         from taskgraph.config import load_graph_config
 411         from taskgraph.util.workertypes import get_worker_type
 412
 413         overrides = {}
 414         if worker_overrides:
 415             for override in worker_overrides:
 416                 alias, worker_pool = override.split("=", 1)
 417                 if alias in overrides:
 418                     print(
 419                         "Can't override worker alias {alias} more than once. "
 420                         "Already set to use {previous}, but also asked to use {new}.".format(
 421                             alias=alias, previous=overrides[alias], new=worker_pool
 422                         )
 423                     )
 424                     sys.exit(1)
 425                 overrides[alias] = worker_pool
 426
 427         if worker_suffixes:
 428             root = build.topsrcdir
 429             root = os.path.join(root, "taskcluster", "ci")
 430             graph_config = load_graph_config(root)
 431             for worker_suffix in worker_suffixes:
 432                 alias, suffix = worker_suffix.split("=", 1)
 433                 if alias in overrides:
 434                     print(
 435                         "Can't override worker alias {alias} more than once. "
 436                         "Already set to use {previous}, but also asked "
 437                         "to add suffix {suffix}.".format(
 438                             alias=alias, previous=overrides[alias], suffix=suffix
 439                         )
 440                     )
 441                     sys.exit(1)
 442                 provisioner, worker_type = get_worker_type(
 443                     graph_config, alias, level="1", release_level="staging",
 444                 )
 445                 overrides[alias] = "{provisioner}/{worker_type}{suffix}".format(
 446                     provisioner=provisioner, worker_type=worker_type, suffix=suffix
 447                 )
 448
 449         if overrides:
 450             return {"worker-overrides": overrides}
 451
 452
 453 all_task_configs = {
 454     'artifact': Artifact,
 455     'browsertime': Browsertime,
 456     'chemspill-prio': ChemspillPrio,
 457     'disable-pgo': DisablePgo,
 458     'env': Environment,
 459     'gecko-profile': GeckoProfile,
 460     'path': Path,
 461     'pernosco': Pernosco,
 462     'rebuild': Rebuild,
 463     'routes': Routes,
 464     'strategy': OptimizeStrategies,
 465     'worker-overrides': WorkerOverrides,
 466 }