taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 from __future__ import absolute_import, print_function, unicode_literals
   9 import argparse
  10 import json
  11 import logging
  12 import os
  13 import sys
  14 import time
  15 import traceback
  16 from functools import partial
  17
  18 from mach.decorators import (
  19     Command,
  20     CommandArgument,
  21     SettingsProvider,
  22     SubCommand,
  23 )
  24
  25 import taskgraph.main
  26 from taskgraph.main import commands as taskgraph_commands
  27
  28 logger = logging.getLogger("taskcluster")
  29
  30
  31 @SettingsProvider
  32 class TaskgraphConfig(object):
  33     @classmethod
  34     def config_settings(cls):
  35         return [
  36             (
  37                 "taskgraph.diffcmd",
  38                 "string",
  39                 "The command to run with `./mach taskgraph --diff`",
  40                 "diff --report-identical-files "
  41                 "--label={attr}@{base} --label={attr}@{cur} -U20",
  42                 {},
  43             )
  44         ]
  45
  46
  47 def strtobool(value):
  48     """Convert string to boolean.
  49
  50     Wraps "distutils.util.strtobool", deferring the import of the package
  51     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  52     |mach bootstrap| would install the required package to enable "distutils.util", but
  53     it can't because mach fails to interpret this file.
  54     """
  55     from distutils.util import strtobool
  56
  57     return bool(strtobool(value))
  58
  59
  60 def get_taskgraph_command_parser(name):
  61     """Given a command name, obtain its argument parser.
  62
  63     Args:
  64         name (str): Name of the command.
  65
  66     Returns:
  67         ArgumentParser: An ArgumentParser instance.
  68     """
  69     command = taskgraph_commands[name]
  70     parser = argparse.ArgumentParser()
  71     for arg in command.func.args:
  72         parser.add_argument(*arg[0], **arg[1])
  73
  74     parser.set_defaults(func=command.func, **command.defaults)
  75     return parser
  76
  77
  78 def get_taskgraph_decision_parser():
  79     parser = get_taskgraph_command_parser("decision")
  80
  81     extra_args = [
  82         (
  83             ["--optimize-target-tasks"],
  84             {
  85                 "type": lambda flag: strtobool(flag),
  86                 "nargs": "?",
  87                 "const": "true",
  88                 "help": "If specified, this indicates whether the target "
  89                 "tasks are eligible for optimization. Otherwise, the default "
  90                 "for the project is used.",
  91             },
  92         ),
  93         (
  94             ["--include-push-tasks"],
  95             {
  96                 "action": "store_true",
  97                 "help": "Whether tasks from the on-push graph should be re-used "
  98                 "in this graph. This allows cron graphs to avoid rebuilding "
  99                 "jobs that were built on-push.",
 100             },
 101         ),
 102         (
 103             ["--rebuild-kind"],
 104             {
 105                 "dest": "rebuild_kinds",
 106                 "action": "append",
 107                 "default": argparse.SUPPRESS,
 108                 "help": "Kinds that should not be re-used from the on-push graph.",
 109             },
 110         ),
 111         (
 112             ["--comm-base-repository"],
 113             {
 114                 "required": False,
 115                 "help": "URL for 'base' comm-* repository to clone",
 116             },
 117         ),
 118         (
 119             ["--comm-head-repository"],
 120             {
 121                 "required": False,
 122                 "help": "URL for 'head' comm-* repository to fetch revision from",
 123             },
 124         ),
 125         (
 126             ["--comm-head-ref"],
 127             {
 128                 "required": False,
 129                 "help": "comm-* Reference (this is same as rev usually for hg)",
 130             },
 131         ),
 132         (
 133             ["--comm-head-rev"],
 134             {
 135                 "required": False,
 136                 "help": "Commit revision to use from head comm-* repository",
 137             },
 138         ),
 139     ]
 140     for arg in extra_args:
 141         parser.add_argument(*arg[0], **arg[1])
 142
 143     return parser
 144
 145
 146 @Command(
 147     "taskgraph",
 148     category="ci",
 149     description="Manipulate TaskCluster task graphs defined in-tree",
 150 )
 151 def taskgraph_command(command_context):
 152     """The taskgraph subcommands all relate to the generation of task graphs
 153     for Gecko continuous integration.  A task graph is a set of tasks linked
 154     by dependencies: for example, a binary must be built before it is tested,
 155     and that build may further depend on various toolchains, libraries, etc.
 156     """
 157
 158
 159 @SubCommand(
 160     "taskgraph",
 161     "tasks",
 162     description="Show all tasks in the taskgraph",
 163     parser=partial(get_taskgraph_command_parser, "tasks"),
 164 )
 165 def taskgraph_tasks(command_context, **options):
 166     return run_show_taskgraph(command_context, **options)
 167
 168
 169 @SubCommand(
 170     "taskgraph",
 171     "full",
 172     description="Show the full taskgraph",
 173     parser=partial(get_taskgraph_command_parser, "full"),
 174 )
 175 def taskgraph_full(command_context, **options):
 176     return run_show_taskgraph(command_context, **options)
 177
 178
 179 @SubCommand(
 180     "taskgraph",
 181     "target",
 182     description="Show the target task set",
 183     parser=partial(get_taskgraph_command_parser, "target"),
 184 )
 185 def taskgraph_target(command_context, **options):
 186     return run_show_taskgraph(command_context, **options)
 187
 188
 189 @SubCommand(
 190     "taskgraph",
 191     "target-graph",
 192     description="Show the target taskgraph",
 193     parser=partial(get_taskgraph_command_parser, "target-graph"),
 194 )
 195 def taskgraph_target_graph(command_context, **options):
 196     return run_show_taskgraph(command_context, **options)
 197
 198
 199 @SubCommand(
 200     "taskgraph",
 201     "optimized",
 202     description="Show the optimized taskgraph",
 203     parser=partial(get_taskgraph_command_parser, "optimized"),
 204 )
 205 def taskgraph_optimized(command_context, **options):
 206     return run_show_taskgraph(command_context, **options)
 207
 208
 209 @SubCommand(
 210     "taskgraph",
 211     "morphed",
 212     description="Show the morphed taskgraph",
 213     parser=partial(get_taskgraph_command_parser, "morphed"),
 214 )
 215 def taskgraph_morphed(command_context, **options):
 216     return run_show_taskgraph(command_context, **options)
 217
 218
 219 def run_show_taskgraph(command_context, **options):
 220     # There are cases where we don't want to set up mach logging (e.g logs
 221     # are being redirected to disk). By monkeypatching the 'setup_logging'
 222     # function we can let 'taskgraph.main' decide whether or not to log to
 223     # the terminal.
 224     taskgraph.main.setup_logging = partial(
 225         setup_logging,
 226         command_context,
 227         quiet=options["quiet"],
 228         verbose=options["verbose"],
 229     )
 230     show_taskgraph = options.pop("func")
 231     return show_taskgraph(options)
 232
 233
 234 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
 235 @CommandArgument(
 236     "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
 237 )
 238 @CommandArgument(
 239     "--quiet", "-q", action="store_true", help="suppress all logging output"
 240 )
 241 @CommandArgument(
 242     "--verbose",
 243     "-v",
 244     action="store_true",
 245     help="include debug-level logging output",
 246 )
 247 @CommandArgument(
 248     "--parameters",
 249     "-p",
 250     default="project=mozilla-central",
 251     help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
 252 )
 253 def taskgraph_actions(command_context, **options):
 254     return show_actions(command_context, options)
 255
 256
 257 @SubCommand(
 258     "taskgraph",
 259     "decision",
 260     description="Run the decision task",
 261     parser=get_taskgraph_decision_parser,
 262 )
 263 def taskgraph_decision(command_context, **options):
 264     """Run the decision task: generate a task graph and submit to
 265     TaskCluster.  This is only meant to be called within decision tasks,
 266     and requires a great many arguments.  Commands like `mach taskgraph
 267     optimized` are better suited to use on the command line, and can take
 268     the parameters file generated by a decision task."""
 269     try:
 270         setup_logging(command_context)
 271         start = time.monotonic()
 272         ret = taskgraph_commands["decision"].func(options)
 273         end = time.monotonic()
 274         if os.environ.get("MOZ_AUTOMATION") == "1":
 275             perfherder_data = {
 276                 "framework": {"name": "build_metrics"},
 277                 "suites": [
 278                     {
 279                         "name": "decision",
 280                         "value": end - start,
 281                         "lowerIsBetter": True,
 282                         "shouldAlert": True,
 283                         "subtests": [],
 284                     }
 285                 ],
 286             }
 287             print(
 288                 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
 289                 file=sys.stderr,
 290             )
 291         return ret
 292     except Exception:
 293         traceback.print_exc()
 294         sys.exit(1)
 295
 296
 297 @SubCommand(
 298     "taskgraph",
 299     "cron",
 300     description="Provide a pointer to the new `.cron.yml` handler.",
 301 )
 302 def taskgraph_cron(command_context, **options):
 303     print(
 304         'Handling of ".cron.yml" files has move to '
 305         "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 306     )
 307     sys.exit(1)
 308
 309
 310 @SubCommand(
 311     "taskgraph",
 312     "action-callback",
 313     description="Run action callback used by action tasks",
 314     parser=partial(get_taskgraph_command_parser, "action-callback"),
 315 )
 316 def action_callback(command_context, **options):
 317     setup_logging(command_context)
 318     taskgraph_commands["action-callback"].func(options)
 319
 320
 321 @SubCommand(
 322     "taskgraph",
 323     "test-action-callback",
 324     description="Run an action callback in a testing mode",
 325     parser=partial(get_taskgraph_command_parser, "test-action-callback"),
 326 )
 327 def test_action_callback(command_context, **options):
 328     setup_logging(command_context)
 329
 330     if not options["parameters"]:
 331         options["parameters"] = "project=mozilla-central"
 332
 333     taskgraph_commands["test-action-callback"].func(options)
 334
 335
 336 def setup_logging(command_context, quiet=False, verbose=True):
 337     """
 338     Set up Python logging for all loggers, sending results to stderr (so
 339     that command output can be redirected easily) and adding the typical
 340     mach timestamp.
 341     """
 342     # remove the old terminal handler
 343     old = command_context.log_manager.replace_terminal_handler(None)
 344
 345     # re-add it, with level and fh set appropriately
 346     if not quiet:
 347         level = logging.DEBUG if verbose else logging.INFO
 348         command_context.log_manager.add_terminal_logging(
 349             fh=sys.stderr,
 350             level=level,
 351             write_interval=old.formatter.write_interval,
 352             write_times=old.formatter.write_times,
 353         )
 354
 355     # all of the taskgraph logging is unstructured logging
 356     command_context.log_manager.enable_unstructured()
 357
 358
 359 def show_actions(command_context, options):
 360     import taskgraph
 361     import taskgraph.actions
 362     import taskgraph.generator
 363     import taskgraph.parameters
 364
 365     try:
 366         setup_logging(
 367             command_context, quiet=options["quiet"], verbose=options["verbose"]
 368         )
 369         parameters = taskgraph.parameters.parameters_loader(options["parameters"])
 370
 371         tgg = taskgraph.generator.TaskGraphGenerator(
 372             root_dir=options.get("root"),
 373             parameters=parameters,
 374         )
 375
 376         actions = taskgraph.actions.render_actions_json(
 377             tgg.parameters,
 378             tgg.graph_config,
 379             decision_task_id="DECISION-TASK",
 380         )
 381         print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
 382     except Exception:
 383         traceback.print_exc()
 384         sys.exit(1)
 385
 386
 387 @Command(
 388     "taskcluster-load-image",
 389     category="ci",
 390     description="Load a pre-built Docker image. Note that you need to "
 391     "have docker installed and running for this to work.",
 392     parser=partial(get_taskgraph_command_parser, "load-image"),
 393 )
 394 def load_image(command_context, **kwargs):
 395     taskgraph_commands["load-image"].func(kwargs)
 396
 397
 398 @Command(
 399     "taskcluster-build-image",
 400     category="ci",
 401     description="Build a Docker image",
 402     parser=partial(get_taskgraph_command_parser, "build-image"),
 403 )
 404 def build_image(command_context, **kwargs):
 405     try:
 406         taskgraph_commands["build-image"].func(kwargs)
 407     except Exception:
 408         traceback.print_exc()
 409         sys.exit(1)
 410
 411
 412 @Command(
 413     "taskcluster-image-digest",
 414     category="ci",
 415     description="Print the digest of the image of this name based on the "
 416     "current contents of the tree.",
 417     parser=partial(get_taskgraph_command_parser, "build-image"),
 418 )
 419 def image_digest(command_context, **kwargs):
 420     taskgraph_commands["image-digest"].func(kwargs)
 421
 422
 423 @Command(
 424     "release-history",
 425     category="ci",
 426     description="Query balrog for release history used by enable partials generation",
 427 )
 428 @CommandArgument(
 429     "-b",
 430     "--branch",
 431     help="The gecko project branch used in balrog, such as "
 432     "mozilla-central, release, maple",
 433 )
 434 @CommandArgument(
 435     "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
 436 )
 437 def generate_partials_builds(command_context, product, branch):
 438     from taskgraph.util.partials import populate_release_history
 439
 440     try:
 441         import yaml
 442
 443         release_history = {"release_history": populate_release_history(product, branch)}
 444         print(
 445             yaml.safe_dump(
 446                 release_history, allow_unicode=True, default_flow_style=False
 447             )
 448         )
 449     except Exception:
 450         traceback.print_exc()
 451         sys.exit(1)