taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14 import traceback
  15 from functools import partial
  16
  17 import gecko_taskgraph.main
  18 from gecko_taskgraph.main import commands as taskgraph_commands
  19 from mach.decorators import Command, CommandArgument, SubCommand
  20
  21 logger = logging.getLogger("taskcluster")
  22
  23
  24 def strtobool(value):
  25     """Convert string to boolean.
  26
  27     Wraps "distutils.util.strtobool", deferring the import of the package
  28     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  29     |mach bootstrap| would install the required package to enable "distutils.util", but
  30     it can't because mach fails to interpret this file.
  31     """
  32     from distutils.util import strtobool
  33
  34     return bool(strtobool(value))
  35
  36
  37 def get_taskgraph_command_parser(name):
  38     """Given a command name, obtain its argument parser.
  39
  40     Args:
  41         name (str): Name of the command.
  42
  43     Returns:
  44         ArgumentParser: An ArgumentParser instance.
  45     """
  46     command = taskgraph_commands[name]
  47     parser = argparse.ArgumentParser()
  48     for arg in command.func.args:
  49         parser.add_argument(*arg[0], **arg[1])
  50
  51     parser.set_defaults(func=command.func, **command.defaults)
  52     return parser
  53
  54
  55 def get_taskgraph_decision_parser():
  56     parser = get_taskgraph_command_parser("decision")
  57
  58     extra_args = [
  59         (
  60             ["--optimize-target-tasks"],
  61             {
  62                 "type": lambda flag: strtobool(flag),
  63                 "nargs": "?",
  64                 "const": "true",
  65                 "help": "If specified, this indicates whether the target "
  66                 "tasks are eligible for optimization. Otherwise, the default "
  67                 "for the project is used.",
  68             },
  69         ),
  70         (
  71             ["--include-push-tasks"],
  72             {
  73                 "action": "store_true",
  74                 "help": "Whether tasks from the on-push graph should be re-used "
  75                 "in this graph. This allows cron graphs to avoid rebuilding "
  76                 "jobs that were built on-push.",
  77             },
  78         ),
  79         (
  80             ["--rebuild-kind"],
  81             {
  82                 "dest": "rebuild_kinds",
  83                 "action": "append",
  84                 "default": argparse.SUPPRESS,
  85                 "help": "Kinds that should not be re-used from the on-push graph.",
  86             },
  87         ),
  88     ]
  89     for arg in extra_args:
  90         parser.add_argument(*arg[0], **arg[1])
  91
  92     return parser
  93
  94
  95 @Command(
  96     "taskgraph",
  97     category="ci",
  98     description="Manipulate TaskCluster task graphs defined in-tree",
  99 )
 100 def taskgraph_command(command_context):
 101     """The taskgraph subcommands all relate to the generation of task graphs
 102     for Gecko continuous integration.  A task graph is a set of tasks linked
 103     by dependencies: for example, a binary must be built before it is tested,
 104     and that build may further depend on various toolchains, libraries, etc.
 105     """
 106
 107
 108 @SubCommand(
 109     "taskgraph",
 110     "tasks",
 111     description="Show all tasks in the taskgraph",
 112     parser=partial(get_taskgraph_command_parser, "tasks"),
 113 )
 114 def taskgraph_tasks(command_context, **options):
 115     return run_show_taskgraph(command_context, **options)
 116
 117
 118 @SubCommand(
 119     "taskgraph",
 120     "full",
 121     description="Show the full taskgraph",
 122     parser=partial(get_taskgraph_command_parser, "full"),
 123 )
 124 def taskgraph_full(command_context, **options):
 125     return run_show_taskgraph(command_context, **options)
 126
 127
 128 @SubCommand(
 129     "taskgraph",
 130     "target",
 131     description="Show the target task set",
 132     parser=partial(get_taskgraph_command_parser, "target"),
 133 )
 134 def taskgraph_target(command_context, **options):
 135     return run_show_taskgraph(command_context, **options)
 136
 137
 138 @SubCommand(
 139     "taskgraph",
 140     "target-graph",
 141     description="Show the target taskgraph",
 142     parser=partial(get_taskgraph_command_parser, "target-graph"),
 143 )
 144 def taskgraph_target_graph(command_context, **options):
 145     return run_show_taskgraph(command_context, **options)
 146
 147
 148 @SubCommand(
 149     "taskgraph",
 150     "optimized",
 151     description="Show the optimized taskgraph",
 152     parser=partial(get_taskgraph_command_parser, "optimized"),
 153 )
 154 def taskgraph_optimized(command_context, **options):
 155     return run_show_taskgraph(command_context, **options)
 156
 157
 158 @SubCommand(
 159     "taskgraph",
 160     "morphed",
 161     description="Show the morphed taskgraph",
 162     parser=partial(get_taskgraph_command_parser, "morphed"),
 163 )
 164 def taskgraph_morphed(command_context, **options):
 165     return run_show_taskgraph(command_context, **options)
 166
 167
 168 def run_show_taskgraph(command_context, **options):
 169     # There are cases where we don't want to set up mach logging (e.g logs
 170     # are being redirected to disk). By monkeypatching the 'setup_logging'
 171     # function we can let 'taskgraph.main' decide whether or not to log to
 172     # the terminal.
 173     gecko_taskgraph.main.setup_logging = partial(
 174         setup_logging,
 175         command_context,
 176         quiet=options["quiet"],
 177         verbose=options["verbose"],
 178     )
 179     show_taskgraph = options.pop("func")
 180     return show_taskgraph(options)
 181
 182
 183 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
 184 @CommandArgument(
 185     "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
 186 )
 187 @CommandArgument(
 188     "--quiet", "-q", action="store_true", help="suppress all logging output"
 189 )
 190 @CommandArgument(
 191     "--verbose",
 192     "-v",
 193     action="store_true",
 194     help="include debug-level logging output",
 195 )
 196 @CommandArgument(
 197     "--parameters",
 198     "-p",
 199     default="project=mozilla-central",
 200     help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
 201 )
 202 def taskgraph_actions(command_context, **options):
 203     return show_actions(command_context, options)
 204
 205
 206 @SubCommand(
 207     "taskgraph",
 208     "decision",
 209     description="Run the decision task",
 210     parser=get_taskgraph_decision_parser,
 211 )
 212 def taskgraph_decision(command_context, **options):
 213     """Run the decision task: generate a task graph and submit to
 214     TaskCluster.  This is only meant to be called within decision tasks,
 215     and requires a great many arguments.  Commands like `mach taskgraph
 216     optimized` are better suited to use on the command line, and can take
 217     the parameters file generated by a decision task."""
 218     try:
 219         setup_logging(command_context)
 220         start = time.monotonic()
 221         ret = taskgraph_commands["decision"].func(options)
 222         end = time.monotonic()
 223         if os.environ.get("MOZ_AUTOMATION") == "1":
 224             perfherder_data = {
 225                 "framework": {"name": "build_metrics"},
 226                 "suites": [
 227                     {
 228                         "name": "decision",
 229                         "value": end - start,
 230                         "lowerIsBetter": True,
 231                         "shouldAlert": True,
 232                         "subtests": [],
 233                     }
 234                 ],
 235             }
 236             print(
 237                 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
 238                 file=sys.stderr,
 239             )
 240         return ret
 241     except Exception:
 242         traceback.print_exc()
 243         sys.exit(1)
 244
 245
 246 @SubCommand(
 247     "taskgraph",
 248     "cron",
 249     description="Provide a pointer to the new `.cron.yml` handler.",
 250 )
 251 def taskgraph_cron(command_context, **options):
 252     print(
 253         'Handling of ".cron.yml" files has move to '
 254         "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 255     )
 256     sys.exit(1)
 257
 258
 259 @SubCommand(
 260     "taskgraph",
 261     "action-callback",
 262     description="Run action callback used by action tasks",
 263     parser=partial(get_taskgraph_command_parser, "action-callback"),
 264 )
 265 def action_callback(command_context, **options):
 266     setup_logging(command_context)
 267     taskgraph_commands["action-callback"].func(options)
 268
 269
 270 @SubCommand(
 271     "taskgraph",
 272     "test-action-callback",
 273     description="Run an action callback in a testing mode",
 274     parser=partial(get_taskgraph_command_parser, "test-action-callback"),
 275 )
 276 def test_action_callback(command_context, **options):
 277     setup_logging(command_context)
 278
 279     if not options["parameters"]:
 280         options["parameters"] = "project=mozilla-central"
 281
 282     taskgraph_commands["test-action-callback"].func(options)
 283
 284
 285 def setup_logging(command_context, quiet=False, verbose=True):
 286     """
 287     Set up Python logging for all loggers, sending results to stderr (so
 288     that command output can be redirected easily) and adding the typical
 289     mach timestamp.
 290     """
 291     # remove the old terminal handler
 292     old = command_context.log_manager.replace_terminal_handler(None)
 293
 294     # re-add it, with level and fh set appropriately
 295     if not quiet:
 296         level = logging.DEBUG if verbose else logging.INFO
 297         command_context.log_manager.add_terminal_logging(
 298             fh=sys.stderr,
 299             level=level,
 300             write_interval=old.formatter.write_interval,
 301             write_times=old.formatter.write_times,
 302         )
 303
 304     # all of the taskgraph logging is unstructured logging
 305     command_context.log_manager.enable_unstructured()
 306
 307
 308 def show_actions(command_context, options):
 309     import gecko_taskgraph
 310     import gecko_taskgraph.actions
 311     from taskgraph.generator import TaskGraphGenerator
 312     from taskgraph.parameters import parameters_loader
 313
 314     try:
 315         setup_logging(
 316             command_context, quiet=options["quiet"], verbose=options["verbose"]
 317         )
 318         parameters = parameters_loader(options["parameters"])
 319
 320         tgg = TaskGraphGenerator(
 321             root_dir=options.get("root"),
 322             parameters=parameters,
 323         )
 324
 325         actions = gecko_taskgraph.actions.render_actions_json(
 326             tgg.parameters,
 327             tgg.graph_config,
 328             decision_task_id="DECISION-TASK",
 329         )
 330         print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
 331     except Exception:
 332         traceback.print_exc()
 333         sys.exit(1)
 334
 335
 336 @Command(
 337     "taskcluster-load-image",
 338     category="ci",
 339     description="Load a pre-built Docker image. Note that you need to "
 340     "have docker installed and running for this to work.",
 341     parser=partial(get_taskgraph_command_parser, "load-image"),
 342 )
 343 def load_image(command_context, **kwargs):
 344     taskgraph_commands["load-image"].func(kwargs)
 345
 346
 347 @Command(
 348     "taskcluster-build-image",
 349     category="ci",
 350     description="Build a Docker image",
 351     parser=partial(get_taskgraph_command_parser, "build-image"),
 352 )
 353 def build_image(command_context, **kwargs):
 354     try:
 355         taskgraph_commands["build-image"].func(kwargs)
 356     except Exception:
 357         traceback.print_exc()
 358         sys.exit(1)
 359
 360
 361 @Command(
 362     "taskcluster-image-digest",
 363     category="ci",
 364     description="Print the digest of the image of this name based on the "
 365     "current contents of the tree.",
 366     parser=partial(get_taskgraph_command_parser, "build-image"),
 367 )
 368 def image_digest(command_context, **kwargs):
 369     taskgraph_commands["image-digest"].func(kwargs)
 370
 371
 372 @Command(
 373     "release-history",
 374     category="ci",
 375     description="Query balrog for release history used by enable partials generation",
 376 )
 377 @CommandArgument(
 378     "-b",
 379     "--branch",
 380     help="The gecko project branch used in balrog, such as "
 381     "mozilla-central, release, maple",
 382 )
 383 @CommandArgument(
 384     "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
 385 )
 386 def generate_partials_builds(command_context, product, branch):
 387     from gecko_taskgraph.util.partials import populate_release_history
 388
 389     try:
 390         import yaml
 391
 392         release_history = {"release_history": populate_release_history(product, branch)}
 393         print(
 394             yaml.safe_dump(
 395                 release_history, allow_unicode=True, default_flow_style=False
 396             )
 397         )
 398     except Exception:
 399         traceback.print_exc()
 400         sys.exit(1)