taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14 import traceback
  15 from functools import partial
  16
  17 import gecko_taskgraph.main
  18 from gecko_taskgraph.main import commands as taskgraph_commands
  19 from mach.decorators import Command, CommandArgument, SubCommand
  20 from mach.util import strtobool
  21
  22 logger = logging.getLogger("taskcluster")
  23
  24
  25 def get_taskgraph_command_parser(name):
  26     """Given a command name, obtain its argument parser.
  27
  28     Args:
  29         name (str): Name of the command.
  30
  31     Returns:
  32         ArgumentParser: An ArgumentParser instance.
  33     """
  34     command = taskgraph_commands[name]
  35     parser = argparse.ArgumentParser()
  36     for arg in command.func.args:
  37         parser.add_argument(*arg[0], **arg[1])
  38
  39     parser.set_defaults(func=command.func, **command.defaults)
  40     return parser
  41
  42
  43 def get_taskgraph_decision_parser():
  44     parser = get_taskgraph_command_parser("decision")
  45
  46     extra_args = [
  47         (
  48             ["--optimize-target-tasks"],
  49             {
  50                 "type": lambda flag: bool(strtobool(flag)),
  51                 "nargs": "?",
  52                 "const": "true",
  53                 "help": "If specified, this indicates whether the target "
  54                 "tasks are eligible for optimization. Otherwise, the default "
  55                 "for the project is used.",
  56             },
  57         ),
  58         (
  59             ["--include-push-tasks"],
  60             {
  61                 "action": "store_true",
  62                 "help": "Whether tasks from the on-push graph should be re-used "
  63                 "in this graph. This allows cron graphs to avoid rebuilding "
  64                 "jobs that were built on-push.",
  65             },
  66         ),
  67         (
  68             ["--rebuild-kind"],
  69             {
  70                 "dest": "rebuild_kinds",
  71                 "action": "append",
  72                 "default": argparse.SUPPRESS,
  73                 "help": "Kinds that should not be re-used from the on-push graph.",
  74             },
  75         ),
  76     ]
  77     for arg in extra_args:
  78         parser.add_argument(*arg[0], **arg[1])
  79
  80     return parser
  81
  82
  83 @Command(
  84     "taskgraph",
  85     category="ci",
  86     description="Manipulate TaskCluster task graphs defined in-tree",
  87 )
  88 def taskgraph_command(command_context):
  89     """The taskgraph subcommands all relate to the generation of task graphs
  90     for Gecko continuous integration.  A task graph is a set of tasks linked
  91     by dependencies: for example, a binary must be built before it is tested,
  92     and that build may further depend on various toolchains, libraries, etc.
  93     """
  94
  95
  96 @SubCommand(
  97     "taskgraph",
  98     "tasks",
  99     description="Show all tasks in the taskgraph",
 100     parser=partial(get_taskgraph_command_parser, "tasks"),
 101 )
 102 def taskgraph_tasks(command_context, **options):
 103     return run_show_taskgraph(command_context, **options)
 104
 105
 106 @SubCommand(
 107     "taskgraph",
 108     "full",
 109     description="Show the full taskgraph",
 110     parser=partial(get_taskgraph_command_parser, "full"),
 111 )
 112 def taskgraph_full(command_context, **options):
 113     return run_show_taskgraph(command_context, **options)
 114
 115
 116 @SubCommand(
 117     "taskgraph",
 118     "target",
 119     description="Show the target task set",
 120     parser=partial(get_taskgraph_command_parser, "target"),
 121 )
 122 def taskgraph_target(command_context, **options):
 123     return run_show_taskgraph(command_context, **options)
 124
 125
 126 @SubCommand(
 127     "taskgraph",
 128     "target-graph",
 129     description="Show the target taskgraph",
 130     parser=partial(get_taskgraph_command_parser, "target-graph"),
 131 )
 132 def taskgraph_target_graph(command_context, **options):
 133     return run_show_taskgraph(command_context, **options)
 134
 135
 136 @SubCommand(
 137     "taskgraph",
 138     "optimized",
 139     description="Show the optimized taskgraph",
 140     parser=partial(get_taskgraph_command_parser, "optimized"),
 141 )
 142 def taskgraph_optimized(command_context, **options):
 143     return run_show_taskgraph(command_context, **options)
 144
 145
 146 @SubCommand(
 147     "taskgraph",
 148     "morphed",
 149     description="Show the morphed taskgraph",
 150     parser=partial(get_taskgraph_command_parser, "morphed"),
 151 )
 152 def taskgraph_morphed(command_context, **options):
 153     return run_show_taskgraph(command_context, **options)
 154
 155
 156 def run_show_taskgraph(command_context, **options):
 157     # There are cases where we don't want to set up mach logging (e.g logs
 158     # are being redirected to disk). By monkeypatching the 'setup_logging'
 159     # function we can let 'taskgraph.main' decide whether or not to log to
 160     # the terminal.
 161     gecko_taskgraph.main.setup_logging = partial(
 162         setup_logging,
 163         command_context,
 164         quiet=options["quiet"],
 165         verbose=options["verbose"],
 166     )
 167     show_taskgraph = options.pop("func")
 168     return show_taskgraph(options)
 169
 170
 171 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
 172 @CommandArgument(
 173     "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
 174 )
 175 @CommandArgument(
 176     "--quiet", "-q", action="store_true", help="suppress all logging output"
 177 )
 178 @CommandArgument(
 179     "--verbose",
 180     "-v",
 181     action="store_true",
 182     help="include debug-level logging output",
 183 )
 184 @CommandArgument(
 185     "--parameters",
 186     "-p",
 187     default="project=mozilla-central",
 188     help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
 189 )
 190 def taskgraph_actions(command_context, **options):
 191     return show_actions(command_context, options)
 192
 193
 194 @SubCommand(
 195     "taskgraph",
 196     "decision",
 197     description="Run the decision task",
 198     parser=get_taskgraph_decision_parser,
 199 )
 200 def taskgraph_decision(command_context, **options):
 201     """Run the decision task: generate a task graph and submit to
 202     TaskCluster.  This is only meant to be called within decision tasks,
 203     and requires a great many arguments.  Commands like `mach taskgraph
 204     optimized` are better suited to use on the command line, and can take
 205     the parameters file generated by a decision task."""
 206     try:
 207         setup_logging(command_context)
 208         start = time.monotonic()
 209         ret = taskgraph_commands["decision"].func(options)
 210         end = time.monotonic()
 211         if os.environ.get("MOZ_AUTOMATION") == "1":
 212             perfherder_data = {
 213                 "framework": {"name": "build_metrics"},
 214                 "suites": [
 215                     {
 216                         "name": "decision",
 217                         "value": end - start,
 218                         "lowerIsBetter": True,
 219                         "shouldAlert": True,
 220                         "subtests": [],
 221                     }
 222                 ],
 223             }
 224             print(
 225                 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
 226                 file=sys.stderr,
 227             )
 228         return ret
 229     except Exception:
 230         traceback.print_exc()
 231         sys.exit(1)
 232
 233
 234 @SubCommand(
 235     "taskgraph",
 236     "cron",
 237     description="Provide a pointer to the new `.cron.yml` handler.",
 238 )
 239 def taskgraph_cron(command_context, **options):
 240     print(
 241         'Handling of ".cron.yml" files has move to '
 242         "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 243     )
 244     sys.exit(1)
 245
 246
 247 @SubCommand(
 248     "taskgraph",
 249     "action-callback",
 250     description="Run action callback used by action tasks",
 251     parser=partial(get_taskgraph_command_parser, "action-callback"),
 252 )
 253 def action_callback(command_context, **options):
 254     setup_logging(command_context)
 255     taskgraph_commands["action-callback"].func(options)
 256
 257
 258 @SubCommand(
 259     "taskgraph",
 260     "test-action-callback",
 261     description="Run an action callback in a testing mode",
 262     parser=partial(get_taskgraph_command_parser, "test-action-callback"),
 263 )
 264 def test_action_callback(command_context, **options):
 265     setup_logging(command_context)
 266
 267     if not options["parameters"]:
 268         options["parameters"] = "project=mozilla-central"
 269
 270     taskgraph_commands["test-action-callback"].func(options)
 271
 272
 273 def setup_logging(command_context, quiet=False, verbose=True):
 274     """
 275     Set up Python logging for all loggers, sending results to stderr (so
 276     that command output can be redirected easily) and adding the typical
 277     mach timestamp.
 278     """
 279     # remove the old terminal handler
 280     old = command_context.log_manager.replace_terminal_handler(None)
 281
 282     # re-add it, with level and fh set appropriately
 283     if not quiet:
 284         level = logging.DEBUG if verbose else logging.INFO
 285         command_context.log_manager.add_terminal_logging(
 286             fh=sys.stderr,
 287             level=level,
 288             write_interval=old.formatter.write_interval,
 289             write_times=old.formatter.write_times,
 290         )
 291
 292     # all of the taskgraph logging is unstructured logging
 293     command_context.log_manager.enable_unstructured()
 294
 295
 296 def show_actions(command_context, options):
 297     import gecko_taskgraph
 298     import gecko_taskgraph.actions
 299     from taskgraph.generator import TaskGraphGenerator
 300     from taskgraph.parameters import parameters_loader
 301
 302     try:
 303         setup_logging(
 304             command_context, quiet=options["quiet"], verbose=options["verbose"]
 305         )
 306         parameters = parameters_loader(options["parameters"])
 307
 308         tgg = TaskGraphGenerator(
 309             root_dir=options.get("root"),
 310             parameters=parameters,
 311         )
 312
 313         actions = gecko_taskgraph.actions.render_actions_json(
 314             tgg.parameters,
 315             tgg.graph_config,
 316             decision_task_id="DECISION-TASK",
 317         )
 318         print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
 319     except Exception:
 320         traceback.print_exc()
 321         sys.exit(1)
 322
 323
 324 @Command(
 325     "taskcluster-load-image",
 326     category="ci",
 327     description="Load a pre-built Docker image. Note that you need to "
 328     "have docker installed and running for this to work.",
 329     parser=partial(get_taskgraph_command_parser, "load-image"),
 330 )
 331 def load_image(command_context, **kwargs):
 332     taskgraph_commands["load-image"].func(kwargs)
 333
 334
 335 @Command(
 336     "taskcluster-build-image",
 337     category="ci",
 338     description="Build a Docker image",
 339     parser=partial(get_taskgraph_command_parser, "build-image"),
 340 )
 341 def build_image(command_context, **kwargs):
 342     try:
 343         taskgraph_commands["build-image"].func(kwargs)
 344     except Exception:
 345         traceback.print_exc()
 346         sys.exit(1)
 347
 348
 349 @Command(
 350     "taskcluster-image-digest",
 351     category="ci",
 352     description="Print the digest of the image of this name based on the "
 353     "current contents of the tree.",
 354     parser=partial(get_taskgraph_command_parser, "build-image"),
 355 )
 356 def image_digest(command_context, **kwargs):
 357     taskgraph_commands["image-digest"].func(kwargs)
 358
 359
 360 @Command(
 361     "release-history",
 362     category="ci",
 363     description="Query balrog for release history used by enable partials generation",
 364 )
 365 @CommandArgument(
 366     "-b",
 367     "--branch",
 368     help="The gecko project branch used in balrog, such as "
 369     "mozilla-central, release, maple",
 370 )
 371 @CommandArgument(
 372     "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
 373 )
 374 def generate_partials_builds(command_context, product, branch):
 375     from gecko_taskgraph.util.partials import populate_release_history
 376
 377     try:
 378         import yaml
 379
 380         release_history = {"release_history": populate_release_history(product, branch)}
 381         print(
 382             yaml.safe_dump(
 383                 release_history, allow_unicode=True, default_flow_style=False
 384             )
 385         )
 386     except Exception:
 387         traceback.print_exc()
 388         sys.exit(1)