taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 import argparse
   9 import json
  10 import logging
  11 import os
  12 import sys
  13 import time
  14 import traceback
  15 from functools import partial
  16
  17 import gecko_taskgraph.main
  18 from gecko_taskgraph.main import commands as taskgraph_commands
  19 from mach.decorators import Command, CommandArgument, SettingsProvider, SubCommand
  20
  21 logger = logging.getLogger("taskcluster")
  22
  23
  24 @SettingsProvider
  25 class TaskgraphConfig(object):
  26     @classmethod
  27     def config_settings(cls):
  28         return [
  29             (
  30                 "taskgraph.diffcmd",
  31                 "string",
  32                 "The command to run with `./mach taskgraph --diff`",
  33                 "diff --report-identical-files "
  34                 "--label={attr}@{base} --label={attr}@{cur} -U20",
  35                 {},
  36             )
  37         ]
  38
  39
  40 def strtobool(value):
  41     """Convert string to boolean.
  42
  43     Wraps "distutils.util.strtobool", deferring the import of the package
  44     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  45     |mach bootstrap| would install the required package to enable "distutils.util", but
  46     it can't because mach fails to interpret this file.
  47     """
  48     from distutils.util import strtobool
  49
  50     return bool(strtobool(value))
  51
  52
  53 def get_taskgraph_command_parser(name):
  54     """Given a command name, obtain its argument parser.
  55
  56     Args:
  57         name (str): Name of the command.
  58
  59     Returns:
  60         ArgumentParser: An ArgumentParser instance.
  61     """
  62     command = taskgraph_commands[name]
  63     parser = argparse.ArgumentParser()
  64     for arg in command.func.args:
  65         parser.add_argument(*arg[0], **arg[1])
  66
  67     parser.set_defaults(func=command.func, **command.defaults)
  68     return parser
  69
  70
  71 def get_taskgraph_decision_parser():
  72     parser = get_taskgraph_command_parser("decision")
  73
  74     extra_args = [
  75         (
  76             ["--optimize-target-tasks"],
  77             {
  78                 "type": lambda flag: strtobool(flag),
  79                 "nargs": "?",
  80                 "const": "true",
  81                 "help": "If specified, this indicates whether the target "
  82                 "tasks are eligible for optimization. Otherwise, the default "
  83                 "for the project is used.",
  84             },
  85         ),
  86         (
  87             ["--include-push-tasks"],
  88             {
  89                 "action": "store_true",
  90                 "help": "Whether tasks from the on-push graph should be re-used "
  91                 "in this graph. This allows cron graphs to avoid rebuilding "
  92                 "jobs that were built on-push.",
  93             },
  94         ),
  95         (
  96             ["--rebuild-kind"],
  97             {
  98                 "dest": "rebuild_kinds",
  99                 "action": "append",
 100                 "default": argparse.SUPPRESS,
 101                 "help": "Kinds that should not be re-used from the on-push graph.",
 102             },
 103         ),
 104     ]
 105     for arg in extra_args:
 106         parser.add_argument(*arg[0], **arg[1])
 107
 108     return parser
 109
 110
 111 @Command(
 112     "taskgraph",
 113     category="ci",
 114     description="Manipulate TaskCluster task graphs defined in-tree",
 115 )
 116 def taskgraph_command(command_context):
 117     """The taskgraph subcommands all relate to the generation of task graphs
 118     for Gecko continuous integration.  A task graph is a set of tasks linked
 119     by dependencies: for example, a binary must be built before it is tested,
 120     and that build may further depend on various toolchains, libraries, etc.
 121     """
 122
 123
 124 @SubCommand(
 125     "taskgraph",
 126     "tasks",
 127     description="Show all tasks in the taskgraph",
 128     parser=partial(get_taskgraph_command_parser, "tasks"),
 129 )
 130 def taskgraph_tasks(command_context, **options):
 131     return run_show_taskgraph(command_context, **options)
 132
 133
 134 @SubCommand(
 135     "taskgraph",
 136     "full",
 137     description="Show the full taskgraph",
 138     parser=partial(get_taskgraph_command_parser, "full"),
 139 )
 140 def taskgraph_full(command_context, **options):
 141     return run_show_taskgraph(command_context, **options)
 142
 143
 144 @SubCommand(
 145     "taskgraph",
 146     "target",
 147     description="Show the target task set",
 148     parser=partial(get_taskgraph_command_parser, "target"),
 149 )
 150 def taskgraph_target(command_context, **options):
 151     return run_show_taskgraph(command_context, **options)
 152
 153
 154 @SubCommand(
 155     "taskgraph",
 156     "target-graph",
 157     description="Show the target taskgraph",
 158     parser=partial(get_taskgraph_command_parser, "target-graph"),
 159 )
 160 def taskgraph_target_graph(command_context, **options):
 161     return run_show_taskgraph(command_context, **options)
 162
 163
 164 @SubCommand(
 165     "taskgraph",
 166     "optimized",
 167     description="Show the optimized taskgraph",
 168     parser=partial(get_taskgraph_command_parser, "optimized"),
 169 )
 170 def taskgraph_optimized(command_context, **options):
 171     return run_show_taskgraph(command_context, **options)
 172
 173
 174 @SubCommand(
 175     "taskgraph",
 176     "morphed",
 177     description="Show the morphed taskgraph",
 178     parser=partial(get_taskgraph_command_parser, "morphed"),
 179 )
 180 def taskgraph_morphed(command_context, **options):
 181     return run_show_taskgraph(command_context, **options)
 182
 183
 184 def run_show_taskgraph(command_context, **options):
 185     # There are cases where we don't want to set up mach logging (e.g logs
 186     # are being redirected to disk). By monkeypatching the 'setup_logging'
 187     # function we can let 'taskgraph.main' decide whether or not to log to
 188     # the terminal.
 189     gecko_taskgraph.main.setup_logging = partial(
 190         setup_logging,
 191         command_context,
 192         quiet=options["quiet"],
 193         verbose=options["verbose"],
 194     )
 195     show_taskgraph = options.pop("func")
 196     return show_taskgraph(options)
 197
 198
 199 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
 200 @CommandArgument(
 201     "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
 202 )
 203 @CommandArgument(
 204     "--quiet", "-q", action="store_true", help="suppress all logging output"
 205 )
 206 @CommandArgument(
 207     "--verbose",
 208     "-v",
 209     action="store_true",
 210     help="include debug-level logging output",
 211 )
 212 @CommandArgument(
 213     "--parameters",
 214     "-p",
 215     default="project=mozilla-central",
 216     help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
 217 )
 218 def taskgraph_actions(command_context, **options):
 219     return show_actions(command_context, options)
 220
 221
 222 @SubCommand(
 223     "taskgraph",
 224     "decision",
 225     description="Run the decision task",
 226     parser=get_taskgraph_decision_parser,
 227 )
 228 def taskgraph_decision(command_context, **options):
 229     """Run the decision task: generate a task graph and submit to
 230     TaskCluster.  This is only meant to be called within decision tasks,
 231     and requires a great many arguments.  Commands like `mach taskgraph
 232     optimized` are better suited to use on the command line, and can take
 233     the parameters file generated by a decision task."""
 234     try:
 235         setup_logging(command_context)
 236         start = time.monotonic()
 237         ret = taskgraph_commands["decision"].func(options)
 238         end = time.monotonic()
 239         if os.environ.get("MOZ_AUTOMATION") == "1":
 240             perfherder_data = {
 241                 "framework": {"name": "build_metrics"},
 242                 "suites": [
 243                     {
 244                         "name": "decision",
 245                         "value": end - start,
 246                         "lowerIsBetter": True,
 247                         "shouldAlert": True,
 248                         "subtests": [],
 249                     }
 250                 ],
 251             }
 252             print(
 253                 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
 254                 file=sys.stderr,
 255             )
 256         return ret
 257     except Exception:
 258         traceback.print_exc()
 259         sys.exit(1)
 260
 261
 262 @SubCommand(
 263     "taskgraph",
 264     "cron",
 265     description="Provide a pointer to the new `.cron.yml` handler.",
 266 )
 267 def taskgraph_cron(command_context, **options):
 268     print(
 269         'Handling of ".cron.yml" files has move to '
 270         "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 271     )
 272     sys.exit(1)
 273
 274
 275 @SubCommand(
 276     "taskgraph",
 277     "action-callback",
 278     description="Run action callback used by action tasks",
 279     parser=partial(get_taskgraph_command_parser, "action-callback"),
 280 )
 281 def action_callback(command_context, **options):
 282     setup_logging(command_context)
 283     taskgraph_commands["action-callback"].func(options)
 284
 285
 286 @SubCommand(
 287     "taskgraph",
 288     "test-action-callback",
 289     description="Run an action callback in a testing mode",
 290     parser=partial(get_taskgraph_command_parser, "test-action-callback"),
 291 )
 292 def test_action_callback(command_context, **options):
 293     setup_logging(command_context)
 294
 295     if not options["parameters"]:
 296         options["parameters"] = "project=mozilla-central"
 297
 298     taskgraph_commands["test-action-callback"].func(options)
 299
 300
 301 def setup_logging(command_context, quiet=False, verbose=True):
 302     """
 303     Set up Python logging for all loggers, sending results to stderr (so
 304     that command output can be redirected easily) and adding the typical
 305     mach timestamp.
 306     """
 307     # remove the old terminal handler
 308     old = command_context.log_manager.replace_terminal_handler(None)
 309
 310     # re-add it, with level and fh set appropriately
 311     if not quiet:
 312         level = logging.DEBUG if verbose else logging.INFO
 313         command_context.log_manager.add_terminal_logging(
 314             fh=sys.stderr,
 315             level=level,
 316             write_interval=old.formatter.write_interval,
 317             write_times=old.formatter.write_times,
 318         )
 319
 320     # all of the taskgraph logging is unstructured logging
 321     command_context.log_manager.enable_unstructured()
 322
 323
 324 def show_actions(command_context, options):
 325     import gecko_taskgraph
 326     import gecko_taskgraph.actions
 327     from taskgraph.generator import TaskGraphGenerator
 328     from taskgraph.parameters import parameters_loader
 329
 330     try:
 331         setup_logging(
 332             command_context, quiet=options["quiet"], verbose=options["verbose"]
 333         )
 334         parameters = parameters_loader(options["parameters"])
 335
 336         tgg = TaskGraphGenerator(
 337             root_dir=options.get("root"),
 338             parameters=parameters,
 339         )
 340
 341         actions = gecko_taskgraph.actions.render_actions_json(
 342             tgg.parameters,
 343             tgg.graph_config,
 344             decision_task_id="DECISION-TASK",
 345         )
 346         print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
 347     except Exception:
 348         traceback.print_exc()
 349         sys.exit(1)
 350
 351
 352 @Command(
 353     "taskcluster-load-image",
 354     category="ci",
 355     description="Load a pre-built Docker image. Note that you need to "
 356     "have docker installed and running for this to work.",
 357     parser=partial(get_taskgraph_command_parser, "load-image"),
 358 )
 359 def load_image(command_context, **kwargs):
 360     taskgraph_commands["load-image"].func(kwargs)
 361
 362
 363 @Command(
 364     "taskcluster-build-image",
 365     category="ci",
 366     description="Build a Docker image",
 367     parser=partial(get_taskgraph_command_parser, "build-image"),
 368 )
 369 def build_image(command_context, **kwargs):
 370     try:
 371         taskgraph_commands["build-image"].func(kwargs)
 372     except Exception:
 373         traceback.print_exc()
 374         sys.exit(1)
 375
 376
 377 @Command(
 378     "taskcluster-image-digest",
 379     category="ci",
 380     description="Print the digest of the image of this name based on the "
 381     "current contents of the tree.",
 382     parser=partial(get_taskgraph_command_parser, "build-image"),
 383 )
 384 def image_digest(command_context, **kwargs):
 385     taskgraph_commands["image-digest"].func(kwargs)
 386
 387
 388 @Command(
 389     "release-history",
 390     category="ci",
 391     description="Query balrog for release history used by enable partials generation",
 392 )
 393 @CommandArgument(
 394     "-b",
 395     "--branch",
 396     help="The gecko project branch used in balrog, such as "
 397     "mozilla-central, release, maple",
 398 )
 399 @CommandArgument(
 400     "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
 401 )
 402 def generate_partials_builds(command_context, product, branch):
 403     from gecko_taskgraph.util.partials import populate_release_history
 404
 405     try:
 406         import yaml
 407
 408         release_history = {"release_history": populate_release_history(product, branch)}
 409         print(
 410             yaml.safe_dump(
 411                 release_history, allow_unicode=True, default_flow_style=False
 412             )
 413         )
 414     except Exception:
 415         traceback.print_exc()
 416         sys.exit(1)