taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 from __future__ import absolute_import, print_function, unicode_literals
   9
  10 import argparse
  11 import json
  12 import logging
  13 import os
  14 from six import text_type
  15 import sys
  16 import traceback
  17 import re
  18
  19 from mach.decorators import (
  20     CommandArgument,
  21     CommandProvider,
  22     Command,
  23     SubCommand,
  24 )
  25
  26 from mozbuild.base import MachCommandBase
  27
  28
  29 def strtobool(value):
  30     """Convert string to boolean.
  31
  32     Wraps "distutils.util.strtobool", deferring the import of the package
  33     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  34     |mach bootstrap| would install the required package to enable "distutils.util", but
  35     it can't because mach fails to interpret this file.
  36     """
  37     from distutils.util import strtobool
  38     return bool(strtobool(value))
  39
  40
  41 class ShowTaskGraphSubCommand(SubCommand):
  42     """A SubCommand with TaskGraph-specific arguments"""
  43
  44     def __call__(self, func):
  45         after = SubCommand.__call__(self, func)
  46         args = [
  47             CommandArgument('--root', '-r',
  48                             help="root of the taskgraph definition relative to topsrcdir"),
  49             CommandArgument('--quiet', '-q', action="store_true",
  50                             help="suppress all logging output"),
  51             CommandArgument('--verbose', '-v', action="store_true",
  52                             help="include debug-level logging output"),
  53             CommandArgument('--json', '-J', action="store_const",
  54                             dest="format", const="json",
  55                             help="Output task graph as a JSON object"),
  56             CommandArgument('--labels', '-L', action="store_const",
  57                             dest="format", const="labels",
  58                             help="Output the label for each task in the task graph (default)"),
  59             CommandArgument('--parameters', '-p', default="project=mozilla-central",
  60                             help="parameters file (.yml or .json; see "
  61                                  "`taskcluster/docs/parameters.rst`)`"),
  62             CommandArgument('--no-optimize', dest="optimize", action="store_false",
  63                             default="true",
  64                             help="do not remove tasks from the graph that are found in the "
  65                             "index (a.k.a. optimize the graph)"),
  66             CommandArgument('--tasks-regex', '--tasks', default=None,
  67                             help="only return tasks with labels matching this regular "
  68                             "expression."),
  69             CommandArgument('--target-kind', default=None,
  70                             help="only return tasks that are of the given kind, "
  71                                  "or their dependencies."),
  72             CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
  73                             help="enable fast task generation for local debugging."),
  74             CommandArgument('-o', '--output-file', default=None,
  75                             help="file path to store generated output."),
  76
  77         ]
  78         for arg in args:
  79             after = arg(after)
  80         return after
  81
  82
  83 @CommandProvider
  84 class MachCommands(MachCommandBase):
  85
  86     @Command('taskgraph', category="ci",
  87              description="Manipulate TaskCluster task graphs defined in-tree")
  88     def taskgraph(self):
  89         """The taskgraph subcommands all relate to the generation of task graphs
  90         for Gecko continuous integration.  A task graph is a set of tasks linked
  91         by dependencies: for example, a binary must be built before it is tested,
  92         and that build may further depend on various toolchains, libraries, etc.
  93         """
  94
  95     @ShowTaskGraphSubCommand('taskgraph', 'tasks',
  96                              description="Show all tasks in the taskgraph")
  97     def taskgraph_tasks(self, **options):
  98         return self.show_taskgraph('full_task_set', options)
  99
 100     @ShowTaskGraphSubCommand('taskgraph', 'full',
 101                              description="Show the full taskgraph")
 102     def taskgraph_full(self, **options):
 103         return self.show_taskgraph('full_task_graph', options)
 104
 105     @ShowTaskGraphSubCommand('taskgraph', 'target',
 106                              description="Show the target task set")
 107     def taskgraph_target(self, **options):
 108         return self.show_taskgraph('target_task_set', options)
 109
 110     @ShowTaskGraphSubCommand('taskgraph', 'target-graph',
 111                              description="Show the target taskgraph")
 112     def taskgraph_target_taskgraph(self, **options):
 113         return self.show_taskgraph('target_task_graph', options)
 114
 115     @ShowTaskGraphSubCommand('taskgraph', 'optimized',
 116                              description="Show the optimized taskgraph")
 117     def taskgraph_optimized(self, **options):
 118         return self.show_taskgraph('optimized_task_graph', options)
 119
 120     @ShowTaskGraphSubCommand('taskgraph', 'morphed',
 121                              description="Show the morphed taskgraph")
 122     def taskgraph_morphed(self, **options):
 123         return self.show_taskgraph('morphed_task_graph', options)
 124
 125     @SubCommand('taskgraph', 'actions',
 126                 description="Write actions.json to stdout")
 127     @CommandArgument('--root', '-r',
 128                      help="root of the taskgraph definition relative to topsrcdir")
 129     @CommandArgument('--quiet', '-q', action="store_true",
 130                      help="suppress all logging output")
 131     @CommandArgument('--verbose', '-v', action="store_true",
 132                      help="include debug-level logging output")
 133     @CommandArgument('--parameters', '-p', default="project=mozilla-central",
 134                      help="parameters file (.yml or .json; see "
 135                           "`taskcluster/docs/parameters.rst`)`")
 136     def taskgraph_actions(self, **options):
 137         return self.show_actions(options)
 138
 139     @SubCommand('taskgraph', 'decision',
 140                 description="Run the decision task")
 141     @CommandArgument('--root', '-r', type=text_type,
 142                      help="root of the taskgraph definition relative to topsrcdir")
 143     @CommandArgument('--base-repository', type=text_type, required=True,
 144                      help='URL for "base" repository to clone')
 145     @CommandArgument('--head-repository', type=text_type, required=True,
 146                      help='URL for "head" repository to fetch revision from')
 147     @CommandArgument('--head-ref', type=text_type, required=True,
 148                      help='Reference (this is same as rev usually for hg)')
 149     @CommandArgument('--head-rev', type=text_type, required=True,
 150                      help='Commit revision to use from head repository')
 151     @CommandArgument('--comm-base-repository', type=text_type, required=False,
 152                      help='URL for "base" comm-* repository to clone')
 153     @CommandArgument('--comm-head-repository', type=text_type, required=False,
 154                      help='URL for "head" comm-* repository to fetch revision from')
 155     @CommandArgument('--comm-head-ref', type=text_type, required=False,
 156                      help='comm-* Reference (this is same as rev usually for hg)')
 157     @CommandArgument('--comm-head-rev', type=text_type, required=False,
 158                      help='Commit revision to use from head comm-* repository')
 159     @CommandArgument(
 160         '--project', type=text_type, required=True,
 161         help='Project to use for creating task graph. Example: --project=try')
 162     @CommandArgument('--pushlog-id', type=text_type, dest='pushlog_id',
 163                      required=True, default='0')
 164     @CommandArgument('--pushdate',
 165                      dest='pushdate',
 166                      required=True,
 167                      type=int,
 168                      default=0)
 169     @CommandArgument('--owner', type=text_type, required=True,
 170                      help='email address of who owns this graph')
 171     @CommandArgument('--level', type=text_type, required=True,
 172                      help='SCM level of this repository')
 173     @CommandArgument('--target-tasks-method', type=text_type,
 174                      help='method for selecting the target tasks to generate')
 175     @CommandArgument('--optimize-target-tasks',
 176                      type=lambda flag: strtobool(flag),
 177                      nargs='?', const='true',
 178                      help='If specified, this indicates whether the target '
 179                           'tasks are eligible for optimization. Otherwise, '
 180                           'the default for the project is used.')
 181     @CommandArgument('--try-task-config-file', type=text_type,
 182                      help='path to try task configuration file')
 183     @CommandArgument('--tasks-for', type=text_type, required=True,
 184                      help='the tasks_for value used to generate this task')
 185     @CommandArgument('--include-push-tasks',
 186                      action='store_true',
 187                      help='Whether tasks from the on-push graph should be re-used '
 188                           'in this graph. This allows cron graphs to avoid rebuilding '
 189                           'jobs that were built on-push.')
 190     @CommandArgument('--rebuild-kind',
 191                      dest='rebuild_kinds',
 192                      action='append',
 193                      default=argparse.SUPPRESS,
 194                      help='Kinds that should not be re-used from the on-push graph.')
 195     def taskgraph_decision(self, **options):
 196         """Run the decision task: generate a task graph and submit to
 197         TaskCluster.  This is only meant to be called within decision tasks,
 198         and requires a great many arguments.  Commands like `mach taskgraph
 199         optimized` are better suited to use on the command line, and can take
 200         the parameters file generated by a decision task.  """
 201
 202         import taskgraph.decision
 203         try:
 204             self.setup_logging()
 205             return taskgraph.decision.taskgraph_decision(options)
 206         except Exception:
 207             traceback.print_exc()
 208             sys.exit(1)
 209
 210     @SubCommand('taskgraph', 'cron',
 211                 description="Run the cron task")
 212     @CommandArgument('--base-repository',
 213                      required=False,
 214                      help='(ignored)')
 215     @CommandArgument('--head-repository',
 216                      required=True,
 217                      help='URL for "head" repository to fetch')
 218     @CommandArgument('--head-ref',
 219                      required=False,
 220                      help='(ignored)')
 221     @CommandArgument('--project',
 222                      required=True,
 223                      help='Project to use for creating tasks. Example: --project=mozilla-central')
 224     @CommandArgument('--level',
 225                      required=True,
 226                      help='SCM level of this repository')
 227     @CommandArgument('--force-run',
 228                      required=False,
 229                      help='If given, force this cronjob to run regardless of time, '
 230                      'and run no others')
 231     @CommandArgument('--no-create',
 232                      required=False,
 233                      action='store_true',
 234                      help='Do not actually create tasks')
 235     @CommandArgument('--root', '-r',
 236                      required=False,
 237                      help="root of the repository to get cron task definitions from")
 238     def taskgraph_cron(self, **options):
 239         """Run the cron task; this task creates zero or more decision tasks.  It is run
 240         from the hooks service on a regular basis."""
 241         import taskgraph.cron
 242         try:
 243             self.setup_logging()
 244             return taskgraph.cron.taskgraph_cron(options)
 245         except Exception:
 246             traceback.print_exc()
 247             sys.exit(1)
 248
 249     @SubCommand('taskgraph', 'action-callback',
 250                 description='Run action callback used by action tasks')
 251     @CommandArgument('--root', '-r', default='taskcluster/ci',
 252                      help="root of the taskgraph definition relative to topsrcdir")
 253     def action_callback(self, **options):
 254         from taskgraph.actions import trigger_action_callback
 255         from taskgraph.actions.util import get_parameters
 256         try:
 257             self.setup_logging()
 258
 259             # the target task for this action (or null if it's a group action)
 260             task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
 261             # the target task group for this action
 262             task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
 263             input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
 264             callback = os.environ.get('ACTION_CALLBACK', None)
 265             root = options['root']
 266
 267             parameters = get_parameters(task_group_id)
 268
 269             return trigger_action_callback(
 270                     task_group_id=task_group_id,
 271                     task_id=task_id,
 272                     input=input,
 273                     callback=callback,
 274                     parameters=parameters,
 275                     root=root,
 276                     test=False)
 277         except Exception:
 278             traceback.print_exc()
 279             sys.exit(1)
 280
 281     @SubCommand('taskgraph', 'test-action-callback',
 282                 description='Run an action callback in a testing mode')
 283     @CommandArgument('--root', '-r', default='taskcluster/ci',
 284                      help="root of the taskgraph definition relative to topsrcdir")
 285     @CommandArgument('--parameters', '-p', default='project=mozilla-central',
 286                      help='parameters file (.yml or .json; see '
 287                           '`taskcluster/docs/parameters.rst`)`')
 288     @CommandArgument('--task-id', default=None,
 289                      help='TaskId to which the action applies')
 290     @CommandArgument('--task-group-id', default=None,
 291                      help='TaskGroupId to which the action applies')
 292     @CommandArgument('--input', default=None,
 293                      help='Action input (.yml or .json)')
 294     @CommandArgument('callback', default=None,
 295                      help='Action callback name (Python function name)')
 296     def test_action_callback(self, **options):
 297         import taskgraph.parameters
 298         import taskgraph.actions
 299         from taskgraph.util import yaml
 300
 301         def load_data(filename):
 302             with open(filename) as f:
 303                 if filename.endswith('.yml'):
 304                     return yaml.load_stream(f)
 305                 elif filename.endswith('.json'):
 306                     return json.load(f)
 307                 else:
 308                     raise Exception("unknown filename {}".format(filename))
 309
 310         try:
 311             self.setup_logging()
 312             task_id = options['task_id']
 313
 314             if options['input']:
 315                 input = load_data(options['input'])
 316             else:
 317                 input = None
 318
 319             parameters = taskgraph.parameters.load_parameters_file(
 320                 options['parameters'],
 321                 strict=False,
 322                 # FIXME: There should be a way to parameterize this.
 323                 trust_domain="gecko",
 324             )
 325             parameters.check()
 326
 327             root = options['root']
 328
 329             return taskgraph.actions.trigger_action_callback(
 330                     task_group_id=options['task_group_id'],
 331                     task_id=task_id,
 332                     input=input,
 333                     callback=options['callback'],
 334                     parameters=parameters,
 335                     root=root,
 336                     test=True)
 337         except Exception:
 338             traceback.print_exc()
 339             sys.exit(1)
 340
 341     def setup_logging(self, quiet=False, verbose=True):
 342         """
 343         Set up Python logging for all loggers, sending results to stderr (so
 344         that command output can be redirected easily) and adding the typical
 345         mach timestamp.
 346         """
 347         # remove the old terminal handler
 348         old = self.log_manager.replace_terminal_handler(None)
 349
 350         # re-add it, with level and fh set appropriately
 351         if not quiet:
 352             level = logging.DEBUG if verbose else logging.INFO
 353             self.log_manager.add_terminal_logging(
 354                 fh=sys.stderr, level=level,
 355                 write_interval=old.formatter.write_interval,
 356                 write_times=old.formatter.write_times)
 357
 358         # all of the taskgraph logging is unstructured logging
 359         self.log_manager.enable_unstructured()
 360
 361     def show_taskgraph(self, graph_attr, options):
 362         import taskgraph.parameters
 363         import taskgraph.generator
 364         import taskgraph
 365         if options['fast']:
 366             taskgraph.fast = True
 367
 368         try:
 369             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
 370             parameters = taskgraph.parameters.parameters_loader(options['parameters'])
 371
 372             tgg = taskgraph.generator.TaskGraphGenerator(
 373                 root_dir=options.get('root'),
 374                 parameters=parameters,
 375                 target_kind=options.get('target_kind'),
 376             )
 377
 378             tg = getattr(tgg, graph_attr)
 379
 380             show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
 381             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
 382
 383             fh = options['output_file']
 384             if fh:
 385                 fh = open(fh, 'w')
 386             show_method(tg, file=fh)
 387         except Exception:
 388             traceback.print_exc()
 389             sys.exit(1)
 390
 391     def show_taskgraph_labels(self, taskgraph, file=None):
 392         for index in taskgraph.graph.visit_postorder():
 393             print(taskgraph.tasks[index].label, file=file)
 394
 395     def show_taskgraph_json(self, taskgraph, file=None):
 396         print(json.dumps(taskgraph.to_json(),
 397               sort_keys=True, indent=2, separators=(',', ': ')),
 398               file=file)
 399
 400     def get_filtered_taskgraph(self, taskgraph, tasksregex):
 401         from taskgraph.graph import Graph
 402         from taskgraph.taskgraph import TaskGraph
 403         """
 404         This class method filters all the tasks on basis of a regular expression
 405         and returns a new TaskGraph object
 406         """
 407         # return original taskgraph if no regular expression is passed
 408         if not tasksregex:
 409             return taskgraph
 410         named_links_dict = taskgraph.graph.named_links_dict()
 411         filteredtasks = {}
 412         filterededges = set()
 413         regexprogram = re.compile(tasksregex)
 414
 415         for key in taskgraph.graph.visit_postorder():
 416             task = taskgraph.tasks[key]
 417             if regexprogram.match(task.label):
 418                 filteredtasks[key] = task
 419                 for depname, dep in named_links_dict[key].iteritems():
 420                     if regexprogram.match(dep):
 421                         filterededges.add((key, dep, depname))
 422         filtered_taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))
 423         return filtered_taskgraph
 424
 425     def show_actions(self, options):
 426         import taskgraph.parameters
 427         import taskgraph.generator
 428         import taskgraph
 429         import taskgraph.actions
 430
 431         try:
 432             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
 433             parameters = taskgraph.parameters.parameters_loader(options['parameters'])
 434
 435             tgg = taskgraph.generator.TaskGraphGenerator(
 436                 root_dir=options.get('root'),
 437                 parameters=parameters)
 438
 439             actions = taskgraph.actions.render_actions_json(tgg.parameters, tgg.graph_config)
 440             print(json.dumps(actions, sort_keys=True, indent=2, separators=(',', ': ')))
 441         except Exception:
 442             traceback.print_exc()
 443             sys.exit(1)
 444
 445
 446 @CommandProvider
 447 class TaskClusterImagesProvider(MachCommandBase):
 448     @Command('taskcluster-load-image', category="ci",
 449              description="Load a pre-built Docker image. Note that you need to "
 450                          "have docker installed and running for this to work.")
 451     @CommandArgument('--task-id',
 452                      help="Load the image at public/image.tar.zst in this task, "
 453                           "rather than searching the index")
 454     @CommandArgument('-t', '--tag',
 455                      help="tag that the image should be loaded as. If not "
 456                           "image will be loaded with tag from the tarball",
 457                      metavar="name:tag")
 458     @CommandArgument('image_name', nargs='?',
 459                      help="Load the image of this name based on the current "
 460                           "contents of the tree (as built for mozilla-central "
 461                           "or mozilla-inbound)")
 462     def load_image(self, image_name, task_id, tag):
 463         self._ensure_zstd()
 464         from taskgraph.docker import load_image_by_name, load_image_by_task_id
 465         if not image_name and not task_id:
 466             print("Specify either IMAGE-NAME or TASK-ID")
 467             sys.exit(1)
 468         try:
 469             if task_id:
 470                 ok = load_image_by_task_id(task_id, tag)
 471             else:
 472                 ok = load_image_by_name(image_name, tag)
 473             if not ok:
 474                 sys.exit(1)
 475         except Exception:
 476             traceback.print_exc()
 477             sys.exit(1)
 478
 479     @Command('taskcluster-build-image', category='ci',
 480              description='Build a Docker image')
 481     @CommandArgument('image_name',
 482                      help='Name of the image to build')
 483     @CommandArgument('-t', '--tag',
 484                      help="tag that the image should be built as.",
 485                      metavar="name:tag")
 486     @CommandArgument('--context-only',
 487                      help="File name the context tarball should be written to."
 488                           "with this option it will only build the context.tar.",
 489                      metavar='context.tar')
 490     def build_image(self, image_name, tag, context_only):
 491         from taskgraph.docker import build_image, build_context
 492         try:
 493             if context_only is None:
 494                 build_image(image_name, tag, os.environ)
 495             else:
 496                 build_context(image_name, context_only, os.environ)
 497         except Exception:
 498             traceback.print_exc()
 499             sys.exit(1)
 500
 501
 502 @CommandProvider
 503 class TaskClusterPartialsData(object):
 504     @Command('release-history', category="ci",
 505              description="Query balrog for release history used by enable partials generation")
 506     @CommandArgument('-b', '--branch',
 507                      help="The gecko project branch used in balrog, such as "
 508                           "mozilla-central, release, maple")
 509     @CommandArgument('--product', default='Firefox',
 510                      help="The product identifier, such as 'Firefox'")
 511     def generate_partials_builds(self, product, branch):
 512         from taskgraph.util.partials import populate_release_history
 513         try:
 514             import yaml
 515             release_history = {'release_history': populate_release_history(product, branch)}
 516             print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
 517         except Exception:
 518             traceback.print_exc()
 519             sys.exit(1)