taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 from __future__ import absolute_import, print_function, unicode_literals
   9
  10 import argparse
  11 import json
  12 import logging
  13 import os
  14 from six import text_type
  15 import six
  16 import sys
  17 import traceback
  18 import re
  19
  20 from mach.decorators import (
  21     CommandArgument,
  22     CommandProvider,
  23     Command,
  24     SubCommand,
  25 )
  26
  27 from mozbuild.base import MachCommandBase
  28
  29
  30 def strtobool(value):
  31     """Convert string to boolean.
  32
  33     Wraps "distutils.util.strtobool", deferring the import of the package
  34     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  35     |mach bootstrap| would install the required package to enable "distutils.util", but
  36     it can't because mach fails to interpret this file.
  37     """
  38     from distutils.util import strtobool
  39     return bool(strtobool(value))
  40
  41
  42 class ShowTaskGraphSubCommand(SubCommand):
  43     """A SubCommand with TaskGraph-specific arguments"""
  44
  45     def __call__(self, func):
  46         after = SubCommand.__call__(self, func)
  47         args = [
  48             CommandArgument('--root', '-r',
  49                             help="root of the taskgraph definition relative to topsrcdir"),
  50             CommandArgument('--quiet', '-q', action="store_true",
  51                             help="suppress all logging output"),
  52             CommandArgument('--verbose', '-v', action="store_true",
  53                             help="include debug-level logging output"),
  54             CommandArgument('--json', '-J', action="store_const",
  55                             dest="format", const="json",
  56                             help="Output task graph as a JSON object"),
  57             CommandArgument('--labels', '-L', action="store_const",
  58                             dest="format", const="labels",
  59                             help="Output the label for each task in the task graph (default)"),
  60             CommandArgument('--parameters', '-p', default="project=mozilla-central",
  61                             help="parameters file (.yml or .json; see "
  62                                  "`taskcluster/docs/parameters.rst`)`"),
  63             CommandArgument('--no-optimize', dest="optimize", action="store_false",
  64                             default="true",
  65                             help="do not remove tasks from the graph that are found in the "
  66                             "index (a.k.a. optimize the graph)"),
  67             CommandArgument('--tasks-regex', '--tasks', default=None,
  68                             help="only return tasks with labels matching this regular "
  69                             "expression."),
  70             CommandArgument('--target-kind', default=None,
  71                             help="only return tasks that are of the given kind, "
  72                                  "or their dependencies."),
  73             CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
  74                             help="enable fast task generation for local debugging."),
  75             CommandArgument('-o', '--output-file', default=None,
  76                             help="file path to store generated output."),
  77
  78         ]
  79         for arg in args:
  80             after = arg(after)
  81         return after
  82
  83
  84 @CommandProvider
  85 class MachCommands(MachCommandBase):
  86
  87     @Command('taskgraph', category="ci",
  88              description="Manipulate TaskCluster task graphs defined in-tree")
  89     def taskgraph(self):
  90         """The taskgraph subcommands all relate to the generation of task graphs
  91         for Gecko continuous integration.  A task graph is a set of tasks linked
  92         by dependencies: for example, a binary must be built before it is tested,
  93         and that build may further depend on various toolchains, libraries, etc.
  94         """
  95
  96     @ShowTaskGraphSubCommand('taskgraph', 'tasks',
  97                              description="Show all tasks in the taskgraph")
  98     def taskgraph_tasks(self, **options):
  99         return self.show_taskgraph('full_task_set', options)
 100
 101     @ShowTaskGraphSubCommand('taskgraph', 'full',
 102                              description="Show the full taskgraph")
 103     def taskgraph_full(self, **options):
 104         return self.show_taskgraph('full_task_graph', options)
 105
 106     @ShowTaskGraphSubCommand('taskgraph', 'target',
 107                              description="Show the target task set")
 108     def taskgraph_target(self, **options):
 109         return self.show_taskgraph('target_task_set', options)
 110
 111     @ShowTaskGraphSubCommand('taskgraph', 'target-graph',
 112                              description="Show the target taskgraph")
 113     def taskgraph_target_taskgraph(self, **options):
 114         return self.show_taskgraph('target_task_graph', options)
 115
 116     @ShowTaskGraphSubCommand('taskgraph', 'optimized',
 117                              description="Show the optimized taskgraph")
 118     def taskgraph_optimized(self, **options):
 119         return self.show_taskgraph('optimized_task_graph', options)
 120
 121     @ShowTaskGraphSubCommand('taskgraph', 'morphed',
 122                              description="Show the morphed taskgraph")
 123     def taskgraph_morphed(self, **options):
 124         return self.show_taskgraph('morphed_task_graph', options)
 125
 126     @SubCommand('taskgraph', 'actions',
 127                 description="Write actions.json to stdout")
 128     @CommandArgument('--root', '-r',
 129                      help="root of the taskgraph definition relative to topsrcdir")
 130     @CommandArgument('--quiet', '-q', action="store_true",
 131                      help="suppress all logging output")
 132     @CommandArgument('--verbose', '-v', action="store_true",
 133                      help="include debug-level logging output")
 134     @CommandArgument('--parameters', '-p', default="project=mozilla-central",
 135                      help="parameters file (.yml or .json; see "
 136                           "`taskcluster/docs/parameters.rst`)`")
 137     def taskgraph_actions(self, **options):
 138         return self.show_actions(options)
 139
 140     @SubCommand('taskgraph', 'decision',
 141                 description="Run the decision task")
 142     @CommandArgument('--root', '-r', type=text_type,
 143                      help="root of the taskgraph definition relative to topsrcdir")
 144     @CommandArgument('--base-repository', type=text_type, required=True,
 145                      help='URL for "base" repository to clone')
 146     @CommandArgument('--head-repository', type=text_type, required=True,
 147                      help='URL for "head" repository to fetch revision from')
 148     @CommandArgument('--head-ref', type=text_type, required=True,
 149                      help='Reference (this is same as rev usually for hg)')
 150     @CommandArgument('--head-rev', type=text_type, required=True,
 151                      help='Commit revision to use from head repository')
 152     @CommandArgument('--comm-base-repository', type=text_type, required=False,
 153                      help='URL for "base" comm-* repository to clone')
 154     @CommandArgument('--comm-head-repository', type=text_type, required=False,
 155                      help='URL for "head" comm-* repository to fetch revision from')
 156     @CommandArgument('--comm-head-ref', type=text_type, required=False,
 157                      help='comm-* Reference (this is same as rev usually for hg)')
 158     @CommandArgument('--comm-head-rev', type=text_type, required=False,
 159                      help='Commit revision to use from head comm-* repository')
 160     @CommandArgument(
 161         '--project', type=text_type, required=True,
 162         help='Project to use for creating task graph. Example: --project=try')
 163     @CommandArgument('--pushlog-id', type=text_type, dest='pushlog_id',
 164                      required=True, default='0')
 165     @CommandArgument('--pushdate',
 166                      dest='pushdate',
 167                      required=True,
 168                      type=int,
 169                      default=0)
 170     @CommandArgument('--owner', type=text_type, required=True,
 171                      help='email address of who owns this graph')
 172     @CommandArgument('--level', type=text_type, required=True,
 173                      help='SCM level of this repository')
 174     @CommandArgument('--target-tasks-method', type=text_type,
 175                      help='method for selecting the target tasks to generate')
 176     @CommandArgument('--optimize-target-tasks',
 177                      type=lambda flag: strtobool(flag),
 178                      nargs='?', const='true',
 179                      help='If specified, this indicates whether the target '
 180                           'tasks are eligible for optimization. Otherwise, '
 181                           'the default for the project is used.')
 182     @CommandArgument('--try-task-config-file', type=text_type,
 183                      help='path to try task configuration file')
 184     @CommandArgument('--tasks-for', type=text_type, required=True,
 185                      help='the tasks_for value used to generate this task')
 186     @CommandArgument('--include-push-tasks',
 187                      action='store_true',
 188                      help='Whether tasks from the on-push graph should be re-used '
 189                           'in this graph. This allows cron graphs to avoid rebuilding '
 190                           'jobs that were built on-push.')
 191     @CommandArgument('--rebuild-kind',
 192                      dest='rebuild_kinds',
 193                      action='append',
 194                      default=argparse.SUPPRESS,
 195                      help='Kinds that should not be re-used from the on-push graph.')
 196     def taskgraph_decision(self, **options):
 197         """Run the decision task: generate a task graph and submit to
 198         TaskCluster.  This is only meant to be called within decision tasks,
 199         and requires a great many arguments.  Commands like `mach taskgraph
 200         optimized` are better suited to use on the command line, and can take
 201         the parameters file generated by a decision task.  """
 202
 203         import taskgraph.decision
 204         try:
 205             self.setup_logging()
 206             return taskgraph.decision.taskgraph_decision(options)
 207         except Exception:
 208             traceback.print_exc()
 209             sys.exit(1)
 210
 211     @SubCommand('taskgraph', 'cron',
 212                 description="Provide a pointer to the new `.cron.yml` handler.")
 213     def taskgraph_cron(self, **options):
 214         print(
 215             'Handling of ".cron.yml" files has move to '
 216             "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 217         )
 218         sys.exit(1)
 219
 220     @SubCommand('taskgraph', 'action-callback',
 221                 description='Run action callback used by action tasks')
 222     @CommandArgument('--root', '-r', default='taskcluster/ci',
 223                      help="root of the taskgraph definition relative to topsrcdir")
 224     def action_callback(self, **options):
 225         from taskgraph.actions import trigger_action_callback
 226         from taskgraph.actions.util import get_parameters
 227         try:
 228             self.setup_logging()
 229
 230             # the target task for this action (or null if it's a group action)
 231             task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
 232             # the target task group for this action
 233             task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
 234             input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
 235             callback = os.environ.get('ACTION_CALLBACK', None)
 236             root = options['root']
 237
 238             parameters = get_parameters(task_group_id)
 239
 240             return trigger_action_callback(
 241                     task_group_id=task_group_id,
 242                     task_id=task_id,
 243                     input=input,
 244                     callback=callback,
 245                     parameters=parameters,
 246                     root=root,
 247                     test=False)
 248         except Exception:
 249             traceback.print_exc()
 250             sys.exit(1)
 251
 252     @SubCommand('taskgraph', 'test-action-callback',
 253                 description='Run an action callback in a testing mode')
 254     @CommandArgument('--root', '-r', default='taskcluster/ci',
 255                      help="root of the taskgraph definition relative to topsrcdir")
 256     @CommandArgument('--parameters', '-p', default='project=mozilla-central',
 257                      help='parameters file (.yml or .json; see '
 258                           '`taskcluster/docs/parameters.rst`)`')
 259     @CommandArgument('--task-id', default=None,
 260                      help='TaskId to which the action applies')
 261     @CommandArgument('--task-group-id', default=None,
 262                      help='TaskGroupId to which the action applies')
 263     @CommandArgument('--input', default=None,
 264                      help='Action input (.yml or .json)')
 265     @CommandArgument('callback', default=None,
 266                      help='Action callback name (Python function name)')
 267     def test_action_callback(self, **options):
 268         import taskgraph.parameters
 269         import taskgraph.actions
 270         from taskgraph.util import yaml
 271
 272         def load_data(filename):
 273             with open(filename) as f:
 274                 if filename.endswith('.yml'):
 275                     return yaml.load_stream(f)
 276                 elif filename.endswith('.json'):
 277                     return json.load(f)
 278                 else:
 279                     raise Exception("unknown filename {}".format(filename))
 280
 281         try:
 282             self.setup_logging()
 283             task_id = options['task_id']
 284
 285             if options['input']:
 286                 input = load_data(options['input'])
 287             else:
 288                 input = None
 289
 290             parameters = taskgraph.parameters.load_parameters_file(
 291                 options['parameters'],
 292                 strict=False,
 293                 # FIXME: There should be a way to parameterize this.
 294                 trust_domain="gecko",
 295             )
 296             parameters.check()
 297
 298             root = options['root']
 299
 300             return taskgraph.actions.trigger_action_callback(
 301                     task_group_id=options['task_group_id'],
 302                     task_id=task_id,
 303                     input=input,
 304                     callback=options['callback'],
 305                     parameters=parameters,
 306                     root=root,
 307                     test=True)
 308         except Exception:
 309             traceback.print_exc()
 310             sys.exit(1)
 311
 312     def setup_logging(self, quiet=False, verbose=True):
 313         """
 314         Set up Python logging for all loggers, sending results to stderr (so
 315         that command output can be redirected easily) and adding the typical
 316         mach timestamp.
 317         """
 318         # remove the old terminal handler
 319         old = self.log_manager.replace_terminal_handler(None)
 320
 321         # re-add it, with level and fh set appropriately
 322         if not quiet:
 323             level = logging.DEBUG if verbose else logging.INFO
 324             self.log_manager.add_terminal_logging(
 325                 fh=sys.stderr, level=level,
 326                 write_interval=old.formatter.write_interval,
 327                 write_times=old.formatter.write_times)
 328
 329         # all of the taskgraph logging is unstructured logging
 330         self.log_manager.enable_unstructured()
 331
 332     def show_taskgraph(self, graph_attr, options):
 333         import taskgraph.parameters
 334         import taskgraph.generator
 335         import taskgraph
 336         if options['fast']:
 337             taskgraph.fast = True
 338
 339         try:
 340             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
 341             parameters = taskgraph.parameters.parameters_loader(
 342                 options['parameters'],
 343                 overrides={'target-kind': options.get('target_kind')},
 344                 strict=False,
 345             )
 346
 347             tgg = taskgraph.generator.TaskGraphGenerator(
 348                 root_dir=options.get('root'),
 349                 parameters=parameters,
 350             )
 351
 352             tg = getattr(tgg, graph_attr)
 353
 354             show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
 355             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
 356
 357             fh = options['output_file']
 358             if fh:
 359                 fh = open(fh, 'w')
 360             show_method(tg, file=fh)
 361         except Exception:
 362             traceback.print_exc()
 363             sys.exit(1)
 364
 365     def show_taskgraph_labels(self, taskgraph, file=None):
 366         for index in taskgraph.graph.visit_postorder():
 367             print(taskgraph.tasks[index].label, file=file)
 368
 369     def show_taskgraph_json(self, taskgraph, file=None):
 370         print(json.dumps(taskgraph.to_json(),
 371               sort_keys=True, indent=2, separators=(',', ': ')),
 372               file=file)
 373
 374     def get_filtered_taskgraph(self, taskgraph, tasksregex):
 375         from taskgraph.graph import Graph
 376         from taskgraph.taskgraph import TaskGraph
 377         """
 378         This class method filters all the tasks on basis of a regular expression
 379         and returns a new TaskGraph object
 380         """
 381         # return original taskgraph if no regular expression is passed
 382         if not tasksregex:
 383             return taskgraph
 384         named_links_dict = taskgraph.graph.named_links_dict()
 385         filteredtasks = {}
 386         filterededges = set()
 387         regexprogram = re.compile(tasksregex)
 388
 389         for key in taskgraph.graph.visit_postorder():
 390             task = taskgraph.tasks[key]
 391             if regexprogram.match(task.label):
 392                 filteredtasks[key] = task
 393                 for depname, dep in six.iteritems(named_links_dict[key]):
 394                     if regexprogram.match(dep):
 395                         filterededges.add((key, dep, depname))
 396         filtered_taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))
 397         return filtered_taskgraph
 398
 399     def show_actions(self, options):
 400         import taskgraph.parameters
 401         import taskgraph.generator
 402         import taskgraph
 403         import taskgraph.actions
 404
 405         try:
 406             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
 407             parameters = taskgraph.parameters.parameters_loader(options['parameters'])
 408
 409             tgg = taskgraph.generator.TaskGraphGenerator(
 410                 root_dir=options.get('root'),
 411                 parameters=parameters,
 412             )
 413
 414             actions = taskgraph.actions.render_actions_json(
 415                 tgg.parameters, tgg.graph_config, decision_task_id="DECISION-TASK",
 416                 )
 417             print(json.dumps(actions, sort_keys=True, indent=2, separators=(',', ': ')))
 418         except Exception:
 419             traceback.print_exc()
 420             sys.exit(1)
 421
 422
 423 @CommandProvider
 424 class TaskClusterImagesProvider(MachCommandBase):
 425     @Command('taskcluster-load-image', category="ci",
 426              description="Load a pre-built Docker image. Note that you need to "
 427                          "have docker installed and running for this to work.")
 428     @CommandArgument('--task-id',
 429                      help="Load the image at public/image.tar.zst in this task, "
 430                           "rather than searching the index")
 431     @CommandArgument('-t', '--tag',
 432                      help="tag that the image should be loaded as. If not "
 433                           "image will be loaded with tag from the tarball",
 434                      metavar="name:tag")
 435     @CommandArgument('image_name', nargs='?',
 436                      help="Load the image of this name based on the current "
 437                           "contents of the tree (as built for mozilla-central "
 438                           "or mozilla-inbound)")
 439     def load_image(self, image_name, task_id, tag):
 440         self._ensure_zstd()
 441         from taskgraph.docker import load_image_by_name, load_image_by_task_id
 442         if not image_name and not task_id:
 443             print("Specify either IMAGE-NAME or TASK-ID")
 444             sys.exit(1)
 445         try:
 446             if task_id:
 447                 ok = load_image_by_task_id(task_id, tag)
 448             else:
 449                 ok = load_image_by_name(image_name, tag)
 450             if not ok:
 451                 sys.exit(1)
 452         except Exception:
 453             traceback.print_exc()
 454             sys.exit(1)
 455
 456     @Command('taskcluster-build-image', category='ci',
 457              description='Build a Docker image')
 458     @CommandArgument('image_name',
 459                      help='Name of the image to build')
 460     @CommandArgument('-t', '--tag',
 461                      help="tag that the image should be built as.",
 462                      metavar="name:tag")
 463     @CommandArgument('--context-only',
 464                      help="File name the context tarball should be written to."
 465                           "with this option it will only build the context.tar.",
 466                      metavar='context.tar')
 467     def build_image(self, image_name, tag, context_only):
 468         from taskgraph.docker import build_image, build_context
 469         try:
 470             if context_only is None:
 471                 build_image(image_name, tag, os.environ)
 472             else:
 473                 build_context(image_name, context_only, os.environ)
 474         except Exception:
 475             traceback.print_exc()
 476             sys.exit(1)
 477
 478
 479 @CommandProvider
 480 class TaskClusterPartialsData(MachCommandBase):
 481     @Command('release-history', category="ci",
 482              description="Query balrog for release history used by enable partials generation")
 483     @CommandArgument('-b', '--branch',
 484                      help="The gecko project branch used in balrog, such as "
 485                           "mozilla-central, release, maple")
 486     @CommandArgument('--product', default='Firefox',
 487                      help="The product identifier, such as 'Firefox'")
 488     def generate_partials_builds(self, product, branch):
 489         from taskgraph.util.partials import populate_release_history
 490         try:
 491             import yaml
 492             release_history = {'release_history': populate_release_history(product, branch)}
 493             print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
 494         except Exception:
 495             traceback.print_exc()
 496             sys.exit(1)