taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 from __future__ import absolute_import, print_function, unicode_literals
   9
  10 import json
  11 import logging
  12 import os
  13 import sys
  14 import traceback
  15 import re
  16
  17 from mach.decorators import (
  18     CommandArgument,
  19     CommandProvider,
  20     Command,
  21     SubCommand,
  22 )
  23
  24 from mozbuild.base import MachCommandBase
  25
  26
  27 class ShowTaskGraphSubCommand(SubCommand):
  28     """A SubCommand with TaskGraph-specific arguments"""
  29
  30     def __call__(self, func):
  31         after = SubCommand.__call__(self, func)
  32         args = [
  33             CommandArgument('--root', '-r',
  34                             help="root of the taskgraph definition relative to topsrcdir"),
  35             CommandArgument('--quiet', '-q', action="store_true",
  36                             help="suppress all logging output"),
  37             CommandArgument('--verbose', '-v', action="store_true",
  38                             help="include debug-level logging output"),
  39             CommandArgument('--json', '-J', action="store_const",
  40                             dest="format", const="json",
  41                             help="Output task graph as a JSON object"),
  42             CommandArgument('--labels', '-L', action="store_const",
  43                             dest="format", const="labels",
  44                             help="Output the label for each task in the task graph (default)"),
  45             CommandArgument('--parameters', '-p', default="project=mozilla-central",
  46                             help="parameters file (.yml or .json; see "
  47                                  "`taskcluster/docs/parameters.rst`)`"),
  48             CommandArgument('--no-optimize', dest="optimize", action="store_false",
  49                             default="true",
  50                             help="do not remove tasks from the graph that are found in the "
  51                             "index (a.k.a. optimize the graph)"),
  52             CommandArgument('--tasks-regex', '--tasks', default=None,
  53                             help="only return tasks with labels matching this regular "
  54                             "expression."),
  55             CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
  56                             help="enable fast task generation for local debugging.")
  57
  58         ]
  59         for arg in args:
  60             after = arg(after)
  61         return after
  62
  63
  64 @CommandProvider
  65 class MachCommands(MachCommandBase):
  66
  67     @Command('taskgraph', category="ci",
  68              description="Manipulate TaskCluster task graphs defined in-tree")
  69     def taskgraph(self):
  70         """The taskgraph subcommands all relate to the generation of task graphs
  71         for Gecko continuous integration.  A task graph is a set of tasks linked
  72         by dependencies: for example, a binary must be built before it is tested,
  73         and that build may further depend on various toolchains, libraries, etc.
  74         """
  75
  76     @ShowTaskGraphSubCommand('taskgraph', 'tasks',
  77                              description="Show all tasks in the taskgraph")
  78     def taskgraph_tasks(self, **options):
  79         return self.show_taskgraph('full_task_set', options)
  80
  81     @ShowTaskGraphSubCommand('taskgraph', 'full',
  82                              description="Show the full taskgraph")
  83     def taskgraph_full(self, **options):
  84         return self.show_taskgraph('full_task_graph', options)
  85
  86     @ShowTaskGraphSubCommand('taskgraph', 'target',
  87                              description="Show the target task set")
  88     def taskgraph_target(self, **options):
  89         return self.show_taskgraph('target_task_set', options)
  90
  91     @ShowTaskGraphSubCommand('taskgraph', 'target-graph',
  92                              description="Show the target taskgraph")
  93     def taskgraph_target_taskgraph(self, **options):
  94         return self.show_taskgraph('target_task_graph', options)
  95
  96     @ShowTaskGraphSubCommand('taskgraph', 'optimized',
  97                              description="Show the optimized taskgraph")
  98     def taskgraph_optimized(self, **options):
  99         return self.show_taskgraph('optimized_task_graph', options)
 100
 101     @ShowTaskGraphSubCommand('taskgraph', 'morphed',
 102                              description="Show the morphed taskgraph")
 103     def taskgraph_morphed(self, **options):
 104         return self.show_taskgraph('morphed_task_graph', options)
 105
 106     @SubCommand('taskgraph', 'decision',
 107                 description="Run the decision task")
 108     @CommandArgument('--root', '-r',
 109                      help="root of the taskgraph definition relative to topsrcdir")
 110     @CommandArgument('--base-repository',
 111                      required=True,
 112                      help='URL for "base" repository to clone')
 113     @CommandArgument('--head-repository',
 114                      required=True,
 115                      help='URL for "head" repository to fetch revision from')
 116     @CommandArgument('--head-ref',
 117                      required=True,
 118                      help='Reference (this is same as rev usually for hg)')
 119     @CommandArgument('--head-rev',
 120                      required=True,
 121                      help='Commit revision to use from head repository')
 122     @CommandArgument('--comm-base-repository',
 123                      required=False,
 124                      help='URL for "base" comm-* repository to clone')
 125     @CommandArgument('--comm-head-repository',
 126                      required=False,
 127                      help='URL for "head" comm-* repository to fetch revision from')
 128     @CommandArgument('--comm-head-ref',
 129                      required=False,
 130                      help='comm-* Reference (this is same as rev usually for hg)')
 131     @CommandArgument('--comm-head-rev',
 132                      required=False,
 133                      help='Commit revision to use from head comm-* repository')
 134     @CommandArgument('--message',
 135                      required=True,
 136                      help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
 137     @CommandArgument('--project',
 138                      required=True,
 139                      help='Project to use for creating task graph. Example: --project=try')
 140     @CommandArgument('--pushlog-id',
 141                      dest='pushlog_id',
 142                      required=True,
 143                      default=0)
 144     @CommandArgument('--pushdate',
 145                      dest='pushdate',
 146                      required=True,
 147                      type=int,
 148                      default=0)
 149     @CommandArgument('--owner',
 150                      required=True,
 151                      help='email address of who owns this graph')
 152     @CommandArgument('--level',
 153                      required=True,
 154                      help='SCM level of this repository')
 155     @CommandArgument('--target-tasks-method',
 156                      help='method for selecting the target tasks to generate')
 157     @CommandArgument('--try-task-config-file',
 158                      help='path to try task configuration file')
 159     def taskgraph_decision(self, **options):
 160         """Run the decision task: generate a task graph and submit to
 161         TaskCluster.  This is only meant to be called within decision tasks,
 162         and requires a great many arguments.  Commands like `mach taskgraph
 163         optimized` are better suited to use on the command line, and can take
 164         the parameters file generated by a decision task.  """
 165
 166         import taskgraph.decision
 167         try:
 168             self.setup_logging()
 169             return taskgraph.decision.taskgraph_decision(options)
 170         except Exception:
 171             traceback.print_exc()
 172             sys.exit(1)
 173
 174     @SubCommand('taskgraph', 'cron',
 175                 description="Run the cron task")
 176     @CommandArgument('--base-repository',
 177                      required=False,
 178                      help='(ignored)')
 179     @CommandArgument('--head-repository',
 180                      required=True,
 181                      help='URL for "head" repository to fetch')
 182     @CommandArgument('--head-ref',
 183                      required=False,
 184                      help='(ignored)')
 185     @CommandArgument('--project',
 186                      required=True,
 187                      help='Project to use for creating tasks. Example: --project=mozilla-central')
 188     @CommandArgument('--level',
 189                      required=True,
 190                      help='SCM level of this repository')
 191     @CommandArgument('--force-run',
 192                      required=False,
 193                      help='If given, force this cronjob to run regardless of time, '
 194                      'and run no others')
 195     @CommandArgument('--no-create',
 196                      required=False,
 197                      action='store_true',
 198                      help='Do not actually create tasks')
 199     @CommandArgument('--root', '-r',
 200                      required=False,
 201                      help="root of the repository to get cron task definitions from")
 202     def taskgraph_cron(self, **options):
 203         """Run the cron task; this task creates zero or more decision tasks.  It is run
 204         from the hooks service on a regular basis."""
 205         import taskgraph.cron
 206         try:
 207             self.setup_logging()
 208             return taskgraph.cron.taskgraph_cron(options)
 209         except Exception:
 210             traceback.print_exc()
 211             sys.exit(1)
 212
 213     @SubCommand('taskgraph', 'action-callback',
 214                 description='Run action callback used by action tasks')
 215     @CommandArgument('--root', '-r', default='taskcluster/ci',
 216                      help="root of the taskgraph definition relative to topsrcdir")
 217     def action_callback(self, **options):
 218         import taskgraph.actions
 219         try:
 220             self.setup_logging()
 221
 222             task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
 223             task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
 224             task = json.loads(os.environ.get('ACTION_TASK', 'null'))
 225             input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
 226             callback = os.environ.get('ACTION_CALLBACK', None)
 227             parameters = json.loads(os.environ.get('ACTION_PARAMETERS', '{}'))
 228             root = options['root']
 229
 230             return taskgraph.actions.trigger_action_callback(
 231                     task_group_id=task_group_id,
 232                     task_id=task_id,
 233                     task=task,
 234                     input=input,
 235                     callback=callback,
 236                     parameters=parameters,
 237                     root=root,
 238                     test=False)
 239         except Exception:
 240             traceback.print_exc()
 241             sys.exit(1)
 242
 243     @SubCommand('taskgraph', 'test-action-callback',
 244                 description='Run an action callback in a testing mode')
 245     @CommandArgument('--root', '-r', default='taskcluster/ci',
 246                      help="root of the taskgraph definition relative to topsrcdir")
 247     @CommandArgument('--parameters', '-p', default='project=mozilla-central',
 248                      help='parameters file (.yml or .json; see '
 249                           '`taskcluster/docs/parameters.rst`)`')
 250     @CommandArgument('--task-id', default=None,
 251                      help='TaskId to which the action applies')
 252     @CommandArgument('--task-group-id', default=None,
 253                      help='TaskGroupId to which the action applies')
 254     @CommandArgument('--input', default=None,
 255                      help='Action input (.yml or .json)')
 256     @CommandArgument('--task', default=None,
 257                      help='Task definition (.yml or .json; if omitted, the task will be'
 258                           'fetched from the queue)')
 259     @CommandArgument('callback', default=None,
 260                      help='Action callback name (Python function name)')
 261     def test_action_callback(self, **options):
 262         import taskgraph.parameters
 263         from taskgraph.util.taskcluster import get_task_definition
 264         import taskgraph.actions
 265         import yaml
 266
 267         def load_data(filename):
 268             with open(filename) as f:
 269                 if filename.endswith('.yml'):
 270                     return yaml.safe_load(f)
 271                 elif filename.endswith('.json'):
 272                     return json.load(f)
 273                 else:
 274                     raise Exception("unknown filename {}".format(filename))
 275
 276         try:
 277             self.setup_logging()
 278             task_id = options['task_id']
 279             if options['task']:
 280                 task = load_data(options['task'])
 281             elif task_id:
 282                 task = get_task_definition(task_id)
 283             else:
 284                 task = None
 285
 286             if options['input']:
 287                 input = load_data(options['input'])
 288             else:
 289                 input = None
 290
 291             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
 292             parameters.check()
 293
 294             root = options['root']
 295
 296             return taskgraph.actions.trigger_action_callback(
 297                     task_group_id=options['task_group_id'],
 298                     task_id=task_id,
 299                     task=task,
 300                     input=input,
 301                     callback=options['callback'],
 302                     parameters=parameters,
 303                     root=root,
 304                     test=True)
 305         except Exception:
 306             traceback.print_exc()
 307             sys.exit(1)
 308
 309     def setup_logging(self, quiet=False, verbose=True):
 310         """
 311         Set up Python logging for all loggers, sending results to stderr (so
 312         that command output can be redirected easily) and adding the typical
 313         mach timestamp.
 314         """
 315         # remove the old terminal handler
 316         old = self.log_manager.replace_terminal_handler(None)
 317
 318         # re-add it, with level and fh set appropriately
 319         if not quiet:
 320             level = logging.DEBUG if verbose else logging.INFO
 321             self.log_manager.add_terminal_logging(
 322                 fh=sys.stderr, level=level,
 323                 write_interval=old.formatter.write_interval,
 324                 write_times=old.formatter.write_times)
 325
 326         # all of the taskgraph logging is unstructured logging
 327         self.log_manager.enable_unstructured()
 328
 329     def show_taskgraph(self, graph_attr, options):
 330         import taskgraph.parameters
 331         import taskgraph.target_tasks
 332         import taskgraph.generator
 333         import taskgraph
 334         if options['fast']:
 335             taskgraph.fast = True
 336
 337         try:
 338             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
 339             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
 340             parameters.check()
 341
 342             tgg = taskgraph.generator.TaskGraphGenerator(
 343                 root_dir=options.get('root'),
 344                 parameters=parameters)
 345
 346             tg = getattr(tgg, graph_attr)
 347
 348             show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
 349             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
 350             show_method(tg)
 351         except Exception:
 352             traceback.print_exc()
 353             sys.exit(1)
 354
 355     def show_taskgraph_labels(self, taskgraph):
 356         for index in taskgraph.graph.visit_postorder():
 357             print(taskgraph.tasks[index].label)
 358
 359     def show_taskgraph_json(self, taskgraph):
 360         print(json.dumps(taskgraph.to_json(),
 361               sort_keys=True, indent=2, separators=(',', ': ')))
 362
 363     def get_filtered_taskgraph(self, taskgraph, tasksregex):
 364         from taskgraph.graph import Graph
 365         from taskgraph.taskgraph import TaskGraph
 366         """
 367         This class method filters all the tasks on basis of a regular expression
 368         and returns a new TaskGraph object
 369         """
 370         # return original taskgraph if no regular expression is passed
 371         if not tasksregex:
 372             return taskgraph
 373         named_links_dict = taskgraph.graph.named_links_dict()
 374         filteredtasks = {}
 375         filterededges = set()
 376         regexprogram = re.compile(tasksregex)
 377
 378         for key in taskgraph.graph.visit_postorder():
 379             task = taskgraph.tasks[key]
 380             if regexprogram.match(task.label):
 381                 filteredtasks[key] = task
 382                 for depname, dep in named_links_dict[key].iteritems():
 383                     if regexprogram.match(dep):
 384                         filterededges.add((key, dep, depname))
 385         filtered_taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))
 386         return filtered_taskgraph
 387
 388
 389 @CommandProvider
 390 class TaskClusterImagesProvider(MachCommandBase):
 391     def _ensure_zstd(self):
 392         try:
 393             import zstd
 394             # There are two zstd libraries that exist in the wild, ensure we
 395             # have the right one.
 396             zstd.ZstdCompressor
 397             zstd.ZstdDecompressor
 398         except (ImportError, AttributeError):
 399             self._activate_virtualenv()
 400             self.virtualenv_manager.install_pip_package('zstandard==0.8.1')
 401
 402     @Command('taskcluster-load-image', category="ci",
 403              description="Load a pre-built Docker image")
 404     @CommandArgument('--task-id',
 405                      help="Load the image at public/image.tar.zst in this task,"
 406                           "rather than searching the index")
 407     @CommandArgument('-t', '--tag',
 408                      help="tag that the image should be loaded as. If not "
 409                           "image will be loaded with tag from the tarball",
 410                      metavar="name:tag")
 411     @CommandArgument('image_name', nargs='?',
 412                      help="Load the image of this name based on the current"
 413                           "contents of the tree (as built for mozilla-central"
 414                           "or mozilla-inbound)")
 415     def load_image(self, image_name, task_id, tag):
 416         self._ensure_zstd()
 417         from taskgraph.docker import load_image_by_name, load_image_by_task_id
 418         if not image_name and not task_id:
 419             print("Specify either IMAGE-NAME or TASK-ID")
 420             sys.exit(1)
 421         try:
 422             if task_id:
 423                 ok = load_image_by_task_id(task_id, tag)
 424             else:
 425                 ok = load_image_by_name(image_name, tag)
 426             if not ok:
 427                 sys.exit(1)
 428         except Exception:
 429             traceback.print_exc()
 430             sys.exit(1)
 431
 432     @Command('taskcluster-build-image', category='ci',
 433              description='Build a Docker image')
 434     @CommandArgument('image_name',
 435                      help='Name of the image to build')
 436     @CommandArgument('-t', '--tag',
 437                      help="tag that the image should be built as.",
 438                      metavar="name:tag")
 439     @CommandArgument('--context-only',
 440                      help="File name the context tarball should be written to."
 441                           "with this option it will only build the context.tar.",
 442                      metavar='context.tar')
 443     def build_image(self, image_name, tag, context_only):
 444         from taskgraph.docker import build_image, build_context
 445         try:
 446             if context_only is None:
 447                 build_image(image_name, tag, os.environ)
 448             else:
 449                 build_context(image_name, context_only, os.environ)
 450         except Exception:
 451             traceback.print_exc()
 452             sys.exit(1)
 453
 454
 455 @CommandProvider
 456 class TaskClusterPartialsData(object):
 457     @Command('release-history', category="ci",
 458              description="Query balrog for release history used by enable partials generation")
 459     @CommandArgument('-b', '--branch',
 460                      help="The gecko project branch used in balrog, such as "
 461                           "mozilla-central, release, date")
 462     @CommandArgument('--product', default='Firefox',
 463                      help="The product identifier, such as 'Firefox'")
 464     def generate_partials_builds(self, product, branch):
 465         from taskgraph.util.partials import populate_release_history
 466         try:
 467             import yaml
 468             release_history = {'release_history': populate_release_history(product, branch)}
 469             print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
 470         except Exception:
 471             traceback.print_exc()
 472             sys.exit(1)