taskcluster/mach_commands.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7
   8 from __future__ import absolute_import, print_function, unicode_literals
   9
  10 import argparse
  11 import json
  12 import logging
  13 import os
  14 from six import text_type
  15 import six
  16 import sys
  17 import time
  18 import traceback
  19 import re
  20
  21 from mach.decorators import (
  22     CommandArgument,
  23     CommandProvider,
  24     Command,
  25     SubCommand,
  26 )
  27
  28 from mozbuild.base import MachCommandBase
  29
  30
  31 def strtobool(value):
  32     """Convert string to boolean.
  33
  34     Wraps "distutils.util.strtobool", deferring the import of the package
  35     in case it's not installed. Otherwise, we have a "chicken and egg problem" where
  36     |mach bootstrap| would install the required package to enable "distutils.util", but
  37     it can't because mach fails to interpret this file.
  38     """
  39     from distutils.util import strtobool
  40
  41     return bool(strtobool(value))
  42
  43
  44 class ShowTaskGraphSubCommand(SubCommand):
  45     """A SubCommand with TaskGraph-specific arguments"""
  46
  47     def __call__(self, func):
  48         after = SubCommand.__call__(self, func)
  49         args = [
  50             CommandArgument(
  51                 "--root",
  52                 "-r",
  53                 help="root of the taskgraph definition relative to topsrcdir",
  54             ),
  55             CommandArgument(
  56                 "--quiet", "-q", action="store_true", help="suppress all logging output"
  57             ),
  58             CommandArgument(
  59                 "--verbose",
  60                 "-v",
  61                 action="store_true",
  62                 help="include debug-level logging output",
  63             ),
  64             CommandArgument(
  65                 "--json",
  66                 "-J",
  67                 action="store_const",
  68                 dest="format",
  69                 const="json",
  70                 help="Output task graph as a JSON object",
  71             ),
  72             CommandArgument(
  73                 "--labels",
  74                 "-L",
  75                 action="store_const",
  76                 dest="format",
  77                 const="labels",
  78                 help="Output the label for each task in the task graph (default)",
  79             ),
  80             CommandArgument(
  81                 "--parameters",
  82                 "-p",
  83                 default="project=mozilla-central",
  84                 help="parameters file (.yml or .json; see "
  85                 "`taskcluster/docs/parameters.rst`)`",
  86             ),
  87             CommandArgument(
  88                 "--no-optimize",
  89                 dest="optimize",
  90                 action="store_false",
  91                 default="true",
  92                 help="do not remove tasks from the graph that are found in the "
  93                 "index (a.k.a. optimize the graph)",
  94             ),
  95             CommandArgument(
  96                 "--tasks-regex",
  97                 "--tasks",
  98                 default=None,
  99                 help="only return tasks with labels matching this regular "
 100                 "expression.",
 101             ),
 102             CommandArgument(
 103                 "--target-kind",
 104                 default=None,
 105                 help="only return tasks that are of the given kind, "
 106                 "or their dependencies.",
 107             ),
 108             CommandArgument(
 109                 "-F",
 110                 "--fast",
 111                 dest="fast",
 112                 default=False,
 113                 action="store_true",
 114                 help="enable fast task generation for local debugging.",
 115             ),
 116             CommandArgument(
 117                 "-o",
 118                 "--output-file",
 119                 default=None,
 120                 help="file path to store generated output.",
 121             ),
 122         ]
 123         for arg in args:
 124             after = arg(after)
 125         return after
 126
 127
 128 @CommandProvider
 129 class MachCommands(MachCommandBase):
 130     @Command(
 131         "taskgraph",
 132         category="ci",
 133         description="Manipulate TaskCluster task graphs defined in-tree",
 134     )
 135     def taskgraph(self):
 136         """The taskgraph subcommands all relate to the generation of task graphs
 137         for Gecko continuous integration.  A task graph is a set of tasks linked
 138         by dependencies: for example, a binary must be built before it is tested,
 139         and that build may further depend on various toolchains, libraries, etc.
 140         """
 141
 142     @ShowTaskGraphSubCommand(
 143         "taskgraph", "tasks", description="Show all tasks in the taskgraph"
 144     )
 145     def taskgraph_tasks(self, **options):
 146         return self.show_taskgraph("full_task_set", options)
 147
 148     @ShowTaskGraphSubCommand("taskgraph", "full", description="Show the full taskgraph")
 149     def taskgraph_full(self, **options):
 150         return self.show_taskgraph("full_task_graph", options)
 151
 152     @ShowTaskGraphSubCommand(
 153         "taskgraph", "target", description="Show the target task set"
 154     )
 155     def taskgraph_target(self, **options):
 156         return self.show_taskgraph("target_task_set", options)
 157
 158     @ShowTaskGraphSubCommand(
 159         "taskgraph", "target-graph", description="Show the target taskgraph"
 160     )
 161     def taskgraph_target_taskgraph(self, **options):
 162         return self.show_taskgraph("target_task_graph", options)
 163
 164     @ShowTaskGraphSubCommand(
 165         "taskgraph", "optimized", description="Show the optimized taskgraph"
 166     )
 167     def taskgraph_optimized(self, **options):
 168         return self.show_taskgraph("optimized_task_graph", options)
 169
 170     @ShowTaskGraphSubCommand(
 171         "taskgraph", "morphed", description="Show the morphed taskgraph"
 172     )
 173     def taskgraph_morphed(self, **options):
 174         return self.show_taskgraph("morphed_task_graph", options)
 175
 176     @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
 177     @CommandArgument(
 178         "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
 179     )
 180     @CommandArgument(
 181         "--quiet", "-q", action="store_true", help="suppress all logging output"
 182     )
 183     @CommandArgument(
 184         "--verbose",
 185         "-v",
 186         action="store_true",
 187         help="include debug-level logging output",
 188     )
 189     @CommandArgument(
 190         "--parameters",
 191         "-p",
 192         default="project=mozilla-central",
 193         help="parameters file (.yml or .json; see "
 194         "`taskcluster/docs/parameters.rst`)`",
 195     )
 196     def taskgraph_actions(self, **options):
 197         return self.show_actions(options)
 198
 199     @SubCommand("taskgraph", "decision", description="Run the decision task")
 200     @CommandArgument(
 201         "--root",
 202         "-r",
 203         type=text_type,
 204         help="root of the taskgraph definition relative to topsrcdir",
 205     )
 206     @CommandArgument(
 207         "--base-repository",
 208         type=text_type,
 209         required=True,
 210         help='URL for "base" repository to clone',
 211     )
 212     @CommandArgument(
 213         "--head-repository",
 214         type=text_type,
 215         required=True,
 216         help='URL for "head" repository to fetch revision from',
 217     )
 218     @CommandArgument(
 219         "--head-ref",
 220         type=text_type,
 221         required=True,
 222         help="Reference (this is same as rev usually for hg)",
 223     )
 224     @CommandArgument(
 225         "--head-rev",
 226         type=text_type,
 227         required=True,
 228         help="Commit revision to use from head repository",
 229     )
 230     @CommandArgument(
 231         "--comm-base-repository",
 232         type=text_type,
 233         required=False,
 234         help='URL for "base" comm-* repository to clone',
 235     )
 236     @CommandArgument(
 237         "--comm-head-repository",
 238         type=text_type,
 239         required=False,
 240         help='URL for "head" comm-* repository to fetch revision from',
 241     )
 242     @CommandArgument(
 243         "--comm-head-ref",
 244         type=text_type,
 245         required=False,
 246         help="comm-* Reference (this is same as rev usually for hg)",
 247     )
 248     @CommandArgument(
 249         "--comm-head-rev",
 250         type=text_type,
 251         required=False,
 252         help="Commit revision to use from head comm-* repository",
 253     )
 254     @CommandArgument(
 255         "--project",
 256         type=text_type,
 257         required=True,
 258         help="Project to use for creating task graph. Example: --project=try",
 259     )
 260     @CommandArgument(
 261         "--pushlog-id", type=text_type, dest="pushlog_id", required=True, default="0"
 262     )
 263     @CommandArgument("--pushdate", dest="pushdate", required=True, type=int, default=0)
 264     @CommandArgument(
 265         "--owner",
 266         type=text_type,
 267         required=True,
 268         help="email address of who owns this graph",
 269     )
 270     @CommandArgument(
 271         "--level", type=text_type, required=True, help="SCM level of this repository"
 272     )
 273     @CommandArgument(
 274         "--target-tasks-method",
 275         type=text_type,
 276         help="method for selecting the target tasks to generate",
 277     )
 278     @CommandArgument(
 279         "--optimize-target-tasks",
 280         type=lambda flag: strtobool(flag),
 281         nargs="?",
 282         const="true",
 283         help="If specified, this indicates whether the target "
 284         "tasks are eligible for optimization. Otherwise, "
 285         "the default for the project is used.",
 286     )
 287     @CommandArgument(
 288         "--try-task-config-file",
 289         type=text_type,
 290         help="path to try task configuration file",
 291     )
 292     @CommandArgument(
 293         "--tasks-for",
 294         type=text_type,
 295         required=True,
 296         help="the tasks_for value used to generate this task",
 297     )
 298     @CommandArgument(
 299         "--include-push-tasks",
 300         action="store_true",
 301         help="Whether tasks from the on-push graph should be re-used "
 302         "in this graph. This allows cron graphs to avoid rebuilding "
 303         "jobs that were built on-push.",
 304     )
 305     @CommandArgument(
 306         "--rebuild-kind",
 307         dest="rebuild_kinds",
 308         action="append",
 309         default=argparse.SUPPRESS,
 310         help="Kinds that should not be re-used from the on-push graph.",
 311     )
 312     def taskgraph_decision(self, **options):
 313         """Run the decision task: generate a task graph and submit to
 314         TaskCluster.  This is only meant to be called within decision tasks,
 315         and requires a great many arguments.  Commands like `mach taskgraph
 316         optimized` are better suited to use on the command line, and can take
 317         the parameters file generated by a decision task."""
 318
 319         import taskgraph.decision
 320
 321         try:
 322             self.setup_logging()
 323             start = time.monotonic()
 324             ret = taskgraph.decision.taskgraph_decision(options)
 325             end = time.monotonic()
 326             if os.environ.get("MOZ_AUTOMATION") == "1":
 327                 perfherder_data = {
 328                     "framework": {"name": "build_metrics"},
 329                     "suites": [
 330                         {
 331                             "name": "decision",
 332                             "value": end - start,
 333                             "lowerIsBetter": True,
 334                             "shouldAlert": True,
 335                             "subtests": [],
 336                         }
 337                     ],
 338                 }
 339                 print(
 340                     "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
 341                     file=sys.stderr,
 342                 )
 343             return ret
 344         except Exception:
 345             traceback.print_exc()
 346             sys.exit(1)
 347
 348     @SubCommand(
 349         "taskgraph",
 350         "cron",
 351         description="Provide a pointer to the new `.cron.yml` handler.",
 352     )
 353     def taskgraph_cron(self, **options):
 354         print(
 355             'Handling of ".cron.yml" files has move to '
 356             "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
 357         )
 358         sys.exit(1)
 359
 360     @SubCommand(
 361         "taskgraph",
 362         "action-callback",
 363         description="Run action callback used by action tasks",
 364     )
 365     @CommandArgument(
 366         "--root",
 367         "-r",
 368         default="taskcluster/ci",
 369         help="root of the taskgraph definition relative to topsrcdir",
 370     )
 371     def action_callback(self, **options):
 372         from taskgraph.actions import trigger_action_callback
 373         from taskgraph.actions.util import get_parameters
 374
 375         try:
 376             self.setup_logging()
 377
 378             # the target task for this action (or null if it's a group action)
 379             task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null"))
 380             # the target task group for this action
 381             task_group_id = os.environ.get("ACTION_TASK_GROUP_ID", None)
 382             input = json.loads(os.environ.get("ACTION_INPUT", "null"))
 383             callback = os.environ.get("ACTION_CALLBACK", None)
 384             root = options["root"]
 385
 386             parameters = get_parameters(task_group_id)
 387
 388             return trigger_action_callback(
 389                 task_group_id=task_group_id,
 390                 task_id=task_id,
 391                 input=input,
 392                 callback=callback,
 393                 parameters=parameters,
 394                 root=root,
 395                 test=False,
 396             )
 397         except Exception:
 398             traceback.print_exc()
 399             sys.exit(1)
 400
 401     @SubCommand(
 402         "taskgraph",
 403         "test-action-callback",
 404         description="Run an action callback in a testing mode",
 405     )
 406     @CommandArgument(
 407         "--root",
 408         "-r",
 409         default="taskcluster/ci",
 410         help="root of the taskgraph definition relative to topsrcdir",
 411     )
 412     @CommandArgument(
 413         "--parameters",
 414         "-p",
 415         default="project=mozilla-central",
 416         help="parameters file (.yml or .json; see "
 417         "`taskcluster/docs/parameters.rst`)`",
 418     )
 419     @CommandArgument(
 420         "--task-id", default=None, help="TaskId to which the action applies"
 421     )
 422     @CommandArgument(
 423         "--task-group-id", default=None, help="TaskGroupId to which the action applies"
 424     )
 425     @CommandArgument("--input", default=None, help="Action input (.yml or .json)")
 426     @CommandArgument(
 427         "callback", default=None, help="Action callback name (Python function name)"
 428     )
 429     def test_action_callback(self, **options):
 430         import taskgraph.parameters
 431         import taskgraph.actions
 432         from taskgraph.util import yaml
 433
 434         def load_data(filename):
 435             with open(filename) as f:
 436                 if filename.endswith(".yml"):
 437                     return yaml.load_stream(f)
 438                 elif filename.endswith(".json"):
 439                     return json.load(f)
 440                 else:
 441                     raise Exception("unknown filename {}".format(filename))
 442
 443         try:
 444             self.setup_logging()
 445             task_id = options["task_id"]
 446
 447             if options["input"]:
 448                 input = load_data(options["input"])
 449             else:
 450                 input = None
 451
 452             parameters = taskgraph.parameters.load_parameters_file(
 453                 options["parameters"],
 454                 strict=False,
 455                 # FIXME: There should be a way to parameterize this.
 456                 trust_domain="gecko",
 457             )
 458             parameters.check()
 459
 460             root = options["root"]
 461
 462             return taskgraph.actions.trigger_action_callback(
 463                 task_group_id=options["task_group_id"],
 464                 task_id=task_id,
 465                 input=input,
 466                 callback=options["callback"],
 467                 parameters=parameters,
 468                 root=root,
 469                 test=True,
 470             )
 471         except Exception:
 472             traceback.print_exc()
 473             sys.exit(1)
 474
 475     def setup_logging(self, quiet=False, verbose=True):
 476         """
 477         Set up Python logging for all loggers, sending results to stderr (so
 478         that command output can be redirected easily) and adding the typical
 479         mach timestamp.
 480         """
 481         # remove the old terminal handler
 482         old = self.log_manager.replace_terminal_handler(None)
 483
 484         # re-add it, with level and fh set appropriately
 485         if not quiet:
 486             level = logging.DEBUG if verbose else logging.INFO
 487             self.log_manager.add_terminal_logging(
 488                 fh=sys.stderr,
 489                 level=level,
 490                 write_interval=old.formatter.write_interval,
 491                 write_times=old.formatter.write_times,
 492             )
 493
 494         # all of the taskgraph logging is unstructured logging
 495         self.log_manager.enable_unstructured()
 496
 497     def show_taskgraph(self, graph_attr, options):
 498         import taskgraph.parameters
 499         import taskgraph.generator
 500         import taskgraph
 501
 502         if options["fast"]:
 503             taskgraph.fast = True
 504
 505         try:
 506             self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
 507             parameters = taskgraph.parameters.parameters_loader(
 508                 options["parameters"],
 509                 overrides={"target-kind": options.get("target_kind")},
 510                 strict=False,
 511             )
 512
 513             tgg = taskgraph.generator.TaskGraphGenerator(
 514                 root_dir=options.get("root"),
 515                 parameters=parameters,
 516             )
 517
 518             tg = getattr(tgg, graph_attr)
 519
 520             show_method = getattr(
 521                 self, "show_taskgraph_" + (options["format"] or "labels")
 522             )
 523             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
 524
 525             fh = options["output_file"]
 526             if fh:
 527                 fh = open(fh, "w")
 528             show_method(tg, file=fh)
 529         except Exception:
 530             traceback.print_exc()
 531             sys.exit(1)
 532
 533     def show_taskgraph_labels(self, taskgraph, file=None):
 534         for index in taskgraph.graph.visit_postorder():
 535             print(taskgraph.tasks[index].label, file=file)
 536
 537     def show_taskgraph_json(self, taskgraph, file=None):
 538         print(
 539             json.dumps(
 540                 taskgraph.to_json(), sort_keys=True, indent=2, separators=(",", ": ")
 541             ),
 542             file=file,
 543         )
 544
 545     def get_filtered_taskgraph(self, taskgraph, tasksregex):
 546         from taskgraph.graph import Graph
 547         from taskgraph.taskgraph import TaskGraph
 548
 549         """
 550         This class method filters all the tasks on basis of a regular expression
 551         and returns a new TaskGraph object
 552         """
 553         # return original taskgraph if no regular expression is passed
 554         if not tasksregex:
 555             return taskgraph
 556         named_links_dict = taskgraph.graph.named_links_dict()
 557         filteredtasks = {}
 558         filterededges = set()
 559         regexprogram = re.compile(tasksregex)
 560
 561         for key in taskgraph.graph.visit_postorder():
 562             task = taskgraph.tasks[key]
 563             if regexprogram.match(task.label):
 564                 filteredtasks[key] = task
 565                 for depname, dep in six.iteritems(named_links_dict[key]):
 566                     if regexprogram.match(dep):
 567                         filterededges.add((key, dep, depname))
 568         filtered_taskgraph = TaskGraph(
 569             filteredtasks, Graph(set(filteredtasks), filterededges)
 570         )
 571         return filtered_taskgraph
 572
 573     def show_actions(self, options):
 574         import taskgraph.parameters
 575         import taskgraph.generator
 576         import taskgraph
 577         import taskgraph.actions
 578
 579         try:
 580             self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
 581             parameters = taskgraph.parameters.parameters_loader(options["parameters"])
 582
 583             tgg = taskgraph.generator.TaskGraphGenerator(
 584                 root_dir=options.get("root"),
 585                 parameters=parameters,
 586             )
 587
 588             actions = taskgraph.actions.render_actions_json(
 589                 tgg.parameters,
 590                 tgg.graph_config,
 591                 decision_task_id="DECISION-TASK",
 592             )
 593             print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
 594         except Exception:
 595             traceback.print_exc()
 596             sys.exit(1)
 597
 598
 599 @CommandProvider
 600 class TaskClusterImagesProvider(MachCommandBase):
 601     @Command(
 602         "taskcluster-load-image",
 603         category="ci",
 604         description="Load a pre-built Docker image. Note that you need to "
 605         "have docker installed and running for this to work.",
 606     )
 607     @CommandArgument(
 608         "--task-id",
 609         help="Load the image at public/image.tar.zst in this task, "
 610         "rather than searching the index",
 611     )
 612     @CommandArgument(
 613         "-t",
 614         "--tag",
 615         help="tag that the image should be loaded as. If not "
 616         "image will be loaded with tag from the tarball",
 617         metavar="name:tag",
 618     )
 619     @CommandArgument(
 620         "image_name",
 621         nargs="?",
 622         help="Load the image of this name based on the current "
 623         "contents of the tree (as built for mozilla-central "
 624         "or mozilla-inbound)",
 625     )
 626     def load_image(self, image_name, task_id, tag):
 627         from taskgraph.docker import load_image_by_name, load_image_by_task_id
 628
 629         if not image_name and not task_id:
 630             print("Specify either IMAGE-NAME or TASK-ID")
 631             sys.exit(1)
 632         try:
 633             if task_id:
 634                 ok = load_image_by_task_id(task_id, tag)
 635             else:
 636                 ok = load_image_by_name(image_name, tag)
 637             if not ok:
 638                 sys.exit(1)
 639         except Exception:
 640             traceback.print_exc()
 641             sys.exit(1)
 642
 643     @Command(
 644         "taskcluster-build-image", category="ci", description="Build a Docker image"
 645     )
 646     @CommandArgument("image_name", help="Name of the image to build")
 647     @CommandArgument(
 648         "-t", "--tag", help="tag that the image should be built as.", metavar="name:tag"
 649     )
 650     @CommandArgument(
 651         "--context-only",
 652         help="File name the context tarball should be written to."
 653         "with this option it will only build the context.tar.",
 654         metavar="context.tar",
 655     )
 656     def build_image(self, image_name, tag, context_only):
 657         from taskgraph.docker import build_image, build_context
 658
 659         try:
 660             if context_only is None:
 661                 build_image(image_name, tag, os.environ)
 662             else:
 663                 build_context(image_name, context_only, os.environ)
 664         except Exception:
 665             traceback.print_exc()
 666             sys.exit(1)
 667
 668
 669 @CommandProvider
 670 class TaskClusterPartialsData(MachCommandBase):
 671     @Command(
 672         "release-history",
 673         category="ci",
 674         description="Query balrog for release history used by enable partials generation",
 675     )
 676     @CommandArgument(
 677         "-b",
 678         "--branch",
 679         help="The gecko project branch used in balrog, such as "
 680         "mozilla-central, release, maple",
 681     )
 682     @CommandArgument(
 683         "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
 684     )
 685     def generate_partials_builds(self, product, branch):
 686         from taskgraph.util.partials import populate_release_history
 687
 688         try:
 689             import yaml
 690
 691             release_history = {
 692                 "release_history": populate_release_history(product, branch)
 693             }
 694             print(
 695                 yaml.safe_dump(
 696                     release_history, allow_unicode=True, default_flow_style=False
 697                 )
 698             )
 699         except Exception:
 700             traceback.print_exc()
 701             sys.exit(1)