Bug 1700051: part 35) Reduce accessibility of `mSoftText.mDOMMapping` to `private...
[gecko.git] / taskcluster / mach_commands.py
blobe62c0151f1a3bc6d3ce05fb6ecf23fa92b652280
1 # -*- coding: utf-8 -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 from __future__ import absolute_import, print_function, unicode_literals
9 import argparse
10 import json
11 import logging
12 import os
13 import re
14 import shlex
15 import subprocess
16 import sys
17 import tempfile
18 import time
19 import traceback
21 import six
22 from six import text_type
24 from mach.decorators import (
25 Command,
26 CommandArgument,
27 CommandProvider,
28 SettingsProvider,
29 SubCommand,
31 from mozbuild.base import MachCommandBase
33 logger = logging.getLogger("taskcluster")
36 @SettingsProvider
37 class TaskgraphConfig(object):
38 @classmethod
39 def config_settings(cls):
40 return [
42 "taskgraph.diffcmd",
43 "string",
44 "The command to run with `./mach taskgraph --diff`",
45 "diff --report-identical-files --color=always "
46 "--label={attr}@{base} --label={attr}@{cur} -U20",
47 {},
52 def strtobool(value):
53 """Convert string to boolean.
55 Wraps "distutils.util.strtobool", deferring the import of the package
56 in case it's not installed. Otherwise, we have a "chicken and egg problem" where
57 |mach bootstrap| would install the required package to enable "distutils.util", but
58 it can't because mach fails to interpret this file.
59 """
60 from distutils.util import strtobool
62 return bool(strtobool(value))
65 class ShowTaskGraphSubCommand(SubCommand):
66 """A SubCommand with TaskGraph-specific arguments"""
68 def __call__(self, func):
69 after = SubCommand.__call__(self, func)
70 args = [
71 CommandArgument(
72 "--root",
73 "-r",
74 help="root of the taskgraph definition relative to topsrcdir",
76 CommandArgument(
77 "--quiet", "-q", action="store_true", help="suppress all logging output"
79 CommandArgument(
80 "--verbose",
81 "-v",
82 action="store_true",
83 help="include debug-level logging output",
85 CommandArgument(
86 "--json",
87 "-J",
88 action="store_const",
89 dest="format",
90 const="json",
91 help="Output task graph as a JSON object",
93 CommandArgument(
94 "--labels",
95 "-L",
96 action="store_const",
97 dest="format",
98 const="labels",
99 help="Output the label for each task in the task graph (default)",
101 CommandArgument(
102 "--parameters",
103 "-p",
104 default="project=mozilla-central",
105 help="parameters file (.yml or .json; see "
106 "`taskcluster/docs/parameters.rst`)`",
108 CommandArgument(
109 "--no-optimize",
110 dest="optimize",
111 action="store_false",
112 default="true",
113 help="do not remove tasks from the graph that are found in the "
114 "index (a.k.a. optimize the graph)",
116 CommandArgument(
117 "--tasks-regex",
118 "--tasks",
119 default=None,
120 help="only return tasks with labels matching this regular "
121 "expression.",
123 CommandArgument(
124 "--target-kind",
125 default=None,
126 help="only return tasks that are of the given kind, "
127 "or their dependencies.",
129 CommandArgument(
130 "-F",
131 "--fast",
132 dest="fast",
133 default=False,
134 action="store_true",
135 help="enable fast task generation for local debugging.",
137 CommandArgument(
138 "-o",
139 "--output-file",
140 default=None,
141 help="file path to store generated output.",
143 CommandArgument(
144 "--diff",
145 const="default",
146 nargs="?",
147 default=None,
148 help="Generate and diff the current taskgraph against another revision. "
149 "Without args the base revision will be used. A revision specifier such as "
150 "the hash or `.~1` (hg) or `HEAD~1` (git) can be used as well.",
153 for arg in args:
154 after = arg(after)
155 return after
158 @CommandProvider
159 class MachCommands(MachCommandBase):
160 @Command(
161 "taskgraph",
162 category="ci",
163 description="Manipulate TaskCluster task graphs defined in-tree",
165 def taskgraph(self):
166 """The taskgraph subcommands all relate to the generation of task graphs
167 for Gecko continuous integration. A task graph is a set of tasks linked
168 by dependencies: for example, a binary must be built before it is tested,
169 and that build may further depend on various toolchains, libraries, etc.
172 @ShowTaskGraphSubCommand(
173 "taskgraph", "tasks", description="Show all tasks in the taskgraph"
175 def taskgraph_tasks(self, **options):
176 return self.show_taskgraph("full_task_set", options)
178 @ShowTaskGraphSubCommand("taskgraph", "full", description="Show the full taskgraph")
179 def taskgraph_full(self, **options):
180 return self.show_taskgraph("full_task_graph", options)
182 @ShowTaskGraphSubCommand(
183 "taskgraph", "target", description="Show the target task set"
185 def taskgraph_target(self, **options):
186 return self.show_taskgraph("target_task_set", options)
188 @ShowTaskGraphSubCommand(
189 "taskgraph", "target-graph", description="Show the target taskgraph"
191 def taskgraph_target_taskgraph(self, **options):
192 return self.show_taskgraph("target_task_graph", options)
194 @ShowTaskGraphSubCommand(
195 "taskgraph", "optimized", description="Show the optimized taskgraph"
197 def taskgraph_optimized(self, **options):
198 return self.show_taskgraph("optimized_task_graph", options)
200 @ShowTaskGraphSubCommand(
201 "taskgraph", "morphed", description="Show the morphed taskgraph"
203 def taskgraph_morphed(self, **options):
204 return self.show_taskgraph("morphed_task_graph", options)
206 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
207 @CommandArgument(
208 "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
210 @CommandArgument(
211 "--quiet", "-q", action="store_true", help="suppress all logging output"
213 @CommandArgument(
214 "--verbose",
215 "-v",
216 action="store_true",
217 help="include debug-level logging output",
219 @CommandArgument(
220 "--parameters",
221 "-p",
222 default="project=mozilla-central",
223 help="parameters file (.yml or .json; see "
224 "`taskcluster/docs/parameters.rst`)`",
226 def taskgraph_actions(self, **options):
227 return self.show_actions(options)
229 @SubCommand("taskgraph", "decision", description="Run the decision task")
230 @CommandArgument(
231 "--root",
232 "-r",
233 type=text_type,
234 help="root of the taskgraph definition relative to topsrcdir",
236 @CommandArgument(
237 "--base-repository",
238 type=text_type,
239 required=True,
240 help='URL for "base" repository to clone',
242 @CommandArgument(
243 "--head-repository",
244 type=text_type,
245 required=True,
246 help='URL for "head" repository to fetch revision from',
248 @CommandArgument(
249 "--head-ref",
250 type=text_type,
251 required=True,
252 help="Reference (this is same as rev usually for hg)",
254 @CommandArgument(
255 "--head-rev",
256 type=text_type,
257 required=True,
258 help="Commit revision to use from head repository",
260 @CommandArgument(
261 "--comm-base-repository",
262 type=text_type,
263 required=False,
264 help='URL for "base" comm-* repository to clone',
266 @CommandArgument(
267 "--comm-head-repository",
268 type=text_type,
269 required=False,
270 help='URL for "head" comm-* repository to fetch revision from',
272 @CommandArgument(
273 "--comm-head-ref",
274 type=text_type,
275 required=False,
276 help="comm-* Reference (this is same as rev usually for hg)",
278 @CommandArgument(
279 "--comm-head-rev",
280 type=text_type,
281 required=False,
282 help="Commit revision to use from head comm-* repository",
284 @CommandArgument(
285 "--project",
286 type=text_type,
287 required=True,
288 help="Project to use for creating task graph. Example: --project=try",
290 @CommandArgument(
291 "--pushlog-id", type=text_type, dest="pushlog_id", required=True, default="0"
293 @CommandArgument("--pushdate", dest="pushdate", required=True, type=int, default=0)
294 @CommandArgument(
295 "--owner",
296 type=text_type,
297 required=True,
298 help="email address of who owns this graph",
300 @CommandArgument(
301 "--level", type=text_type, required=True, help="SCM level of this repository"
303 @CommandArgument(
304 "--target-tasks-method",
305 type=text_type,
306 help="method for selecting the target tasks to generate",
308 @CommandArgument(
309 "--optimize-target-tasks",
310 type=lambda flag: strtobool(flag),
311 nargs="?",
312 const="true",
313 help="If specified, this indicates whether the target "
314 "tasks are eligible for optimization. Otherwise, "
315 "the default for the project is used.",
317 @CommandArgument(
318 "--try-task-config-file",
319 type=text_type,
320 help="path to try task configuration file",
322 @CommandArgument(
323 "--tasks-for",
324 type=text_type,
325 required=True,
326 help="the tasks_for value used to generate this task",
328 @CommandArgument(
329 "--include-push-tasks",
330 action="store_true",
331 help="Whether tasks from the on-push graph should be re-used "
332 "in this graph. This allows cron graphs to avoid rebuilding "
333 "jobs that were built on-push.",
335 @CommandArgument(
336 "--rebuild-kind",
337 dest="rebuild_kinds",
338 action="append",
339 default=argparse.SUPPRESS,
340 help="Kinds that should not be re-used from the on-push graph.",
342 def taskgraph_decision(self, **options):
343 """Run the decision task: generate a task graph and submit to
344 TaskCluster. This is only meant to be called within decision tasks,
345 and requires a great many arguments. Commands like `mach taskgraph
346 optimized` are better suited to use on the command line, and can take
347 the parameters file generated by a decision task."""
349 import taskgraph.decision
351 try:
352 self.setup_logging()
353 start = time.monotonic()
354 ret = taskgraph.decision.taskgraph_decision(options)
355 end = time.monotonic()
356 if os.environ.get("MOZ_AUTOMATION") == "1":
357 perfherder_data = {
358 "framework": {"name": "build_metrics"},
359 "suites": [
361 "name": "decision",
362 "value": end - start,
363 "lowerIsBetter": True,
364 "shouldAlert": True,
365 "subtests": [],
369 print(
370 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
371 file=sys.stderr,
373 return ret
374 except Exception:
375 traceback.print_exc()
376 sys.exit(1)
378 @SubCommand(
379 "taskgraph",
380 "cron",
381 description="Provide a pointer to the new `.cron.yml` handler.",
383 def taskgraph_cron(self, **options):
384 print(
385 'Handling of ".cron.yml" files has move to '
386 "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
388 sys.exit(1)
390 @SubCommand(
391 "taskgraph",
392 "action-callback",
393 description="Run action callback used by action tasks",
395 @CommandArgument(
396 "--root",
397 "-r",
398 default="taskcluster/ci",
399 help="root of the taskgraph definition relative to topsrcdir",
401 def action_callback(self, **options):
402 from taskgraph.actions import trigger_action_callback
403 from taskgraph.actions.util import get_parameters
405 try:
406 self.setup_logging()
408 # the target task for this action (or null if it's a group action)
409 task_id = json.loads(os.environ.get("ACTION_TASK_ID", "null"))
410 # the target task group for this action
411 task_group_id = os.environ.get("ACTION_TASK_GROUP_ID", None)
412 input = json.loads(os.environ.get("ACTION_INPUT", "null"))
413 callback = os.environ.get("ACTION_CALLBACK", None)
414 root = options["root"]
416 parameters = get_parameters(task_group_id)
418 return trigger_action_callback(
419 task_group_id=task_group_id,
420 task_id=task_id,
421 input=input,
422 callback=callback,
423 parameters=parameters,
424 root=root,
425 test=False,
427 except Exception:
428 traceback.print_exc()
429 sys.exit(1)
431 @SubCommand(
432 "taskgraph",
433 "test-action-callback",
434 description="Run an action callback in a testing mode",
436 @CommandArgument(
437 "--root",
438 "-r",
439 default="taskcluster/ci",
440 help="root of the taskgraph definition relative to topsrcdir",
442 @CommandArgument(
443 "--parameters",
444 "-p",
445 default="project=mozilla-central",
446 help="parameters file (.yml or .json; see "
447 "`taskcluster/docs/parameters.rst`)`",
449 @CommandArgument(
450 "--task-id", default=None, help="TaskId to which the action applies"
452 @CommandArgument(
453 "--task-group-id", default=None, help="TaskGroupId to which the action applies"
455 @CommandArgument("--input", default=None, help="Action input (.yml or .json)")
456 @CommandArgument(
457 "callback", default=None, help="Action callback name (Python function name)"
459 def test_action_callback(self, **options):
460 import taskgraph.actions
461 import taskgraph.parameters
462 from taskgraph.util import yaml
464 def load_data(filename):
465 with open(filename) as f:
466 if filename.endswith(".yml"):
467 return yaml.load_stream(f)
468 elif filename.endswith(".json"):
469 return json.load(f)
470 else:
471 raise Exception("unknown filename {}".format(filename))
473 try:
474 self.setup_logging()
475 task_id = options["task_id"]
477 if options["input"]:
478 input = load_data(options["input"])
479 else:
480 input = None
482 parameters = taskgraph.parameters.load_parameters_file(
483 options["parameters"],
484 strict=False,
485 # FIXME: There should be a way to parameterize this.
486 trust_domain="gecko",
488 parameters.check()
490 root = options["root"]
492 return taskgraph.actions.trigger_action_callback(
493 task_group_id=options["task_group_id"],
494 task_id=task_id,
495 input=input,
496 callback=options["callback"],
497 parameters=parameters,
498 root=root,
499 test=True,
501 except Exception:
502 traceback.print_exc()
503 sys.exit(1)
505 def setup_logging(self, quiet=False, verbose=True):
507 Set up Python logging for all loggers, sending results to stderr (so
508 that command output can be redirected easily) and adding the typical
509 mach timestamp.
511 # remove the old terminal handler
512 old = self.log_manager.replace_terminal_handler(None)
514 # re-add it, with level and fh set appropriately
515 if not quiet:
516 level = logging.DEBUG if verbose else logging.INFO
517 self.log_manager.add_terminal_logging(
518 fh=sys.stderr,
519 level=level,
520 write_interval=old.formatter.write_interval,
521 write_times=old.formatter.write_times,
524 # all of the taskgraph logging is unstructured logging
525 self.log_manager.enable_unstructured()
527 def show_taskgraph(self, graph_attr, options):
528 self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
530 base_out = ""
531 base_ref = None
532 cur_ref = None
533 if options["diff"]:
534 from mozversioncontrol import get_repository_object
536 vcs = get_repository_object(self.topsrcdir)
537 with vcs:
538 if not vcs.working_directory_clean():
539 print("abort: can't diff taskgraph with dirty working directory")
540 return 1
542 # We want to return the working directory to the current state
543 # as best we can after we're done. In all known cases, using
544 # branch or bookmark (which are both available on the VCS object)
545 # as `branch` is preferable to a specific revision.
546 cur_ref = vcs.branch or vcs.head_ref[:12]
548 if options["diff"] == "default":
549 base_ref = vcs.base_ref
550 else:
551 base_ref = options["diff"]
553 try:
554 vcs.update(base_ref)
555 base_ref = vcs.head_ref[:12]
556 logger.info("Generating {} @ {}".format(graph_attr, base_ref))
557 base_out = self.format_taskgraph(graph_attr, options)
558 finally:
559 vcs.update(cur_ref)
560 logger.info("Generating {} @ {}".format(graph_attr, cur_ref))
562 # Some transforms use global state for checks, so will fail when
563 # running taskgraph a second time in the same session. Reload all
564 # taskgraph modules to avoid this.
565 for mod in sys.modules.copy():
566 if mod.startswith("taskgraph"):
567 del sys.modules[mod]
569 out = self.format_taskgraph(graph_attr, options)
571 if options["diff"]:
572 diffcmd = self._mach_context.settings["taskgraph"]["diffcmd"]
573 diffcmd = diffcmd.format(attr=graph_attr, base=base_ref, cur=cur_ref)
575 with tempfile.NamedTemporaryFile(mode="w") as base:
576 base.write(base_out)
578 with tempfile.NamedTemporaryFile(mode="w") as cur:
579 cur.write(out)
580 out = subprocess.run(
581 shlex.split(diffcmd)
583 base.name,
584 cur.name,
586 capture_output=True,
587 universal_newlines=True,
588 ).stdout
590 fh = options["output_file"]
591 if fh:
592 fh = open(fh, "w")
593 print(out, file=fh)
595 def format_taskgraph(self, graph_attr, options):
596 import taskgraph
597 import taskgraph.generator
598 import taskgraph.parameters
600 if options["fast"]:
601 taskgraph.fast = True
603 try:
604 parameters = taskgraph.parameters.parameters_loader(
605 options["parameters"],
606 overrides={"target-kind": options.get("target_kind")},
607 strict=False,
610 tgg = taskgraph.generator.TaskGraphGenerator(
611 root_dir=options.get("root"),
612 parameters=parameters,
615 tg = getattr(tgg, graph_attr)
616 tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
618 format_method = getattr(
619 self, "format_taskgraph_" + (options["format"] or "labels")
621 return format_method(tg)
622 except Exception:
623 traceback.print_exc()
624 sys.exit(1)
626 def format_taskgraph_labels(self, taskgraph):
627 return "\n".join(
628 taskgraph.tasks[index].label for index in taskgraph.graph.visit_postorder()
631 def format_taskgraph_json(self, taskgraph):
632 return json.dumps(
633 taskgraph.to_json(), sort_keys=True, indent=2, separators=(",", ": ")
636 def get_filtered_taskgraph(self, taskgraph, tasksregex):
637 from taskgraph.graph import Graph
638 from taskgraph.taskgraph import TaskGraph
641 This class method filters all the tasks on basis of a regular expression
642 and returns a new TaskGraph object
644 # return original taskgraph if no regular expression is passed
645 if not tasksregex:
646 return taskgraph
647 named_links_dict = taskgraph.graph.named_links_dict()
648 filteredtasks = {}
649 filterededges = set()
650 regexprogram = re.compile(tasksregex)
652 for key in taskgraph.graph.visit_postorder():
653 task = taskgraph.tasks[key]
654 if regexprogram.match(task.label):
655 filteredtasks[key] = task
656 for depname, dep in six.iteritems(named_links_dict[key]):
657 if regexprogram.match(dep):
658 filterededges.add((key, dep, depname))
659 filtered_taskgraph = TaskGraph(
660 filteredtasks, Graph(set(filteredtasks), filterededges)
662 return filtered_taskgraph
664 def show_actions(self, options):
665 import taskgraph
666 import taskgraph.actions
667 import taskgraph.generator
668 import taskgraph.parameters
670 try:
671 self.setup_logging(quiet=options["quiet"], verbose=options["verbose"])
672 parameters = taskgraph.parameters.parameters_loader(options["parameters"])
674 tgg = taskgraph.generator.TaskGraphGenerator(
675 root_dir=options.get("root"),
676 parameters=parameters,
679 actions = taskgraph.actions.render_actions_json(
680 tgg.parameters,
681 tgg.graph_config,
682 decision_task_id="DECISION-TASK",
684 print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
685 except Exception:
686 traceback.print_exc()
687 sys.exit(1)
690 @CommandProvider
691 class TaskClusterImagesProvider(MachCommandBase):
692 @Command(
693 "taskcluster-load-image",
694 category="ci",
695 description="Load a pre-built Docker image. Note that you need to "
696 "have docker installed and running for this to work.",
698 @CommandArgument(
699 "--task-id",
700 help="Load the image at public/image.tar.zst in this task, "
701 "rather than searching the index",
703 @CommandArgument(
704 "-t",
705 "--tag",
706 help="tag that the image should be loaded as. If not "
707 "image will be loaded with tag from the tarball",
708 metavar="name:tag",
710 @CommandArgument(
711 "image_name",
712 nargs="?",
713 help="Load the image of this name based on the current "
714 "contents of the tree (as built for mozilla-central "
715 "or mozilla-inbound)",
717 def load_image(self, image_name, task_id, tag):
718 from taskgraph.docker import load_image_by_name, load_image_by_task_id
720 if not image_name and not task_id:
721 print("Specify either IMAGE-NAME or TASK-ID")
722 sys.exit(1)
723 try:
724 if task_id:
725 ok = load_image_by_task_id(task_id, tag)
726 else:
727 ok = load_image_by_name(image_name, tag)
728 if not ok:
729 sys.exit(1)
730 except Exception:
731 traceback.print_exc()
732 sys.exit(1)
734 @Command(
735 "taskcluster-build-image", category="ci", description="Build a Docker image"
737 @CommandArgument("image_name", help="Name of the image to build")
738 @CommandArgument(
739 "-t", "--tag", help="tag that the image should be built as.", metavar="name:tag"
741 @CommandArgument(
742 "--context-only",
743 help="File name the context tarball should be written to."
744 "with this option it will only build the context.tar.",
745 metavar="context.tar",
747 def build_image(self, image_name, tag, context_only):
748 from taskgraph.docker import build_context, build_image
750 try:
751 if context_only is None:
752 build_image(image_name, tag, os.environ)
753 else:
754 build_context(image_name, context_only, os.environ)
755 except Exception:
756 traceback.print_exc()
757 sys.exit(1)
760 @CommandProvider
761 class TaskClusterPartialsData(MachCommandBase):
762 @Command(
763 "release-history",
764 category="ci",
765 description="Query balrog for release history used by enable partials generation",
767 @CommandArgument(
768 "-b",
769 "--branch",
770 help="The gecko project branch used in balrog, such as "
771 "mozilla-central, release, maple",
773 @CommandArgument(
774 "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
776 def generate_partials_builds(self, product, branch):
777 from taskgraph.util.partials import populate_release_history
779 try:
780 import yaml
782 release_history = {
783 "release_history": populate_release_history(product, branch)
785 print(
786 yaml.safe_dump(
787 release_history, allow_unicode=True, default_flow_style=False
790 except Exception:
791 traceback.print_exc()
792 sys.exit(1)