1 # -*- coding: utf-8 -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 from __future__
import absolute_import
, print_function
, unicode_literals
22 from six
import text_type
24 from mach
.decorators
import (
31 from mozbuild
.base
import MachCommandBase
33 logger
= logging
.getLogger("taskcluster")
37 class TaskgraphConfig(object):
39 def config_settings(cls
):
44 "The command to run with `./mach taskgraph --diff`",
45 "diff --report-identical-files --color=always "
46 "--label={attr}@{base} --label={attr}@{cur} -U20",
53 """Convert string to boolean.
55 Wraps "distutils.util.strtobool", deferring the import of the package
56 in case it's not installed. Otherwise, we have a "chicken and egg problem" where
57 |mach bootstrap| would install the required package to enable "distutils.util", but
58 it can't because mach fails to interpret this file.
60 from distutils
.util
import strtobool
62 return bool(strtobool(value
))
65 class ShowTaskGraphSubCommand(SubCommand
):
66 """A SubCommand with TaskGraph-specific arguments"""
68 def __call__(self
, func
):
69 after
= SubCommand
.__call
__(self
, func
)
74 help="root of the taskgraph definition relative to topsrcdir",
77 "--quiet", "-q", action
="store_true", help="suppress all logging output"
83 help="include debug-level logging output",
91 help="Output task graph as a JSON object",
99 help="Output the label for each task in the task graph (default)",
104 default
="project=mozilla-central",
105 help="parameters file (.yml or .json; see "
106 "`taskcluster/docs/parameters.rst`)`",
111 action
="store_false",
113 help="do not remove tasks from the graph that are found in the "
114 "index (a.k.a. optimize the graph)",
120 help="only return tasks with labels matching this regular "
126 help="only return tasks that are of the given kind, "
127 "or their dependencies.",
135 help="enable fast task generation for local debugging.",
141 help="file path to store generated output.",
148 help="Generate and diff the current taskgraph against another revision. "
149 "Without args the base revision will be used. A revision specifier such as "
150 "the hash or `.~1` (hg) or `HEAD~1` (git) can be used as well.",
159 class MachCommands(MachCommandBase
):
163 description
="Manipulate TaskCluster task graphs defined in-tree",
166 """The taskgraph subcommands all relate to the generation of task graphs
167 for Gecko continuous integration. A task graph is a set of tasks linked
168 by dependencies: for example, a binary must be built before it is tested,
169 and that build may further depend on various toolchains, libraries, etc.
172 @ShowTaskGraphSubCommand(
173 "taskgraph", "tasks", description
="Show all tasks in the taskgraph"
175 def taskgraph_tasks(self
, **options
):
176 return self
.show_taskgraph("full_task_set", options
)
178 @ShowTaskGraphSubCommand("taskgraph", "full", description
="Show the full taskgraph")
179 def taskgraph_full(self
, **options
):
180 return self
.show_taskgraph("full_task_graph", options
)
182 @ShowTaskGraphSubCommand(
183 "taskgraph", "target", description
="Show the target task set"
185 def taskgraph_target(self
, **options
):
186 return self
.show_taskgraph("target_task_set", options
)
188 @ShowTaskGraphSubCommand(
189 "taskgraph", "target-graph", description
="Show the target taskgraph"
191 def taskgraph_target_taskgraph(self
, **options
):
192 return self
.show_taskgraph("target_task_graph", options
)
194 @ShowTaskGraphSubCommand(
195 "taskgraph", "optimized", description
="Show the optimized taskgraph"
197 def taskgraph_optimized(self
, **options
):
198 return self
.show_taskgraph("optimized_task_graph", options
)
200 @ShowTaskGraphSubCommand(
201 "taskgraph", "morphed", description
="Show the morphed taskgraph"
203 def taskgraph_morphed(self
, **options
):
204 return self
.show_taskgraph("morphed_task_graph", options
)
206 @SubCommand("taskgraph", "actions", description
="Write actions.json to stdout")
208 "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
211 "--quiet", "-q", action
="store_true", help="suppress all logging output"
217 help="include debug-level logging output",
222 default
="project=mozilla-central",
223 help="parameters file (.yml or .json; see "
224 "`taskcluster/docs/parameters.rst`)`",
226 def taskgraph_actions(self
, **options
):
227 return self
.show_actions(options
)
229 @SubCommand("taskgraph", "decision", description
="Run the decision task")
234 help="root of the taskgraph definition relative to topsrcdir",
240 help='URL for "base" repository to clone',
246 help='URL for "head" repository to fetch revision from',
252 help="Reference (this is same as rev usually for hg)",
258 help="Commit revision to use from head repository",
261 "--comm-base-repository",
264 help='URL for "base" comm-* repository to clone',
267 "--comm-head-repository",
270 help='URL for "head" comm-* repository to fetch revision from',
276 help="comm-* Reference (this is same as rev usually for hg)",
282 help="Commit revision to use from head comm-* repository",
288 help="Project to use for creating task graph. Example: --project=try",
291 "--pushlog-id", type=text_type
, dest
="pushlog_id", required
=True, default
="0"
293 @CommandArgument("--pushdate", dest
="pushdate", required
=True, type=int, default
=0)
298 help="email address of who owns this graph",
301 "--level", type=text_type
, required
=True, help="SCM level of this repository"
304 "--target-tasks-method",
306 help="method for selecting the target tasks to generate",
309 "--optimize-target-tasks",
310 type=lambda flag
: strtobool(flag
),
313 help="If specified, this indicates whether the target "
314 "tasks are eligible for optimization. Otherwise, "
315 "the default for the project is used.",
318 "--try-task-config-file",
320 help="path to try task configuration file",
326 help="the tasks_for value used to generate this task",
329 "--include-push-tasks",
331 help="Whether tasks from the on-push graph should be re-used "
332 "in this graph. This allows cron graphs to avoid rebuilding "
333 "jobs that were built on-push.",
337 dest
="rebuild_kinds",
339 default
=argparse
.SUPPRESS
,
340 help="Kinds that should not be re-used from the on-push graph.",
342 def taskgraph_decision(self
, **options
):
343 """Run the decision task: generate a task graph and submit to
344 TaskCluster. This is only meant to be called within decision tasks,
345 and requires a great many arguments. Commands like `mach taskgraph
346 optimized` are better suited to use on the command line, and can take
347 the parameters file generated by a decision task."""
349 import taskgraph
.decision
353 start
= time
.monotonic()
354 ret
= taskgraph
.decision
.taskgraph_decision(options
)
355 end
= time
.monotonic()
356 if os
.environ
.get("MOZ_AUTOMATION") == "1":
358 "framework": {"name": "build_metrics"},
362 "value": end
- start
,
363 "lowerIsBetter": True,
370 "PERFHERDER_DATA: {}".format(json
.dumps(perfherder_data
)),
375 traceback
.print_exc()
381 description
="Provide a pointer to the new `.cron.yml` handler.",
383 def taskgraph_cron(self
, **options
):
385 'Handling of ".cron.yml" files has move to '
386 "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
393 description
="Run action callback used by action tasks",
398 default
="taskcluster/ci",
399 help="root of the taskgraph definition relative to topsrcdir",
401 def action_callback(self
, **options
):
402 from taskgraph
.actions
import trigger_action_callback
403 from taskgraph
.actions
.util
import get_parameters
408 # the target task for this action (or null if it's a group action)
409 task_id
= json
.loads(os
.environ
.get("ACTION_TASK_ID", "null"))
410 # the target task group for this action
411 task_group_id
= os
.environ
.get("ACTION_TASK_GROUP_ID", None)
412 input = json
.loads(os
.environ
.get("ACTION_INPUT", "null"))
413 callback
= os
.environ
.get("ACTION_CALLBACK", None)
414 root
= options
["root"]
416 parameters
= get_parameters(task_group_id
)
418 return trigger_action_callback(
419 task_group_id
=task_group_id
,
423 parameters
=parameters
,
428 traceback
.print_exc()
433 "test-action-callback",
434 description
="Run an action callback in a testing mode",
439 default
="taskcluster/ci",
440 help="root of the taskgraph definition relative to topsrcdir",
445 default
="project=mozilla-central",
446 help="parameters file (.yml or .json; see "
447 "`taskcluster/docs/parameters.rst`)`",
450 "--task-id", default
=None, help="TaskId to which the action applies"
453 "--task-group-id", default
=None, help="TaskGroupId to which the action applies"
455 @CommandArgument("--input", default
=None, help="Action input (.yml or .json)")
457 "callback", default
=None, help="Action callback name (Python function name)"
459 def test_action_callback(self
, **options
):
460 import taskgraph
.actions
461 import taskgraph
.parameters
462 from taskgraph
.util
import yaml
464 def load_data(filename
):
465 with
open(filename
) as f
:
466 if filename
.endswith(".yml"):
467 return yaml
.load_stream(f
)
468 elif filename
.endswith(".json"):
471 raise Exception("unknown filename {}".format(filename
))
475 task_id
= options
["task_id"]
478 input = load_data(options
["input"])
482 parameters
= taskgraph
.parameters
.load_parameters_file(
483 options
["parameters"],
485 # FIXME: There should be a way to parameterize this.
486 trust_domain
="gecko",
490 root
= options
["root"]
492 return taskgraph
.actions
.trigger_action_callback(
493 task_group_id
=options
["task_group_id"],
496 callback
=options
["callback"],
497 parameters
=parameters
,
502 traceback
.print_exc()
505 def setup_logging(self
, quiet
=False, verbose
=True):
507 Set up Python logging for all loggers, sending results to stderr (so
508 that command output can be redirected easily) and adding the typical
511 # remove the old terminal handler
512 old
= self
.log_manager
.replace_terminal_handler(None)
514 # re-add it, with level and fh set appropriately
516 level
= logging
.DEBUG
if verbose
else logging
.INFO
517 self
.log_manager
.add_terminal_logging(
520 write_interval
=old
.formatter
.write_interval
,
521 write_times
=old
.formatter
.write_times
,
524 # all of the taskgraph logging is unstructured logging
525 self
.log_manager
.enable_unstructured()
527 def show_taskgraph(self
, graph_attr
, options
):
528 self
.setup_logging(quiet
=options
["quiet"], verbose
=options
["verbose"])
534 from mozversioncontrol
import get_repository_object
536 vcs
= get_repository_object(self
.topsrcdir
)
538 if not vcs
.working_directory_clean():
539 print("abort: can't diff taskgraph with dirty working directory")
542 # We want to return the working directory to the current state
543 # as best we can after we're done. In all known cases, using
544 # branch or bookmark (which are both available on the VCS object)
545 # as `branch` is preferable to a specific revision.
546 cur_ref
= vcs
.branch
or vcs
.head_ref
[:12]
548 if options
["diff"] == "default":
549 base_ref
= vcs
.base_ref
551 base_ref
= options
["diff"]
555 base_ref
= vcs
.head_ref
[:12]
556 logger
.info("Generating {} @ {}".format(graph_attr
, base_ref
))
557 base_out
= self
.format_taskgraph(graph_attr
, options
)
560 logger
.info("Generating {} @ {}".format(graph_attr
, cur_ref
))
562 # Some transforms use global state for checks, so will fail when
563 # running taskgraph a second time in the same session. Reload all
564 # taskgraph modules to avoid this.
565 for mod
in sys
.modules
.copy():
566 if mod
.startswith("taskgraph"):
569 out
= self
.format_taskgraph(graph_attr
, options
)
572 diffcmd
= self
._mach
_context
.settings
["taskgraph"]["diffcmd"]
573 diffcmd
= diffcmd
.format(attr
=graph_attr
, base
=base_ref
, cur
=cur_ref
)
575 with tempfile
.NamedTemporaryFile(mode
="w") as base
:
578 with tempfile
.NamedTemporaryFile(mode
="w") as cur
:
580 out
= subprocess
.run(
587 universal_newlines
=True,
590 fh
= options
["output_file"]
595 def format_taskgraph(self
, graph_attr
, options
):
597 import taskgraph
.generator
598 import taskgraph
.parameters
601 taskgraph
.fast
= True
604 parameters
= taskgraph
.parameters
.parameters_loader(
605 options
["parameters"],
606 overrides
={"target-kind": options
.get("target_kind")},
610 tgg
= taskgraph
.generator
.TaskGraphGenerator(
611 root_dir
=options
.get("root"),
612 parameters
=parameters
,
615 tg
= getattr(tgg
, graph_attr
)
616 tg
= self
.get_filtered_taskgraph(tg
, options
["tasks_regex"])
618 format_method
= getattr(
619 self
, "format_taskgraph_" + (options
["format"] or "labels")
621 return format_method(tg
)
623 traceback
.print_exc()
626 def format_taskgraph_labels(self
, taskgraph
):
628 taskgraph
.tasks
[index
].label
for index
in taskgraph
.graph
.visit_postorder()
631 def format_taskgraph_json(self
, taskgraph
):
633 taskgraph
.to_json(), sort_keys
=True, indent
=2, separators
=(",", ": ")
636 def get_filtered_taskgraph(self
, taskgraph
, tasksregex
):
637 from taskgraph
.graph
import Graph
638 from taskgraph
.taskgraph
import TaskGraph
641 This class method filters all the tasks on basis of a regular expression
642 and returns a new TaskGraph object
644 # return original taskgraph if no regular expression is passed
647 named_links_dict
= taskgraph
.graph
.named_links_dict()
649 filterededges
= set()
650 regexprogram
= re
.compile(tasksregex
)
652 for key
in taskgraph
.graph
.visit_postorder():
653 task
= taskgraph
.tasks
[key
]
654 if regexprogram
.match(task
.label
):
655 filteredtasks
[key
] = task
656 for depname
, dep
in six
.iteritems(named_links_dict
[key
]):
657 if regexprogram
.match(dep
):
658 filterededges
.add((key
, dep
, depname
))
659 filtered_taskgraph
= TaskGraph(
660 filteredtasks
, Graph(set(filteredtasks
), filterededges
)
662 return filtered_taskgraph
664 def show_actions(self
, options
):
666 import taskgraph
.actions
667 import taskgraph
.generator
668 import taskgraph
.parameters
671 self
.setup_logging(quiet
=options
["quiet"], verbose
=options
["verbose"])
672 parameters
= taskgraph
.parameters
.parameters_loader(options
["parameters"])
674 tgg
= taskgraph
.generator
.TaskGraphGenerator(
675 root_dir
=options
.get("root"),
676 parameters
=parameters
,
679 actions
= taskgraph
.actions
.render_actions_json(
682 decision_task_id
="DECISION-TASK",
684 print(json
.dumps(actions
, sort_keys
=True, indent
=2, separators
=(",", ": ")))
686 traceback
.print_exc()
691 class TaskClusterImagesProvider(MachCommandBase
):
693 "taskcluster-load-image",
695 description
="Load a pre-built Docker image. Note that you need to "
696 "have docker installed and running for this to work.",
700 help="Load the image at public/image.tar.zst in this task, "
701 "rather than searching the index",
706 help="tag that the image should be loaded as. If not "
707 "image will be loaded with tag from the tarball",
713 help="Load the image of this name based on the current "
714 "contents of the tree (as built for mozilla-central "
715 "or mozilla-inbound)",
717 def load_image(self
, image_name
, task_id
, tag
):
718 from taskgraph
.docker
import load_image_by_name
, load_image_by_task_id
720 if not image_name
and not task_id
:
721 print("Specify either IMAGE-NAME or TASK-ID")
725 ok
= load_image_by_task_id(task_id
, tag
)
727 ok
= load_image_by_name(image_name
, tag
)
731 traceback
.print_exc()
735 "taskcluster-build-image", category
="ci", description
="Build a Docker image"
737 @CommandArgument("image_name", help="Name of the image to build")
739 "-t", "--tag", help="tag that the image should be built as.", metavar
="name:tag"
743 help="File name the context tarball should be written to."
744 "with this option it will only build the context.tar.",
745 metavar
="context.tar",
747 def build_image(self
, image_name
, tag
, context_only
):
748 from taskgraph
.docker
import build_context
, build_image
751 if context_only
is None:
752 build_image(image_name
, tag
, os
.environ
)
754 build_context(image_name
, context_only
, os
.environ
)
756 traceback
.print_exc()
761 class TaskClusterPartialsData(MachCommandBase
):
765 description
="Query balrog for release history used by enable partials generation",
770 help="The gecko project branch used in balrog, such as "
771 "mozilla-central, release, maple",
774 "--product", default
="Firefox", help="The product identifier, such as 'Firefox'"
776 def generate_partials_builds(self
, product
, branch
):
777 from taskgraph
.util
.partials
import populate_release_history
783 "release_history": populate_release_history(product
, branch
)
787 release_history
, allow_unicode
=True, default_flow_style
=False
791 traceback
.print_exc()