1 # -*- coding: utf-8 -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 from __future__
import absolute_import
, print_function
, unicode_literals
16 from functools
import partial
18 from mach
.decorators
import (
26 from taskgraph
.main
import commands
as taskgraph_commands
28 logger
= logging
.getLogger("taskcluster")
32 class TaskgraphConfig(object):
34 def config_settings(cls
):
39 "The command to run with `./mach taskgraph --diff`",
40 "diff --report-identical-files "
41 "--label={attr}@{base} --label={attr}@{cur} -U20",
48 """Convert string to boolean.
50 Wraps "distutils.util.strtobool", deferring the import of the package
51 in case it's not installed. Otherwise, we have a "chicken and egg problem" where
52 |mach bootstrap| would install the required package to enable "distutils.util", but
53 it can't because mach fails to interpret this file.
55 from distutils
.util
import strtobool
57 return bool(strtobool(value
))
60 def get_taskgraph_command_parser(name
):
61 """Given a command name, obtain its argument parser.
64 name (str): Name of the command.
67 ArgumentParser: An ArgumentParser instance.
69 command
= taskgraph_commands
[name
]
70 parser
= argparse
.ArgumentParser()
71 for arg
in command
.func
.args
:
72 parser
.add_argument(*arg
[0], **arg
[1])
74 parser
.set_defaults(func
=command
.func
, **command
.defaults
)
78 def get_taskgraph_decision_parser():
79 parser
= get_taskgraph_command_parser("decision")
83 ["--optimize-target-tasks"],
85 "type": lambda flag
: strtobool(flag
),
88 "help": "If specified, this indicates whether the target "
89 "tasks are eligible for optimization. Otherwise, the default "
90 "for the project is used.",
94 ["--include-push-tasks"],
96 "action": "store_true",
97 "help": "Whether tasks from the on-push graph should be re-used "
98 "in this graph. This allows cron graphs to avoid rebuilding "
99 "jobs that were built on-push.",
105 "dest": "rebuild_kinds",
107 "default": argparse
.SUPPRESS
,
108 "help": "Kinds that should not be re-used from the on-push graph.",
112 ["--comm-base-repository"],
115 "help": "URL for 'base' comm-* repository to clone",
119 ["--comm-head-repository"],
122 "help": "URL for 'head' comm-* repository to fetch revision from",
129 "help": "comm-* Reference (this is same as rev usually for hg)",
136 "help": "Commit revision to use from head comm-* repository",
140 for arg
in extra_args
:
141 parser
.add_argument(*arg
[0], **arg
[1])
149 description
="Manipulate TaskCluster task graphs defined in-tree",
151 def taskgraph_command(command_context
):
152 """The taskgraph subcommands all relate to the generation of task graphs
153 for Gecko continuous integration. A task graph is a set of tasks linked
154 by dependencies: for example, a binary must be built before it is tested,
155 and that build may further depend on various toolchains, libraries, etc.
162 description
="Show all tasks in the taskgraph",
163 parser
=partial(get_taskgraph_command_parser
, "tasks"),
165 def taskgraph_tasks(command_context
, **options
):
166 return run_show_taskgraph(command_context
, **options
)
172 description
="Show the full taskgraph",
173 parser
=partial(get_taskgraph_command_parser
, "full"),
175 def taskgraph_full(command_context
, **options
):
176 return run_show_taskgraph(command_context
, **options
)
182 description
="Show the target task set",
183 parser
=partial(get_taskgraph_command_parser
, "target"),
185 def taskgraph_target(command_context
, **options
):
186 return run_show_taskgraph(command_context
, **options
)
192 description
="Show the target taskgraph",
193 parser
=partial(get_taskgraph_command_parser
, "target-graph"),
195 def taskgraph_target_graph(command_context
, **options
):
196 return run_show_taskgraph(command_context
, **options
)
202 description
="Show the optimized taskgraph",
203 parser
=partial(get_taskgraph_command_parser
, "optimized"),
205 def taskgraph_optimized(command_context
, **options
):
206 return run_show_taskgraph(command_context
, **options
)
212 description
="Show the morphed taskgraph",
213 parser
=partial(get_taskgraph_command_parser
, "morphed"),
215 def taskgraph_morphed(command_context
, **options
):
216 return run_show_taskgraph(command_context
, **options
)
219 def run_show_taskgraph(command_context
, **options
):
220 # There are cases where we don't want to set up mach logging (e.g logs
221 # are being redirected to disk). By monkeypatching the 'setup_logging'
222 # function we can let 'taskgraph.main' decide whether or not to log to
224 taskgraph
.main
.setup_logging
= partial(
227 quiet
=options
["quiet"],
228 verbose
=options
["verbose"],
230 show_taskgraph
= options
.pop("func")
231 return show_taskgraph(options
)
234 @SubCommand("taskgraph", "actions", description
="Write actions.json to stdout")
236 "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
239 "--quiet", "-q", action
="store_true", help="suppress all logging output"
245 help="include debug-level logging output",
250 default
="project=mozilla-central",
251 help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
253 def taskgraph_actions(command_context
, **options
):
254 return show_actions(command_context
, options
)
260 description
="Run the decision task",
261 parser
=get_taskgraph_decision_parser
,
263 def taskgraph_decision(command_context
, **options
):
264 """Run the decision task: generate a task graph and submit to
265 TaskCluster. This is only meant to be called within decision tasks,
266 and requires a great many arguments. Commands like `mach taskgraph
267 optimized` are better suited to use on the command line, and can take
268 the parameters file generated by a decision task."""
270 setup_logging(command_context
)
271 start
= time
.monotonic()
272 ret
= taskgraph_commands
["decision"].func(options
)
273 end
= time
.monotonic()
274 if os
.environ
.get("MOZ_AUTOMATION") == "1":
276 "framework": {"name": "build_metrics"},
280 "value": end
- start
,
281 "lowerIsBetter": True,
288 "PERFHERDER_DATA: {}".format(json
.dumps(perfherder_data
)),
293 traceback
.print_exc()
300 description
="Provide a pointer to the new `.cron.yml` handler.",
302 def taskgraph_cron(command_context
, **options
):
304 'Handling of ".cron.yml" files has move to '
305 "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
313 description
="Run action callback used by action tasks",
314 parser
=partial(get_taskgraph_command_parser
, "action-callback"),
316 def action_callback(command_context
, **options
):
317 setup_logging(command_context
)
318 taskgraph_commands
["action-callback"].func(options
)
323 "test-action-callback",
324 description
="Run an action callback in a testing mode",
325 parser
=partial(get_taskgraph_command_parser
, "test-action-callback"),
327 def test_action_callback(command_context
, **options
):
328 setup_logging(command_context
)
330 if not options
["parameters"]:
331 options
["parameters"] = "project=mozilla-central"
333 taskgraph_commands
["test-action-callback"].func(options
)
336 def setup_logging(command_context
, quiet
=False, verbose
=True):
338 Set up Python logging for all loggers, sending results to stderr (so
339 that command output can be redirected easily) and adding the typical
342 # remove the old terminal handler
343 old
= command_context
.log_manager
.replace_terminal_handler(None)
345 # re-add it, with level and fh set appropriately
347 level
= logging
.DEBUG
if verbose
else logging
.INFO
348 command_context
.log_manager
.add_terminal_logging(
351 write_interval
=old
.formatter
.write_interval
,
352 write_times
=old
.formatter
.write_times
,
355 # all of the taskgraph logging is unstructured logging
356 command_context
.log_manager
.enable_unstructured()
359 def show_actions(command_context
, options
):
361 import taskgraph
.actions
362 import taskgraph
.generator
363 import taskgraph
.parameters
367 command_context
, quiet
=options
["quiet"], verbose
=options
["verbose"]
369 parameters
= taskgraph
.parameters
.parameters_loader(options
["parameters"])
371 tgg
= taskgraph
.generator
.TaskGraphGenerator(
372 root_dir
=options
.get("root"),
373 parameters
=parameters
,
376 actions
= taskgraph
.actions
.render_actions_json(
379 decision_task_id
="DECISION-TASK",
381 print(json
.dumps(actions
, sort_keys
=True, indent
=2, separators
=(",", ": ")))
383 traceback
.print_exc()
388 "taskcluster-load-image",
390 description
="Load a pre-built Docker image. Note that you need to "
391 "have docker installed and running for this to work.",
392 parser
=partial(get_taskgraph_command_parser
, "load-image"),
394 def load_image(command_context
, **kwargs
):
395 taskgraph_commands
["load-image"].func(kwargs
)
399 "taskcluster-build-image",
401 description
="Build a Docker image",
402 parser
=partial(get_taskgraph_command_parser
, "build-image"),
404 def build_image(command_context
, **kwargs
):
406 taskgraph_commands
["build-image"].func(kwargs
)
408 traceback
.print_exc()
413 "taskcluster-image-digest",
415 description
="Print the digest of the image of this name based on the "
416 "current contents of the tree.",
417 parser
=partial(get_taskgraph_command_parser
, "build-image"),
419 def image_digest(command_context
, **kwargs
):
420 taskgraph_commands
["image-digest"].func(kwargs
)
426 description
="Query balrog for release history used by enable partials generation",
431 help="The gecko project branch used in balrog, such as "
432 "mozilla-central, release, maple",
435 "--product", default
="Firefox", help="The product identifier, such as 'Firefox'"
437 def generate_partials_builds(command_context
, product
, branch
):
438 from taskgraph
.util
.partials
import populate_release_history
443 release_history
= {"release_history": populate_release_history(product
, branch
)}
446 release_history
, allow_unicode
=True, default_flow_style
=False
450 traceback
.print_exc()