no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD CLOSED TREE
[gecko.git] / taskcluster / mach_commands.py
blob78099f6eea1317e08fd316c2e1cc35e0fe7fef39
1 # -*- coding: utf-8 -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 import argparse
9 import json
10 import logging
11 import os
12 import sys
13 import time
14 import traceback
15 from functools import partial
17 import gecko_taskgraph.main
18 from gecko_taskgraph.main import commands as taskgraph_commands
19 from mach.decorators import Command, CommandArgument, SubCommand
21 logger = logging.getLogger("taskcluster")
24 def strtobool(value):
25 """Convert string to boolean.
27 Wraps "distutils.util.strtobool", deferring the import of the package
28 in case it's not installed. Otherwise, we have a "chicken and egg problem" where
29 |mach bootstrap| would install the required package to enable "distutils.util", but
30 it can't because mach fails to interpret this file.
31 """
32 from distutils.util import strtobool
34 return bool(strtobool(value))
37 def get_taskgraph_command_parser(name):
38 """Given a command name, obtain its argument parser.
40 Args:
41 name (str): Name of the command.
43 Returns:
44 ArgumentParser: An ArgumentParser instance.
45 """
46 command = taskgraph_commands[name]
47 parser = argparse.ArgumentParser()
48 for arg in command.func.args:
49 parser.add_argument(*arg[0], **arg[1])
51 parser.set_defaults(func=command.func, **command.defaults)
52 return parser
55 def get_taskgraph_decision_parser():
56 parser = get_taskgraph_command_parser("decision")
58 extra_args = [
60 ["--optimize-target-tasks"],
62 "type": lambda flag: strtobool(flag),
63 "nargs": "?",
64 "const": "true",
65 "help": "If specified, this indicates whether the target "
66 "tasks are eligible for optimization. Otherwise, the default "
67 "for the project is used.",
71 ["--include-push-tasks"],
73 "action": "store_true",
74 "help": "Whether tasks from the on-push graph should be re-used "
75 "in this graph. This allows cron graphs to avoid rebuilding "
76 "jobs that were built on-push.",
80 ["--rebuild-kind"],
82 "dest": "rebuild_kinds",
83 "action": "append",
84 "default": argparse.SUPPRESS,
85 "help": "Kinds that should not be re-used from the on-push graph.",
89 for arg in extra_args:
90 parser.add_argument(*arg[0], **arg[1])
92 return parser
95 @Command(
96 "taskgraph",
97 category="ci",
98 description="Manipulate TaskCluster task graphs defined in-tree",
100 def taskgraph_command(command_context):
101 """The taskgraph subcommands all relate to the generation of task graphs
102 for Gecko continuous integration. A task graph is a set of tasks linked
103 by dependencies: for example, a binary must be built before it is tested,
104 and that build may further depend on various toolchains, libraries, etc.
108 @SubCommand(
109 "taskgraph",
110 "tasks",
111 description="Show all tasks in the taskgraph",
112 parser=partial(get_taskgraph_command_parser, "tasks"),
114 def taskgraph_tasks(command_context, **options):
115 return run_show_taskgraph(command_context, **options)
118 @SubCommand(
119 "taskgraph",
120 "full",
121 description="Show the full taskgraph",
122 parser=partial(get_taskgraph_command_parser, "full"),
124 def taskgraph_full(command_context, **options):
125 return run_show_taskgraph(command_context, **options)
128 @SubCommand(
129 "taskgraph",
130 "target",
131 description="Show the target task set",
132 parser=partial(get_taskgraph_command_parser, "target"),
134 def taskgraph_target(command_context, **options):
135 return run_show_taskgraph(command_context, **options)
138 @SubCommand(
139 "taskgraph",
140 "target-graph",
141 description="Show the target taskgraph",
142 parser=partial(get_taskgraph_command_parser, "target-graph"),
144 def taskgraph_target_graph(command_context, **options):
145 return run_show_taskgraph(command_context, **options)
148 @SubCommand(
149 "taskgraph",
150 "optimized",
151 description="Show the optimized taskgraph",
152 parser=partial(get_taskgraph_command_parser, "optimized"),
154 def taskgraph_optimized(command_context, **options):
155 return run_show_taskgraph(command_context, **options)
158 @SubCommand(
159 "taskgraph",
160 "morphed",
161 description="Show the morphed taskgraph",
162 parser=partial(get_taskgraph_command_parser, "morphed"),
164 def taskgraph_morphed(command_context, **options):
165 return run_show_taskgraph(command_context, **options)
168 def run_show_taskgraph(command_context, **options):
169 # There are cases where we don't want to set up mach logging (e.g logs
170 # are being redirected to disk). By monkeypatching the 'setup_logging'
171 # function we can let 'taskgraph.main' decide whether or not to log to
172 # the terminal.
173 gecko_taskgraph.main.setup_logging = partial(
174 setup_logging,
175 command_context,
176 quiet=options["quiet"],
177 verbose=options["verbose"],
179 show_taskgraph = options.pop("func")
180 return show_taskgraph(options)
183 @SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
184 @CommandArgument(
185 "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
187 @CommandArgument(
188 "--quiet", "-q", action="store_true", help="suppress all logging output"
190 @CommandArgument(
191 "--verbose",
192 "-v",
193 action="store_true",
194 help="include debug-level logging output",
196 @CommandArgument(
197 "--parameters",
198 "-p",
199 default="project=mozilla-central",
200 help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
202 def taskgraph_actions(command_context, **options):
203 return show_actions(command_context, options)
206 @SubCommand(
207 "taskgraph",
208 "decision",
209 description="Run the decision task",
210 parser=get_taskgraph_decision_parser,
212 def taskgraph_decision(command_context, **options):
213 """Run the decision task: generate a task graph and submit to
214 TaskCluster. This is only meant to be called within decision tasks,
215 and requires a great many arguments. Commands like `mach taskgraph
216 optimized` are better suited to use on the command line, and can take
217 the parameters file generated by a decision task."""
218 try:
219 setup_logging(command_context)
220 start = time.monotonic()
221 ret = taskgraph_commands["decision"].func(options)
222 end = time.monotonic()
223 if os.environ.get("MOZ_AUTOMATION") == "1":
224 perfherder_data = {
225 "framework": {"name": "build_metrics"},
226 "suites": [
228 "name": "decision",
229 "value": end - start,
230 "lowerIsBetter": True,
231 "shouldAlert": True,
232 "subtests": [],
236 print(
237 "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
238 file=sys.stderr,
240 return ret
241 except Exception:
242 traceback.print_exc()
243 sys.exit(1)
246 @SubCommand(
247 "taskgraph",
248 "cron",
249 description="Provide a pointer to the new `.cron.yml` handler.",
251 def taskgraph_cron(command_context, **options):
252 print(
253 'Handling of ".cron.yml" files has move to '
254 "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
256 sys.exit(1)
259 @SubCommand(
260 "taskgraph",
261 "action-callback",
262 description="Run action callback used by action tasks",
263 parser=partial(get_taskgraph_command_parser, "action-callback"),
265 def action_callback(command_context, **options):
266 setup_logging(command_context)
267 taskgraph_commands["action-callback"].func(options)
270 @SubCommand(
271 "taskgraph",
272 "test-action-callback",
273 description="Run an action callback in a testing mode",
274 parser=partial(get_taskgraph_command_parser, "test-action-callback"),
276 def test_action_callback(command_context, **options):
277 setup_logging(command_context)
279 if not options["parameters"]:
280 options["parameters"] = "project=mozilla-central"
282 taskgraph_commands["test-action-callback"].func(options)
285 def setup_logging(command_context, quiet=False, verbose=True):
287 Set up Python logging for all loggers, sending results to stderr (so
288 that command output can be redirected easily) and adding the typical
289 mach timestamp.
291 # remove the old terminal handler
292 old = command_context.log_manager.replace_terminal_handler(None)
294 # re-add it, with level and fh set appropriately
295 if not quiet:
296 level = logging.DEBUG if verbose else logging.INFO
297 command_context.log_manager.add_terminal_logging(
298 fh=sys.stderr,
299 level=level,
300 write_interval=old.formatter.write_interval,
301 write_times=old.formatter.write_times,
304 # all of the taskgraph logging is unstructured logging
305 command_context.log_manager.enable_unstructured()
308 def show_actions(command_context, options):
309 import gecko_taskgraph
310 import gecko_taskgraph.actions
311 from taskgraph.generator import TaskGraphGenerator
312 from taskgraph.parameters import parameters_loader
314 try:
315 setup_logging(
316 command_context, quiet=options["quiet"], verbose=options["verbose"]
318 parameters = parameters_loader(options["parameters"])
320 tgg = TaskGraphGenerator(
321 root_dir=options.get("root"),
322 parameters=parameters,
325 actions = gecko_taskgraph.actions.render_actions_json(
326 tgg.parameters,
327 tgg.graph_config,
328 decision_task_id="DECISION-TASK",
330 print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
331 except Exception:
332 traceback.print_exc()
333 sys.exit(1)
336 @Command(
337 "taskcluster-load-image",
338 category="ci",
339 description="Load a pre-built Docker image. Note that you need to "
340 "have docker installed and running for this to work.",
341 parser=partial(get_taskgraph_command_parser, "load-image"),
343 def load_image(command_context, **kwargs):
344 taskgraph_commands["load-image"].func(kwargs)
347 @Command(
348 "taskcluster-build-image",
349 category="ci",
350 description="Build a Docker image",
351 parser=partial(get_taskgraph_command_parser, "build-image"),
353 def build_image(command_context, **kwargs):
354 try:
355 taskgraph_commands["build-image"].func(kwargs)
356 except Exception:
357 traceback.print_exc()
358 sys.exit(1)
361 @Command(
362 "taskcluster-image-digest",
363 category="ci",
364 description="Print the digest of the image of this name based on the "
365 "current contents of the tree.",
366 parser=partial(get_taskgraph_command_parser, "build-image"),
368 def image_digest(command_context, **kwargs):
369 taskgraph_commands["image-digest"].func(kwargs)
372 @Command(
373 "release-history",
374 category="ci",
375 description="Query balrog for release history used by enable partials generation",
377 @CommandArgument(
378 "-b",
379 "--branch",
380 help="The gecko project branch used in balrog, such as "
381 "mozilla-central, release, maple",
383 @CommandArgument(
384 "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
386 def generate_partials_builds(command_context, product, branch):
387 from gecko_taskgraph.util.partials import populate_release_history
389 try:
390 import yaml
392 release_history = {"release_history": populate_release_history(product, branch)}
393 print(
394 yaml.safe_dump(
395 release_history, allow_unicode=True, default_flow_style=False
398 except Exception:
399 traceback.print_exc()
400 sys.exit(1)