Backed out 2 changesets (bug 1900622) for causing Bug 1908553 and ktlint failure...
[gecko.git] / tools / tryselect / mach_commands.py
blob47ec3ae8ebe74157c02024a7e6d48f830ad4a0ea
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 import argparse
7 import importlib
8 import os
9 import sys
11 from mach.decorators import Command, SubCommand
12 from mach.util import get_state_dir
13 from mozbuild.base import BuildEnvironmentNotFoundException
14 from mozbuild.util import memoize
16 CONFIG_ENVIRONMENT_NOT_FOUND = """
17 No config environment detected. This means we are unable to properly
18 detect test files in the specified paths or tags. Please run:
20 $ mach configure
22 and try again.
23 """.lstrip()
26 class get_parser:
27 def __init__(self, selector):
28 self.selector = selector
30 def __call__(self):
31 mod = importlib.import_module("tryselect.selectors.{}".format(self.selector))
32 return getattr(mod, "{}Parser".format(self.selector.capitalize()))()
35 def generic_parser():
36 from tryselect.cli import BaseTryParser
38 parser = BaseTryParser()
39 parser.add_argument("argv", nargs=argparse.REMAINDER)
40 return parser
43 def init(command_context):
44 from tryselect import push
46 push.MAX_HISTORY = command_context._mach_context.settings["try"]["maxhistory"]
49 @memoize
50 def presets(command_context):
51 from tryselect.preset import MergedHandler
53 # Create our handler using both local and in-tree presets. The first
54 # path in this list will be treated as the 'user' file for the purposes
55 # of saving and editing. All subsequent paths are 'read-only'. We check
56 # an environment variable first for testing purposes.
57 if os.environ.get("MACH_TRY_PRESET_PATHS"):
58 preset_paths = os.environ["MACH_TRY_PRESET_PATHS"].split(os.pathsep)
59 else:
60 preset_paths = [
61 os.path.join(get_state_dir(), "try_presets.yml"),
62 os.path.join(
63 command_context.topsrcdir, "tools", "tryselect", "try_presets.yml"
67 return MergedHandler(*preset_paths)
70 def handle_presets(
71 command_context, preset_action=None, save=None, preset=None, **kwargs
73 """Handle preset related arguments.
75 This logic lives here so that the underlying selectors don't need
76 special preset handling. They can all save and load presets the same
77 way.
78 """
79 from tryselect.util.dicttools import merge
81 user_presets = presets(command_context).handlers[0]
82 if preset_action == "list":
83 presets(command_context).list()
84 sys.exit()
86 if preset_action == "edit":
87 user_presets.edit()
88 sys.exit()
90 parser = command_context._mach_context.handler.parser
91 subcommand = command_context._mach_context.handler.subcommand
92 if "preset" not in parser.common_groups:
93 return kwargs
95 default = parser.get_default
96 if save:
97 selector = (
98 subcommand or command_context._mach_context.settings["try"]["default"]
101 # Only save non-default values for simplicity.
102 kwargs = {k: v for k, v in kwargs.items() if v != default(k)}
103 user_presets.save(save, selector=selector, **kwargs)
104 print("preset saved, run with: --preset={}".format(save))
105 sys.exit()
107 if preset:
108 if preset not in presets(command_context):
109 command_context._mach_context.parser.error(
110 "preset '{}' does not exist".format(preset)
113 name = preset
114 preset = presets(command_context)[name]
115 selector = preset.pop("selector")
116 preset.pop("description", None) # description isn't used by any selectors
118 if not subcommand:
119 subcommand = selector
120 elif subcommand != selector:
121 print(
122 "error: preset '{}' exists for a different selector "
123 "(did you mean to run 'mach try {}' instead?)".format(name, selector)
125 sys.exit(1)
127 # Order of precedence is defaults -> presets -> cli. Configuration
128 # from the right overwrites configuration from the left.
129 defaults = {}
130 nondefaults = {}
131 for k, v in kwargs.items():
132 if v == default(k):
133 defaults[k] = v
134 else:
135 nondefaults[k] = v
137 kwargs = merge(defaults, preset, nondefaults)
139 return kwargs
142 def handle_try_params(command_context, **kwargs):
143 from tryselect.util.dicttools import merge
145 to_validate = []
146 kwargs.setdefault("try_config_params", {})
147 for cls in command_context._mach_context.handler.parser.task_configs.values():
148 params = cls.get_parameters(**kwargs)
149 if params is not None:
150 to_validate.append(cls)
151 kwargs["try_config_params"] = merge(kwargs["try_config_params"], params)
153 for name in cls.dests:
154 del kwargs[name]
156 # Validate task_configs after they have all been parsed to avoid
157 # depending on the order they were processed.
158 for cls in to_validate:
159 cls.validate(**kwargs)
160 return kwargs
163 def run(command_context, **kwargs):
164 kwargs = handle_presets(command_context, **kwargs)
166 if command_context._mach_context.handler.parser.task_configs:
167 kwargs = handle_try_params(command_context, **kwargs)
169 mod = importlib.import_module(
170 "tryselect.selectors.{}".format(
171 command_context._mach_context.handler.subcommand
174 return mod.run(**kwargs)
177 @Command(
178 "try",
179 category="ci",
180 description="Push selected tasks to the try server",
181 parser=generic_parser,
182 virtualenv_name="try",
184 def try_default(command_context, argv=None, **kwargs):
185 """Push selected tests to the try server.
187 The |mach try| command is a frontend for scheduling tasks to
188 run on try server using selectors. A selector is a subcommand
189 that provides its own set of command line arguments and are
190 listed below.
192 If no subcommand is specified, the `auto` selector is run by
193 default. Run |mach try auto --help| for more information on
194 scheduling with the `auto` selector.
196 init(command_context)
197 subcommand = command_context._mach_context.handler.subcommand
198 # We do special handling of presets here so that `./mach try --preset foo`
199 # works no matter what subcommand 'foo' was saved with.
200 preset = kwargs["preset"]
201 if preset:
202 if preset not in presets(command_context):
203 command_context._mach_context.handler.parser.error(
204 "preset '{}' does not exist".format(preset)
207 subcommand = presets(command_context)[preset]["selector"]
209 sub = subcommand or command_context._mach_context.settings["try"]["default"]
210 return command_context._mach_context.commands.dispatch(
211 "try", command_context._mach_context, subcommand=sub, argv=argv, **kwargs
215 @SubCommand(
216 "try",
217 "fuzzy",
218 description="Select tasks on try using a fuzzy finder",
219 parser=get_parser("fuzzy"),
220 virtualenv_name="try",
222 def try_fuzzy(command_context, **kwargs):
223 """Select which tasks to run with a fuzzy finding interface (fzf).
225 When entering the fzf interface you'll be confronted by two panes. The
226 one on the left contains every possible task you can schedule, the one
227 on the right contains the list of selected tasks. In other words, the
228 tasks that will be scheduled once you you press <enter>.
230 At first fzf will automatically select whichever task is under your
231 cursor, which simplifies the case when you are looking for a single
232 task. But normally you'll want to select many tasks. To accomplish
233 you'll generally start by typing a query in the search bar to filter
234 down the list of tasks (see Extended Search below). Then you'll either:
236 A) Move the cursor to each task you want and press <tab> to select it.
237 Notice it now shows up in the pane to the right.
241 B) Press <ctrl-a> to select every task that matches your filter.
243 You can delete your query, type a new one and select further tasks as
244 many times as you like. Once you are happy with your selection, press
245 <enter> to push the selected tasks to try.
247 All selected task labels and their dependencies will be scheduled. This
248 means you can select a test task and its build will automatically be
249 filled in.
252 Keyboard Shortcuts
253 ------------------
255 When in the fuzzy finder interface, start typing to filter down the
256 task list. Then use the following keyboard shortcuts to select tasks:
258 Ctrl-K / Up => Move cursor up
259 Ctrl-J / Down => Move cursor down
260 Tab => Select task + move cursor down
261 Shift-Tab => Select task + move cursor up
262 Ctrl-A => Select all currently filtered tasks
263 Ctrl-D => De-select all currently filtered tasks
264 Ctrl-T => Toggle select all currently filtered tasks
265 Alt-Bspace => Clear query from input bar
266 Enter => Accept selection and exit
267 Ctrl-C / Esc => Cancel selection and exit
268 ? => Toggle preview pane
270 There are many more shortcuts enabled by default, you can also define
271 your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS
272 environment variable. See `man fzf` for more info.
275 Extended Search
276 ---------------
278 When typing in search terms, the following modifiers can be applied:
280 'word: exact match (line must contain the literal string "word")
281 ^word: exact prefix match (line must start with literal "word")
282 word$: exact suffix match (line must end with literal "word")
283 !word: exact negation match (line must not contain literal "word")
284 'a | 'b: OR operator (joins two exact match operators together)
286 For example:
288 ^start 'exact | !ignore fuzzy end$
291 Documentation
292 -------------
294 For more detailed documentation, please see:
295 https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html
297 init(command_context)
298 if kwargs.pop("interactive"):
299 kwargs["query"].append("INTERACTIVE")
301 if kwargs.pop("intersection"):
302 kwargs["intersect_query"] = kwargs["query"]
303 del kwargs["query"]
305 if kwargs.get("save") and not kwargs.get("query"):
306 # If saving preset without -q/--query, allow user to use the
307 # interface to build the query.
308 kwargs_copy = kwargs.copy()
309 kwargs_copy["dry_run"] = True
310 kwargs_copy["save"] = None
311 kwargs["query"] = run(command_context, save_query=True, **kwargs_copy)
312 if not kwargs["query"]:
313 return
315 if kwargs.get("tag") and kwargs.get("paths"):
316 command_context._mach_context.handler.parser.error(
317 "ERROR: only --tag or <path> can be used, not both."
319 return
321 if kwargs.get("paths"):
322 kwargs["test_paths"] = kwargs["paths"]
324 if kwargs.get("tag"):
325 kwargs["test_tag"] = kwargs["tag"]
327 return run(command_context, **kwargs)
330 @SubCommand(
331 "try",
332 "chooser",
333 description="Schedule tasks by selecting them from a web interface.",
334 parser=get_parser("chooser"),
335 virtualenv_name="try",
337 def try_chooser(command_context, **kwargs):
338 """Push tasks selected from a web interface to try.
340 This selector will build the taskgraph and spin up a dynamically
341 created 'trychooser-like' web-page on the localhost. After a selection
342 has been made, pressing the 'Push' button will automatically push the
343 selection to try.
345 init(command_context)
346 command_context.activate_virtualenv()
348 return run(command_context, **kwargs)
351 @SubCommand(
352 "try",
353 "auto",
354 description="Automatically determine which tasks to run. This runs the same "
355 "set of tasks that would be run on autoland. This "
356 "selector is EXPERIMENTAL.",
357 parser=get_parser("auto"),
358 virtualenv_name="try",
360 def try_auto(command_context, **kwargs):
361 init(command_context)
362 return run(command_context, **kwargs)
365 @SubCommand(
366 "try",
367 "again",
368 description="Schedule a previously generated (non try syntax) push again.",
369 parser=get_parser("again"),
370 virtualenv_name="try",
372 def try_again(command_context, **kwargs):
373 init(command_context)
374 return run(command_context, **kwargs)
377 @SubCommand(
378 "try",
379 "empty",
380 description="Push to try without scheduling any tasks.",
381 parser=get_parser("empty"),
382 virtualenv_name="try",
384 def try_empty(command_context, **kwargs):
385 """Push to try, running no builds or tests
387 This selector does not prompt you to run anything, it just pushes
388 your patches to try, running no builds or tests by default. After
389 the push finishes, you can manually add desired jobs to your push
390 via Treeherder's Add New Jobs feature, located in the per-push
391 menu.
393 init(command_context)
394 return run(command_context, **kwargs)
397 @SubCommand(
398 "try",
399 "syntax",
400 description="Select tasks on try using try syntax",
401 parser=get_parser("syntax"),
402 virtualenv_name="try",
404 def try_syntax(command_context, **kwargs):
405 """Push the current tree to try, with the specified syntax.
407 Build options, platforms and regression tests may be selected
408 using the usual try options (-b, -p and -u respectively). In
409 addition, tests in a given directory may be automatically
410 selected by passing that directory as a positional argument to the
411 command. For example:
413 mach try -b d -p linux64 dom testing/web-platform/tests/dom
415 would schedule a try run for linux64 debug consisting of all
416 tests under dom/ and testing/web-platform/tests/dom.
418 Test selection using positional arguments is available for
419 mochitests, reftests, xpcshell tests and web-platform-tests.
421 Tests may be also filtered by passing --tag to the command,
422 which will run only tests marked as having the specified
423 tags e.g.
425 mach try -b d -p win64 --tag media
427 would run all tests tagged 'media' on Windows 64.
429 If both positional arguments or tags and -u are supplied, the
430 suites in -u will be run in full. Where tests are selected by
431 positional argument they will be run in a single chunk.
433 If no build option is selected, both debug and opt will be
434 scheduled. If no platform is selected a default is taken from
435 the AUTOTRY_PLATFORM_HINT environment variable, if set.
437 The command requires either its own mercurial extension ("push-to-try",
438 installable from mach vcs-setup) or a git repo using git-cinnabar
439 (installable from mach vcs-setup).
442 init(command_context)
443 try:
444 if command_context.substs.get("MOZ_ARTIFACT_BUILDS"):
445 kwargs["local_artifact_build"] = True
446 except BuildEnvironmentNotFoundException:
447 # If we don't have a build locally, we can't tell whether
448 # an artifact build is desired, but we still want the
449 # command to succeed, if possible.
450 pass
452 config_status = os.path.join(command_context.topobjdir, "config.status")
453 if (kwargs["paths"] or kwargs["tags"]) and not config_status:
454 print(CONFIG_ENVIRONMENT_NOT_FOUND)
455 sys.exit(1)
457 return run(command_context, **kwargs)
460 @SubCommand(
461 "try",
462 "coverage",
463 description="Select tasks on try using coverage data",
464 parser=get_parser("coverage"),
465 virtualenv_name="try",
467 def try_coverage(command_context, **kwargs):
468 """Select which tasks to use using coverage data."""
469 init(command_context)
470 return run(command_context, **kwargs)
473 @SubCommand(
474 "try",
475 "release",
476 description="Push the current tree to try, configured for a staging release.",
477 parser=get_parser("release"),
478 virtualenv_name="try",
480 def try_release(command_context, **kwargs):
481 """Push the current tree to try, configured for a staging release."""
482 init(command_context)
483 return run(command_context, **kwargs)
486 @SubCommand(
487 "try",
488 "scriptworker",
489 description="Run scriptworker tasks against a recent release.",
490 parser=get_parser("scriptworker"),
491 virtualenv_name="try",
493 def try_scriptworker(command_context, **kwargs):
494 """Run scriptworker tasks against a recent release.
496 Requires VPN and shipit access.
498 init(command_context)
499 return run(command_context, **kwargs)
502 @SubCommand(
503 "try",
504 "compare",
505 description="Push two try jobs, one on your current commit and another on the one you specify",
506 parser=get_parser("compare"),
507 virtualenv_name="try",
509 def try_compare(command_context, **kwargs):
510 init(command_context)
511 return run(command_context, **kwargs)
514 @SubCommand(
515 "try",
516 "perf",
517 description="Try selector for running performance tests.",
518 parser=get_parser("perf"),
519 virtualenv_name="try",
521 def try_perf(command_context, **kwargs):
522 init(command_context)
523 return run(command_context, **kwargs)