tools/tryselect/selectors/fuzzy.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import os
   7 import platform
   8 import subprocess
   9 import six
  10 import sys
  11 from distutils.spawn import find_executable
  12 from distutils.version import StrictVersion
  13
  14 from mozbuild.base import MozbuildObject
  15 from mozboot.util import get_state_dir
  16 from mozterm import Terminal
  17
  18 from ..cli import BaseTryParser
  19 from ..tasks import generate_tasks, filter_tasks_by_paths
  20 from ..push import check_working_directory, push_to_try, generate_try_task_config
  21 from ..util.manage_estimates import (
  22     download_task_history_data,
  23     make_trimmed_taskgraph_cache,
  24 )
  25
  26 from taskgraph.target_tasks import filter_by_uncommon_try_tasks
  27
  28 terminal = Terminal()
  29
  30 here = os.path.abspath(os.path.dirname(__file__))
  31 build = MozbuildObject.from_environment(cwd=here)
  32
  33 PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py")
  34
  35 FZF_NOT_FOUND = """
  36 Could not find the `fzf` binary.
  37
  38 The `mach try fuzzy` command depends on fzf. Please install it following the
  39 appropriate instructions for your platform:
  40
  41     https://github.com/junegunn/fzf#installation
  42
  43 Only the binary is required, if you do not wish to install the shell and
  44 editor integrations, download the appropriate binary and put it on your $PATH:
  45
  46     https://github.com/junegunn/fzf/releases
  47 """.lstrip()
  48
  49 FZF_VERSION_FAILED = """
  50 Could not obtain the 'fzf' version.
  51
  52 The 'mach try fuzzy' command depends on fzf, and requires version > 0.20.0
  53 for some of the features. Please install it following the appropriate
  54 instructions for your platform:
  55
  56     https://github.com/junegunn/fzf#installation
  57
  58 Only the binary is required, if you do not wish to install the shell and
  59 editor integrations, download the appropriate binary and put it on your $PATH:
  60
  61     https://github.com/junegunn/fzf/releases
  62 """.lstrip()
  63
  64 FZF_INSTALL_FAILED = """
  65 Failed to install fzf.
  66
  67 Please install fzf manually following the appropriate instructions for your
  68 platform:
  69
  70     https://github.com/junegunn/fzf#installation
  71
  72 Only the binary is required, if you do not wish to install the shell and
  73 editor integrations, download the appropriate binary and put it on your $PATH:
  74
  75     https://github.com/junegunn/fzf/releases
  76 """.lstrip()
  77
  78 FZF_HEADER = """
  79 For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
  80 {shortcuts}
  81 """.strip()
  82
  83 fzf_shortcuts = {
  84     "ctrl-a": "select-all",
  85     "ctrl-d": "deselect-all",
  86     "ctrl-t": "toggle-all",
  87     "alt-bspace": "beginning-of-line+kill-line",
  88     "?": "toggle-preview",
  89 }
  90
  91 fzf_header_shortcuts = [
  92     ("select", "tab"),
  93     ("accept", "enter"),
  94     ("cancel", "ctrl-c"),
  95     ("select-all", "ctrl-a"),
  96     ("cursor-up", "up"),
  97     ("cursor-down", "down"),
  98 ]
  99
 100
 101 class FuzzyParser(BaseTryParser):
 102     name = "fuzzy"
 103     arguments = [
 104         [
 105             ["-q", "--query"],
 106             {
 107                 "metavar": "STR",
 108                 "action": "append",
 109                 "default": [],
 110                 "help": "Use the given query instead of entering the selection "
 111                 "interface. Equivalent to typing <query><ctrl-a><enter> "
 112                 "from the interface. Specifying multiple times schedules "
 113                 "the union of computed tasks.",
 114             },
 115         ],
 116         [
 117             ["-i", "--interactive"],
 118             {
 119                 "action": "store_true",
 120                 "default": False,
 121                 "help": "Force running fzf interactively even when using presets or "
 122                 "queries with -q/--query.",
 123             },
 124         ],
 125         [
 126             ["-x", "--and"],
 127             {
 128                 "dest": "intersection",
 129                 "action": "store_true",
 130                 "default": False,
 131                 "help": "When specifying queries on the command line with -q/--query, "
 132                 "use the intersection of tasks rather than the union. This is "
 133                 "especially useful for post filtering presets.",
 134             },
 135         ],
 136         [
 137             ["-e", "--exact"],
 138             {
 139                 "action": "store_true",
 140                 "default": False,
 141                 "help": "Enable exact match mode. Terms will use an exact match "
 142                 "by default, and terms prefixed with ' will become fuzzy.",
 143             },
 144         ],
 145         [
 146             ["-u", "--update"],
 147             {
 148                 "action": "store_true",
 149                 "default": False,
 150                 "help": "Update fzf before running.",
 151             },
 152         ],
 153         [
 154             ["-s", "--show-estimates"],
 155             {
 156                 "action": "store_true",
 157                 "default": False,
 158                 "help": "Show task duration estimates.",
 159             },
 160         ],
 161         [
 162             ["--disable-target-task-filter"],
 163             {
 164                 "action": "store_true",
 165                 "default": False,
 166                 "help": "Some tasks run on mozilla-central but are filtered out "
 167                 "of the default list due to resource constraints. This flag "
 168                 "disables this filtering.",
 169             },
 170         ],
 171     ]
 172     common_groups = ["push", "task", "preset"]
 173     task_configs = [
 174         "artifact",
 175         "browsertime",
 176         "chemspill-prio",
 177         "disable-pgo",
 178         "env",
 179         "gecko-profile",
 180         "path",
 181         "pernosco",
 182         "rebuild",
 183         "routes",
 184         "worker-overrides",
 185     ]
 186
 187
 188 def run_cmd(cmd, cwd=None):
 189     is_win = platform.system() == "Windows"
 190     return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
 191
 192
 193 def run_fzf_install_script(fzf_path):
 194     if platform.system() == "Windows":
 195         cmd = ["bash", "-c", "./install --bin"]
 196     else:
 197         cmd = ["./install", "--bin"]
 198
 199     if run_cmd(cmd, cwd=fzf_path):
 200         print(FZF_INSTALL_FAILED)
 201         sys.exit(1)
 202
 203
 204 def should_force_fzf_update(fzf_bin):
 205     cmd = [fzf_bin, "--version"]
 206     try:
 207         fzf_version = subprocess.check_output(cmd)
 208     except subprocess.CalledProcessError:
 209         print(FZF_VERSION_FAILED)
 210         sys.exit(1)
 211
 212     # Some fzf versions have extra, e.g 0.18.0 (ff95134)
 213     fzf_version = six.ensure_text(fzf_version.split()[0])
 214
 215     # 0.20.0 introduced passing selections through a temporary file,
 216     # which is good for large ctrl-a actions.
 217     if StrictVersion(fzf_version) < StrictVersion("0.20.0"):
 218         print("fzf version is old, forcing update.")
 219         return True
 220     return False
 221
 222
 223 def fzf_bootstrap(update=False):
 224     """Bootstrap fzf if necessary and return path to the executable.
 225
 226     The bootstrap works by cloning the fzf repository and running the included
 227     `install` script. If update is True, we will pull the repository and re-run
 228     the install script.
 229     """
 230     fzf_bin = find_executable("fzf")
 231     if fzf_bin and should_force_fzf_update(fzf_bin):
 232         update = True
 233
 234     if fzf_bin and not update:
 235         return fzf_bin
 236
 237     fzf_path = os.path.join(get_state_dir(), "fzf")
 238
 239     # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH
 240     # Swap to os.path.commonpath when we're not on Py2
 241     if fzf_bin and update and not fzf_bin.startswith(fzf_path):
 242         print(
 243             "fzf installed somewhere other than {}, please update manually".format(
 244                 fzf_path
 245             )
 246         )
 247         sys.exit(1)
 248
 249     def get_fzf():
 250         return find_executable("fzf", os.path.join(fzf_path, "bin"))
 251
 252     if os.path.isdir(fzf_path):
 253         if update:
 254             ret = run_cmd(["git", "pull"], cwd=fzf_path)
 255             if ret:
 256                 print("Update fzf failed.")
 257                 sys.exit(1)
 258
 259             run_fzf_install_script(fzf_path)
 260             return get_fzf()
 261
 262         fzf_bin = get_fzf()
 263         if not fzf_bin or should_force_fzf_update(fzf_bin):
 264             return fzf_bootstrap(update=True)
 265
 266         return fzf_bin
 267
 268     if not update:
 269         install = input("Could not detect fzf, install it now? [y/n]: ")
 270         if install.lower() != "y":
 271             return
 272
 273     if not find_executable("git"):
 274         print("Git not found.")
 275         print(FZF_INSTALL_FAILED)
 276         sys.exit(1)
 277
 278     cmd = ["git", "clone", "--depth", "1", "https://github.com/junegunn/fzf.git"]
 279     if subprocess.call(cmd, cwd=os.path.dirname(fzf_path)):
 280         print(FZF_INSTALL_FAILED)
 281         sys.exit(1)
 282
 283     run_fzf_install_script(fzf_path)
 284
 285     print("Installed fzf to {}".format(fzf_path))
 286     return get_fzf()
 287
 288
 289 def format_header():
 290     shortcuts = []
 291     for action, key in fzf_header_shortcuts:
 292         shortcuts.append(
 293             "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format(
 294                 t=terminal, action=action, key=key
 295             )
 296         )
 297     return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal)
 298
 299
 300 def run_fzf(cmd, tasks):
 301     env = dict(os.environ)
 302     env.update(
 303         {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])}
 304     )
 305     proc = subprocess.Popen(
 306         cmd,
 307         stdout=subprocess.PIPE,
 308         stdin=subprocess.PIPE,
 309         env=env,
 310         universal_newlines=True,
 311     )
 312     out = proc.communicate("\n".join(tasks))[0].splitlines()
 313
 314     selected = []
 315     query = None
 316     if out:
 317         query = out[0]
 318         selected = out[1:]
 319     return query, selected
 320
 321
 322 def run(
 323     update=False,
 324     query=None,
 325     intersect_query=None,
 326     try_config=None,
 327     full=False,
 328     parameters=None,
 329     save_query=False,
 330     push=True,
 331     message="{msg}",
 332     test_paths=None,
 333     exact=False,
 334     closed_tree=False,
 335     show_estimates=False,
 336     disable_target_task_filter=False,
 337 ):
 338     fzf = fzf_bootstrap(update)
 339
 340     if not fzf:
 341         print(FZF_NOT_FOUND)
 342         return 1
 343
 344     check_working_directory(push)
 345     tg = generate_tasks(
 346         parameters, full=full, disable_target_task_filter=disable_target_task_filter
 347     )
 348     all_tasks = sorted(tg.tasks.keys())
 349
 350     # graph_Cache created by generate_tasks, recreate the path to that file.
 351     cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph")
 352     if full:
 353         graph_cache = os.path.join(cache_dir, "full_task_graph")
 354         dep_cache = os.path.join(cache_dir, "full_task_dependencies")
 355         target_set = os.path.join(cache_dir, "full_task_set")
 356     else:
 357         graph_cache = os.path.join(cache_dir, "target_task_graph")
 358         dep_cache = os.path.join(cache_dir, "target_task_dependencies")
 359         target_set = os.path.join(cache_dir, "target_task_set")
 360
 361     if show_estimates:
 362         download_task_history_data(cache_dir=cache_dir)
 363         make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set)
 364
 365     if not full and not disable_target_task_filter:
 366         # Put all_tasks into a list because it's used multiple times, and "filter()"
 367         # returns a consumable iterator.
 368         all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks))
 369
 370     if test_paths:
 371         all_tasks = filter_tasks_by_paths(all_tasks, test_paths)
 372         if not all_tasks:
 373             return 1
 374
 375     key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()]
 376     base_cmd = [
 377         fzf,
 378         "-m",
 379         "--bind",
 380         ",".join(key_shortcuts),
 381         "--header",
 382         format_header(),
 383         "--preview-window=right:30%",
 384         "--print-query",
 385     ]
 386
 387     if show_estimates:
 388         base_cmd.extend(
 389             [
 390                 "--preview",
 391                 '{} {} -g {} -s -c {} -t "{{+f}}"'.format(
 392                     sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir
 393                 ),
 394             ]
 395         )
 396     else:
 397         base_cmd.extend(
 398             [
 399                 "--preview",
 400                 '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT),
 401             ]
 402         )
 403
 404     if exact:
 405         base_cmd.append("--exact")
 406
 407     selected = set()
 408     queries = []
 409
 410     def get_tasks(query_arg=None, candidate_tasks=all_tasks):
 411         cmd = base_cmd[:]
 412         if query_arg and query_arg != "INTERACTIVE":
 413             cmd.extend(["-f", query_arg])
 414
 415         query_str, tasks = run_fzf(cmd, sorted(candidate_tasks))
 416         queries.append(query_str)
 417         return set(tasks)
 418
 419     for q in query or []:
 420         selected |= get_tasks(q)
 421
 422     for q in intersect_query or []:
 423         if not selected:
 424             tasks = get_tasks(q)
 425             selected |= tasks
 426         else:
 427             tasks = get_tasks(q, selected)
 428             selected &= tasks
 429
 430     if not queries:
 431         selected = get_tasks()
 432
 433     if not selected:
 434         print("no tasks selected")
 435         return
 436
 437     if save_query:
 438         return queries
 439
 440     # build commit message
 441     msg = "Fuzzy"
 442     args = ["query={}".format(q) for q in queries]
 443     if test_paths:
 444         args.append("paths={}".format(":".join(test_paths)))
 445     if args:
 446         msg = "{} {}".format(msg, "&".join(args))
 447     return push_to_try(
 448         "fuzzy",
 449         message.format(msg=msg),
 450         try_task_config=generate_try_task_config("fuzzy", selected, try_config),
 451         push=push,
 452         closed_tree=closed_tree,
 453     )