tools/tryselect/util/fzf.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import os
   7 import platform
   8 import shutil
   9 import subprocess
  10 import sys
  11 from distutils.spawn import find_executable
  12
  13 import mozfile
  14 import six
  15 from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks
  16 from mach.util import get_state_dir
  17 from mozboot.util import http_download_and_save
  18 from mozbuild.base import MozbuildObject
  19 from mozterm import Terminal
  20 from packaging.version import Version
  21
  22 from ..push import check_working_directory
  23 from ..tasks import generate_tasks
  24 from ..util.manage_estimates import (
  25     download_task_history_data,
  26     make_trimmed_taskgraph_cache,
  27 )
  28
  29 terminal = Terminal()
  30
  31 here = os.path.abspath(os.path.dirname(__file__))
  32 build = MozbuildObject.from_environment(cwd=here)
  33
  34 PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py")
  35
  36 FZF_MIN_VERSION = "0.20.0"
  37 FZF_CURRENT_VERSION = "0.29.0"
  38
  39 # It would make more sense to have the full filename be the key; but that makes
  40 # the line too long and ./mach lint and black can't agree about what to about that.
  41 # You can get these from the github release, e.g.
  42 #          https://github.com/junegunn/fzf/releases/download/0.24.1/fzf_0.24.1_checksums.txt
  43 # However the darwin releases may not be included, so double check you have everything
  44 FZF_CHECKSUMS = {
  45     "linux_armv5.tar.gz": "61d3c2aa77b977ba694836fd1134da9272bd97ee490ececaf87959b985820111",
  46     "linux_armv6.tar.gz": "db6b30fcbbd99ac4cf7e3ff6c5db1d3c0afcbe37d10ec3961bdc43e8c4f2e4f9",
  47     "linux_armv7.tar.gz": "ed86f0e91e41d2cea7960a78e3eb175dc2a5fc1510380c195d0c3559bfdc701c",
  48     "linux_arm64.tar.gz": "47988d8b68905541cbc26587db3ed1cfa8bc3aa8da535120abb4229b988f259e",
  49     "linux_amd64.tar.gz": "0106f458b933be65edb0e8f0edb9a16291a79167836fd26a77ff5496269b5e9a",
  50     "windows_armv5.zip": "08eaac45b3600d82608d292c23e7312696e7e11b6278b292feba25e8eb91c712",
  51     "windows_armv6.zip": "8b6618726a9d591a45120fddebc29f4164e01ce6639ed9aa8fc79ab03eefcfed",
  52     "windows_armv7.zip": "c167117b4c08f4f098446291115871ce5f14a8a8b22f0ca70e1b4342452ab5d7",
  53     "windows_arm64.zip": "0cda7bf68850a3e867224a05949612405e63a4421d52396c1a6c9427d4304d72",
  54     "windows_amd64.zip": "f0797ceee089017108c80b09086c71b8eec43d4af11ce939b78b1d5cfd202540",
  55     "darwin_arm64.zip": "2571b4d381f1fc691e7603bbc8113a67116da2404751ebb844818d512dd62b4b",
  56     "darwin_amd64.zip": "bc541e8ae0feb94efa96424bfe0b944f746db04e22f5cccfe00709925839a57f",
  57     "openbsd_amd64.tar.gz": "b62343827ff83949c09d5e2c8ca0c1198d05f733c9a779ec37edd840541ccdab",
  58     "freebsd_amd64.tar.gz": "f0367f2321c070d103589c7c7eb6a771bc7520820337a6c2fbb75be37ff783a9",
  59 }
  60
  61 FZF_INSTALL_MANUALLY = """
  62 The `mach try fuzzy` command depends on fzf. Please install it following the
  63 appropriate instructions for your platform:
  64
  65     https://github.com/junegunn/fzf#installation
  66
  67 Only the binary is required, if you do not wish to install the shell and
  68 editor integrations, download the appropriate binary and put it on your $PATH:
  69
  70     https://github.com/junegunn/fzf/releases
  71 """.lstrip()
  72
  73 FZF_COULD_NOT_DETERMINE_PLATFORM = (
  74     """
  75 Could not automatically obtain the `fzf` binary because we could not determine
  76 your Operating System.
  77
  78 """.lstrip()
  79     + FZF_INSTALL_MANUALLY
  80 )
  81
  82 FZF_COULD_NOT_DETERMINE_MACHINE = (
  83     """
  84 Could not automatically obtain the `fzf` binary because we could not determine
  85 your machine type.  It's reported as '%s' and we don't handle that case; but fzf
  86 may still be available as a prebuilt binary.
  87
  88 """.lstrip()
  89     + FZF_INSTALL_MANUALLY
  90 )
  91
  92 FZF_NOT_SUPPORTED_X86 = (
  93     """
  94 We don't believe that a prebuilt binary for `fzf` if available on %s, but we
  95 could be wrong.
  96
  97 """.lstrip()
  98     + FZF_INSTALL_MANUALLY
  99 )
 100
 101 FZF_NOT_FOUND = (
 102     """
 103 Could not find the `fzf` binary.
 104
 105 """.lstrip()
 106     + FZF_INSTALL_MANUALLY
 107 )
 108
 109 FZF_VERSION_FAILED = (
 110     """
 111 Could not obtain the 'fzf' version; we require version > 0.20.0 for some of
 112 the features.
 113
 114 """.lstrip()
 115     + FZF_INSTALL_MANUALLY
 116 )
 117
 118 FZF_INSTALL_FAILED = (
 119     """
 120 Failed to install fzf.
 121
 122 """.lstrip()
 123     + FZF_INSTALL_MANUALLY
 124 )
 125
 126 FZF_HEADER = """
 127 For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
 128 {shortcuts}
 129 """.strip()
 130
 131 fzf_shortcuts = {
 132     "ctrl-a": "select-all",
 133     "ctrl-d": "deselect-all",
 134     "ctrl-t": "toggle-all",
 135     "alt-bspace": "beginning-of-line+kill-line",
 136     "?": "toggle-preview",
 137 }
 138
 139 fzf_header_shortcuts = [
 140     ("select", "tab"),
 141     ("accept", "enter"),
 142     ("cancel", "ctrl-c"),
 143     ("select-all", "ctrl-a"),
 144     ("cursor-up", "up"),
 145     ("cursor-down", "down"),
 146 ]
 147
 148
 149 def get_fzf_platform():
 150     if platform.machine() in ["i386", "i686"]:
 151         print(FZF_NOT_SUPPORTED_X86 % platform.machine())
 152         sys.exit(1)
 153
 154     if platform.system().lower() == "windows":
 155         if platform.machine().lower() in ["x86_64", "amd64"]:
 156             return "windows_amd64.zip"
 157         elif platform.machine().lower() == "arm64":
 158             return "windows_arm64.zip"
 159         else:
 160             print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
 161             sys.exit(1)
 162     elif platform.system().lower() == "darwin":
 163         if platform.machine().lower() in ["x86_64", "amd64"]:
 164             return "darwin_amd64.zip"
 165         elif platform.machine().lower() == "arm64":
 166             return "darwin_arm64.zip"
 167         else:
 168             print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
 169             sys.exit(1)
 170     elif platform.system().lower() == "linux":
 171         if platform.machine().lower() in ["x86_64", "amd64"]:
 172             return "linux_amd64.tar.gz"
 173         elif platform.machine().lower() == "arm64":
 174             return "linux_arm64.tar.gz"
 175         else:
 176             print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine())
 177             sys.exit(1)
 178     else:
 179         print(FZF_COULD_NOT_DETERMINE_PLATFORM)
 180         sys.exit(1)
 181
 182
 183 def get_fzf_state_dir():
 184     return os.path.join(get_state_dir(), "fzf")
 185
 186
 187 def get_fzf_filename():
 188     return "fzf-%s-%s" % (FZF_CURRENT_VERSION, get_fzf_platform())
 189
 190
 191 def get_fzf_download_link():
 192     return "https://github.com/junegunn/fzf/releases/download/%s/%s" % (
 193         FZF_CURRENT_VERSION,
 194         get_fzf_filename(),
 195     )
 196
 197
 198 def clean_up_state_dir():
 199     """
 200     We used to have a checkout of fzf that we would update.
 201     Now we only download the bin and cpin the hash; so if
 202     we find the old git checkout, wipe it
 203     """
 204
 205     fzf_path = os.path.join(get_state_dir(), "fzf")
 206     git_path = os.path.join(fzf_path, ".git")
 207     if os.path.isdir(git_path):
 208         shutil.rmtree(fzf_path, ignore_errors=True)
 209
 210     # Also delete any existing fzf binary
 211     fzf_bin = find_executable("fzf", fzf_path)
 212     if fzf_bin:
 213         mozfile.remove(fzf_bin)
 214
 215     # Make sure the state dir is present
 216     if not os.path.isdir(fzf_path):
 217         os.makedirs(fzf_path)
 218
 219
 220 def download_and_install_fzf():
 221     clean_up_state_dir()
 222     download_link = get_fzf_download_link()
 223     download_file = get_fzf_filename()
 224     download_destination_path = get_fzf_state_dir()
 225     download_destination_file = os.path.join(download_destination_path, download_file)
 226     http_download_and_save(
 227         download_link, download_destination_file, FZF_CHECKSUMS[get_fzf_platform()]
 228     )
 229
 230     mozfile.extract(download_destination_file, download_destination_path)
 231     mozfile.remove(download_destination_file)
 232
 233
 234 def get_fzf_version(fzf_bin):
 235     cmd = [fzf_bin, "--version"]
 236     try:
 237         fzf_version = subprocess.check_output(cmd)
 238     except subprocess.CalledProcessError:
 239         print(FZF_VERSION_FAILED)
 240         sys.exit(1)
 241
 242     # Some fzf versions have extra, e.g 0.18.0 (ff95134)
 243     fzf_version = six.ensure_text(fzf_version.split()[0])
 244
 245     return fzf_version
 246
 247
 248 def should_force_fzf_update(fzf_bin):
 249     fzf_version = get_fzf_version(fzf_bin)
 250
 251     # 0.20.0 introduced passing selections through a temporary file,
 252     # which is good for large ctrl-a actions.
 253     if Version(fzf_version) < Version(FZF_MIN_VERSION):
 254         print("fzf version is old, you must update to use ./mach try fuzzy.")
 255         return True
 256     return False
 257
 258
 259 def fzf_bootstrap(update=False):
 260     """
 261     Bootstrap fzf if necessary and return path to the executable.
 262
 263     This function is a bit complicated. We fetch a new version of fzf if:
 264     1) an existing fzf is too outdated
 265     2) the user says --update and we are behind the recommended version
 266     3) no fzf can be found and
 267       3a) user passes --update
 268       3b) user agrees to a prompt
 269
 270     """
 271     fzf_path = get_fzf_state_dir()
 272
 273     fzf_bin = find_executable("fzf")
 274     if not fzf_bin:
 275         fzf_bin = find_executable("fzf", fzf_path)
 276
 277     if fzf_bin and should_force_fzf_update(fzf_bin):  # Case (1)
 278         update = True
 279
 280     if fzf_bin and not update:
 281         return fzf_bin
 282
 283     elif fzf_bin and update:
 284         # Case 2
 285         fzf_version = get_fzf_version(fzf_bin)
 286         if Version(fzf_version) < Version(FZF_CURRENT_VERSION) and update:
 287             # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH
 288             # Swap to os.path.commonpath when we're not on Py2
 289             if fzf_bin and update and not fzf_bin.startswith(fzf_path):
 290                 print(
 291                     "fzf installed somewhere other than {}, please update manually".format(
 292                         fzf_path
 293                     )
 294                 )
 295                 sys.exit(1)
 296
 297             download_and_install_fzf()
 298             print("Updated fzf to {}".format(FZF_CURRENT_VERSION))
 299         else:
 300             print("fzf is the recommended version and does not need an update")
 301
 302     else:  # not fzf_bin:
 303         if not update:
 304             # Case 3b
 305             install = input("Could not detect fzf, install it now? [y/n]: ")
 306             if install.lower() != "y":
 307                 return
 308
 309         # Case 3a and 3b-fall-through
 310         download_and_install_fzf()
 311         fzf_bin = find_executable("fzf", fzf_path)
 312         print("Installed fzf to {}".format(fzf_path))
 313
 314     return fzf_bin
 315
 316
 317 def format_header():
 318     shortcuts = []
 319     for action, key in fzf_header_shortcuts:
 320         shortcuts.append(
 321             "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format(
 322                 t=terminal, action=action, key=key
 323             )
 324         )
 325     return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal)
 326
 327
 328 def run_fzf(cmd, tasks):
 329     env = dict(os.environ)
 330     env.update(
 331         {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])}
 332     )
 333     # Make sure fzf uses Windows' shell rather than MozillaBuild bash or
 334     # whatever our caller uses, since it doesn't quote the arguments properly
 335     # and thus windows paths like: C:\moz\foo end up as C:mozfoo...
 336     if platform.system() == "Windows":
 337         env["SHELL"] = env["COMSPEC"]
 338     proc = subprocess.Popen(
 339         cmd,
 340         stdout=subprocess.PIPE,
 341         stdin=subprocess.PIPE,
 342         env=env,
 343         universal_newlines=True,
 344     )
 345     out = proc.communicate("\n".join(tasks))[0].splitlines()
 346
 347     selected = []
 348     query = None
 349     if out:
 350         query = out[0]
 351         selected = out[1:]
 352     return query, selected
 353
 354
 355 def setup_tasks_for_fzf(
 356     push,
 357     parameters,
 358     full=False,
 359     disable_target_task_filter=False,
 360     show_estimates=True,
 361 ):
 362     check_working_directory(push)
 363     tg = generate_tasks(
 364         parameters, full=full, disable_target_task_filter=disable_target_task_filter
 365     )
 366     all_tasks = sorted(tg.tasks.keys())
 367
 368     # graph_Cache created by generate_tasks, recreate the path to that file.
 369     cache_dir = os.path.join(
 370         get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph"
 371     )
 372     if full:
 373         graph_cache = os.path.join(cache_dir, "full_task_graph")
 374         dep_cache = os.path.join(cache_dir, "full_task_dependencies")
 375         target_set = os.path.join(cache_dir, "full_task_set")
 376     else:
 377         graph_cache = os.path.join(cache_dir, "target_task_graph")
 378         dep_cache = os.path.join(cache_dir, "target_task_dependencies")
 379         target_set = os.path.join(cache_dir, "target_task_set")
 380
 381     if show_estimates:
 382         download_task_history_data(cache_dir=cache_dir)
 383         make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set)
 384
 385     if not full and not disable_target_task_filter:
 386         # Put all_tasks into a list because it's used multiple times, and "filter()"
 387         # returns a consumable iterator.
 388         all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks))
 389
 390     return all_tasks, dep_cache, cache_dir
 391
 392
 393 def build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=True):
 394     key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()]
 395     base_cmd = [
 396         fzf,
 397         "-m",
 398         "--bind",
 399         ",".join(key_shortcuts),
 400         "--header",
 401         format_header(),
 402         "--preview-window=right:30%",
 403         "--print-query",
 404     ]
 405
 406     if show_estimates:
 407         base_cmd.extend(
 408             [
 409                 "--preview",
 410                 '{} {} -g {} -s -c {} -t "{{+f}}"'.format(
 411                     sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir
 412                 ),
 413             ]
 414         )
 415     else:
 416         base_cmd.extend(
 417             [
 418                 "--preview",
 419                 '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT),
 420             ]
 421         )
 422
 423     return base_cmd