python/mozboot/bin/bootstrap.py

   1 #!/usr/bin/env python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4 # You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # This script provides one-line bootstrap support to configure systems to build
   7 # the tree. It does so by cloning the repo before calling directly into `mach
   8 # bootstrap`.
   9
  10 # Note that this script can't assume anything in particular about the host
  11 # Python environment (except that it's run with a sufficiently recent version of
  12 # Python 3), so we are restricted to stdlib modules.
  13
  14 import sys
  15
  16 major, minor = sys.version_info[:2]
  17 if (major < 3) or (major == 3 and minor < 7):
  18     print(
  19         "Bootstrap currently only runs on Python 3.7+."
  20         "Please try re-running with python3.7+."
  21     )
  22     sys.exit(1)
  23
  24 import ctypes
  25 import os
  26 import shutil
  27 import subprocess
  28 import tempfile
  29 from optparse import OptionParser
  30 from pathlib import Path
  31
  32 CLONE_MERCURIAL_PULL_FAIL = """
  33 Failed to pull from hg.mozilla.org.
  34
  35 This is most likely because of unstable network connection.
  36 Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually,
  37 or download a mercurial bundle and use it:
  38 https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html"""
  39
  40 WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys")
  41 VCS_HUMAN_READABLE = {
  42     "hg": "Mercurial",
  43     "git": "Git",
  44 }
  45
  46
  47 def which(name):
  48     """Python implementation of which.
  49
  50     It returns the path of an executable or None if it couldn't be found.
  51     """
  52     search_dirs = os.environ["PATH"].split(os.pathsep)
  53     potential_names = [name]
  54     if WINDOWS:
  55         potential_names.insert(0, name + ".exe")
  56
  57     for path in search_dirs:
  58         for executable_name in potential_names:
  59             test = Path(path) / executable_name
  60             if test.is_file() and os.access(test, os.X_OK):
  61                 return test
  62
  63     return None
  64
  65
  66 def validate_clone_dest(dest: Path):
  67     dest = dest.resolve()
  68
  69     if not dest.exists():
  70         return dest
  71
  72     if not dest.is_dir():
  73         print(f"ERROR! Destination {dest} exists but is not a directory.")
  74         return None
  75
  76     if not any(dest.iterdir()):
  77         return dest
  78     else:
  79         print(f"ERROR! Destination directory {dest} exists but is nonempty.")
  80         print(
  81             f"To re-bootstrap the existing checkout, go into '{dest}' and run './mach bootstrap'."
  82         )
  83         return None
  84
  85
  86 def input_clone_dest(vcs, no_interactive):
  87     repo_name = "mozilla-unified"
  88     print(f"Cloning into {repo_name} using {VCS_HUMAN_READABLE[vcs]}...")
  89     while True:
  90         dest = None
  91         if not no_interactive:
  92             dest = input(
  93                 f"Destination directory for clone (leave empty to use "
  94                 f"default destination of {repo_name}): "
  95             ).strip()
  96         if not dest:
  97             dest = repo_name
  98         dest = validate_clone_dest(Path(dest).expanduser())
  99         if dest:
 100             return dest
 101         if no_interactive:
 102             return None
 103
 104
 105 def hg_clone_firefox(hg: Path, dest: Path, head_repo, head_rev):
 106     # We create an empty repo then modify the config before adding data.
 107     # This is necessary to ensure storage settings are optimally
 108     # configured.
 109     args = [
 110         str(hg),
 111         # The unified repo is generaldelta, so ensure the client is as
 112         # well.
 113         "--config",
 114         "format.generaldelta=true",
 115         "init",
 116         str(dest),
 117     ]
 118     res = subprocess.call(args)
 119     if res:
 120         print("unable to create destination repo; please try cloning manually")
 121         return None
 122
 123     # Strictly speaking, this could overwrite a config based on a template
 124     # the user has installed. Let's pretend this problem doesn't exist
 125     # unless someone complains about it.
 126     with open(dest / ".hg" / "hgrc", "a") as fh:
 127         fh.write("[paths]\n")
 128         fh.write("default = https://hg.mozilla.org/mozilla-unified\n")
 129         fh.write("\n")
 130
 131         # The server uses aggressivemergedeltas which can blow up delta chain
 132         # length. This can cause performance to tank due to delta chains being
 133         # too long. Limit the delta chain length to something reasonable
 134         # to bound revlog read time.
 135         fh.write("[format]\n")
 136         fh.write("# This is necessary to keep performance in check\n")
 137         fh.write("maxchainlen = 10000\n")
 138
 139     # Pulling a specific revision into an empty repository induces a lot of
 140     # load on the Mercurial server, so we always pull from mozilla-unified (which,
 141     # when done from an empty repository, is equivalent to a clone), and then pull
 142     # the specific revision we want (if we want a specific one, otherwise we just
 143     # use the "central" bookmark), at which point it will be an incremental pull,
 144     # that the server can process more easily.
 145     # This is the same thing that robustcheckout does on automation.
 146     res = subprocess.call(
 147         [str(hg), "pull", "https://hg.mozilla.org/mozilla-unified"], cwd=str(dest)
 148     )
 149     if not res and head_repo:
 150         res = subprocess.call(
 151             [str(hg), "pull", head_repo, "-r", head_rev], cwd=str(dest)
 152         )
 153     print("")
 154     if res:
 155         print(CLONE_MERCURIAL_PULL_FAIL % dest)
 156         return None
 157
 158     head_rev = head_rev or "central"
 159     print(f'updating to "{head_rev}" - the development head of Gecko and Firefox')
 160     res = subprocess.call([str(hg), "update", "-r", head_rev], cwd=str(dest))
 161     if res:
 162         print(
 163             f"error updating; you will need to `cd {dest} && hg update -r central` "
 164             "manually"
 165         )
 166     return dest
 167
 168
 169 def git_clone_firefox(git: Path, dest: Path, watchman: Path, head_repo, head_rev):
 170     tempdir = None
 171     cinnabar = None
 172     env = dict(os.environ)
 173     try:
 174         cinnabar = which("git-cinnabar")
 175         if not cinnabar:
 176             from urllib.request import urlopen
 177
 178             cinnabar_url = "https://github.com/glandium/git-cinnabar/"
 179             # If git-cinnabar isn't installed already, that's fine; we can
 180             # download a temporary copy. `mach bootstrap` will install a copy
 181             # in the state dir; we don't want to copy all that logic to this
 182             # tiny bootstrapping script.
 183             tempdir = Path(tempfile.mkdtemp())
 184             with open(tempdir / "download.py", "wb") as fh:
 185                 shutil.copyfileobj(
 186                     urlopen(f"{cinnabar_url}/raw/master/download.py"), fh
 187                 )
 188
 189             subprocess.check_call(
 190                 [sys.executable, str(tempdir / "download.py")],
 191                 cwd=str(tempdir),
 192             )
 193             env["PATH"] = str(tempdir) + os.pathsep + env["PATH"]
 194             print(
 195                 "WARNING! git-cinnabar is required for Firefox development  "
 196                 "with git. After the clone is complete, the bootstrapper "
 197                 "will ask if you would like to configure git; answer yes, "
 198                 "and be sure to add git-cinnabar to your PATH according to "
 199                 "the bootstrapper output."
 200             )
 201
 202         # We're guaranteed to have `git-cinnabar` installed now.
 203         # Configure git per the git-cinnabar requirements.
 204         subprocess.check_call(
 205             [
 206                 str(git),
 207                 "clone",
 208                 "--no-checkout",
 209                 "hg::https://hg.mozilla.org/mozilla-unified",
 210                 str(dest),
 211             ],
 212             env=env,
 213         )
 214         subprocess.check_call(
 215             [str(git), "config", "fetch.prune", "true"], cwd=str(dest), env=env
 216         )
 217         subprocess.check_call(
 218             [str(git), "config", "pull.ff", "only"], cwd=str(dest), env=env
 219         )
 220
 221         if head_repo:
 222             subprocess.check_call(
 223                 [str(git), "cinnabar", "fetch", f"hg::{head_repo}", head_rev],
 224                 cwd=str(dest),
 225                 env=env,
 226             )
 227
 228         subprocess.check_call(
 229             [str(git), "checkout", "FETCH_HEAD" if head_rev else "bookmarks/central"],
 230             cwd=str(dest),
 231             env=env,
 232         )
 233
 234         watchman_sample = dest / ".git/hooks/fsmonitor-watchman.sample"
 235         # Older versions of git didn't include fsmonitor-watchman.sample.
 236         if watchman and watchman_sample.exists():
 237             print("Configuring watchman")
 238             watchman_config = dest / ".git/hooks/query-watchman"
 239             if not watchman_config.exists():
 240                 print(f"Copying {watchman_sample} to {watchman_config}")
 241                 copy_args = [
 242                     "cp",
 243                     ".git/hooks/fsmonitor-watchman.sample",
 244                     ".git/hooks/query-watchman",
 245                 ]
 246                 subprocess.check_call(copy_args, cwd=str(dest))
 247
 248             config_args = [
 249                 str(git),
 250                 "config",
 251                 "core.fsmonitor",
 252                 ".git/hooks/query-watchman",
 253             ]
 254             subprocess.check_call(config_args, cwd=str(dest), env=env)
 255         return dest
 256     finally:
 257         if tempdir:
 258             shutil.rmtree(str(tempdir))
 259
 260
 261 def add_microsoft_defender_antivirus_exclusions(dest, no_system_changes):
 262     if no_system_changes:
 263         return
 264
 265     if not WINDOWS:
 266         return
 267
 268     powershell_exe = which("powershell")
 269
 270     if not powershell_exe:
 271         return
 272
 273     def print_attempt_exclusion(path):
 274         print(
 275             f"Attempting to add exclusion path to Microsoft Defender Antivirus for: {path}"
 276         )
 277
 278     powershell_exe = str(powershell_exe)
 279     paths = []
 280
 281     # mozilla-unified / clone dest
 282     repo_dir = Path.cwd() / dest
 283     paths.append(repo_dir)
 284     print_attempt_exclusion(repo_dir)
 285
 286     # MOZILLABUILD
 287     mozillabuild_dir = os.getenv("MOZILLABUILD")
 288     if mozillabuild_dir:
 289         paths.append(mozillabuild_dir)
 290         print_attempt_exclusion(mozillabuild_dir)
 291
 292     # .mozbuild
 293     mozbuild_dir = Path.home() / ".mozbuild"
 294     paths.append(mozbuild_dir)
 295     print_attempt_exclusion(mozbuild_dir)
 296
 297     args = ";".join(f"Add-MpPreference -ExclusionPath '{path}'" for path in paths)
 298     command = f'-Command "{args}"'
 299
 300     # This will attempt to run as administrator by triggering a UAC prompt
 301     # for admin credentials. If "No" is selected, no exclusions are added.
 302     ctypes.windll.shell32.ShellExecuteW(None, "runas", powershell_exe, command, None, 0)
 303
 304
 305 def clone(options):
 306     vcs = options.vcs
 307     no_interactive = options.no_interactive
 308     no_system_changes = options.no_system_changes
 309
 310     if vcs == "hg":
 311         hg = which("hg")
 312         if not hg:
 313             print("Mercurial is not installed. Mercurial is required to clone Firefox.")
 314             try:
 315                 # We're going to recommend people install the Mercurial package with
 316                 # pip3. That will work if `pip3` installs binaries to a location
 317                 # that's in the PATH, but it might not be. To help out, if we CAN
 318                 # import "mercurial" (in which case it's already been installed),
 319                 # offer that as a solution.
 320                 import mercurial  # noqa: F401
 321
 322                 print(
 323                     "Hint: have you made sure that Mercurial is installed to a "
 324                     "location in your PATH?"
 325                 )
 326             except ImportError:
 327                 print("Try installing hg with `pip3 install Mercurial`.")
 328             return None
 329         binary = hg
 330     else:
 331         binary = which(vcs)
 332         if not binary:
 333             print("Git is not installed.")
 334             print("Try installing git using your system package manager.")
 335             return None
 336
 337     dest = input_clone_dest(vcs, no_interactive)
 338     if not dest:
 339         return None
 340
 341     add_microsoft_defender_antivirus_exclusions(dest, no_system_changes)
 342
 343     print(f"Cloning Firefox {VCS_HUMAN_READABLE[vcs]} repository to {dest}")
 344
 345     head_repo = os.environ.get("GECKO_HEAD_REPOSITORY")
 346     head_rev = os.environ.get("GECKO_HEAD_REV")
 347
 348     if vcs == "hg":
 349         return hg_clone_firefox(binary, dest, head_repo, head_rev)
 350     else:
 351         watchman = which("watchman")
 352         return git_clone_firefox(binary, dest, watchman, head_repo, head_rev)
 353
 354
 355 def bootstrap(srcdir: Path, application_choice, no_interactive, no_system_changes):
 356     args = [sys.executable, "mach"]
 357
 358     if no_interactive:
 359         # --no-interactive is a global argument, not a command argument,
 360         # so it needs to be specified before "bootstrap" is appended.
 361         args += ["--no-interactive"]
 362
 363     args += ["bootstrap"]
 364
 365     if application_choice:
 366         args += ["--application-choice", application_choice]
 367     if no_system_changes:
 368         args += ["--no-system-changes"]
 369
 370     print("Running `%s`" % " ".join(args))
 371     return subprocess.call(args, cwd=str(srcdir))
 372
 373
 374 def main(args):
 375     parser = OptionParser()
 376     parser.add_option(
 377         "--application-choice",
 378         dest="application_choice",
 379         help='Pass in an application choice (see "APPLICATIONS" in '
 380         "python/mozboot/mozboot/bootstrap.py) instead of using the "
 381         "default interactive prompt.",
 382     )
 383     parser.add_option(
 384         "--vcs",
 385         dest="vcs",
 386         default="hg",
 387         choices=["git", "hg"],
 388         help="VCS (hg or git) to use for downloading the source code, "
 389         "instead of using the default interactive prompt.",
 390     )
 391     parser.add_option(
 392         "--no-interactive",
 393         dest="no_interactive",
 394         action="store_true",
 395         help="Answer yes to any (Y/n) interactive prompts.",
 396     )
 397     parser.add_option(
 398         "--no-system-changes",
 399         dest="no_system_changes",
 400         action="store_true",
 401         help="Only executes actions that leave the system " "configuration alone.",
 402     )
 403
 404     options, leftover = parser.parse_args(args)
 405     try:
 406         srcdir = clone(options)
 407         if not srcdir:
 408             return 1
 409         print("Clone complete.")
 410         print(
 411             "If you need to run the tooling bootstrapping again, "
 412             "then consider running './mach bootstrap' instead."
 413         )
 414         if not options.no_interactive:
 415             remove_bootstrap_file = input(
 416                 "Unless you are going to have more local copies of Firefox source code, "
 417                 "this 'bootstrap.py' file is no longer needed and can be deleted. "
 418                 "Clean up the bootstrap.py file? (Y/n)"
 419             )
 420             if not remove_bootstrap_file:
 421                 remove_bootstrap_file = "y"
 422         if options.no_interactive or remove_bootstrap_file == "y":
 423             try:
 424                 Path(sys.argv[0]).unlink()
 425             except FileNotFoundError:
 426                 print("File could not be found !")
 427         return bootstrap(
 428             srcdir,
 429             options.application_choice,
 430             options.no_interactive,
 431             options.no_system_changes,
 432         )
 433     except Exception:
 434         print("Could not bootstrap Firefox! Consider filing a bug.")
 435         raise
 436
 437
 438 if __name__ == "__main__":
 439     sys.exit(main(sys.argv))