build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from pathlib import Path
  10
  11 if sys.version_info[0] < 3:
  12     import __builtin__ as builtins
  13
  14     class MetaPathFinder(object):
  15         pass
  16
  17
  18 else:
  19     from importlib.abc import MetaPathFinder
  20
  21 from types import ModuleType
  22
  23 STATE_DIR_FIRST_RUN = """
  24 Mach and the build system store shared state in a common directory
  25 on the filesystem. The following directory will be created:
  26
  27   {}
  28
  29 If you would like to use a different directory, rename or move it to your
  30 desired location, and set the MOZBUILD_STATE_PATH environment variable
  31 accordingly.
  32 """.strip()
  33
  34
  35 CATEGORIES = {
  36     "build": {
  37         "short": "Build Commands",
  38         "long": "Interact with the build system",
  39         "priority": 80,
  40     },
  41     "post-build": {
  42         "short": "Post-build Commands",
  43         "long": "Common actions performed after completing a build.",
  44         "priority": 70,
  45     },
  46     "testing": {
  47         "short": "Testing",
  48         "long": "Run tests.",
  49         "priority": 60,
  50     },
  51     "ci": {
  52         "short": "CI",
  53         "long": "Taskcluster commands",
  54         "priority": 59,
  55     },
  56     "devenv": {
  57         "short": "Development Environment",
  58         "long": "Set up and configure your development environment.",
  59         "priority": 50,
  60     },
  61     "build-dev": {
  62         "short": "Low-level Build System Interaction",
  63         "long": "Interact with specific parts of the build system.",
  64         "priority": 20,
  65     },
  66     "misc": {
  67         "short": "Potpourri",
  68         "long": "Potent potables and assorted snacks.",
  69         "priority": 10,
  70     },
  71     "release": {
  72         "short": "Release automation",
  73         "long": "Commands for used in release automation.",
  74         "priority": 5,
  75     },
  76     "disabled": {
  77         "short": "Disabled",
  78         "long": "The disabled commands are hidden by default. Use -v to display them. "
  79         "These commands are unavailable for your current context, "
  80         'run "mach <command>" to see why.',
  81         "priority": 0,
  82     },
  83 }
  84
  85
  86 def _activate_python_environment(topsrcdir, get_state_dir):
  87     from mach.site import MachSiteManager
  88
  89     mach_environment = MachSiteManager.from_environment(
  90         topsrcdir,
  91         get_state_dir,
  92     )
  93     mach_environment.activate()
  94
  95
  96 def _maybe_activate_mozillabuild_environment():
  97     if sys.platform != "win32":
  98         return
  99
 100     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 101     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 102     assert mozillabuild.exists(), (
 103         f'MozillaBuild was not found at "{mozillabuild}".\n'
 104         "If it's installed in a different location, please "
 105         'set the "MOZILLABUILD" environment variable '
 106         "accordingly."
 107     )
 108
 109     use_msys2 = (mozillabuild / "msys2").exists()
 110     if use_msys2:
 111         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 112     else:
 113         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 114
 115     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 116     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 117     for new_path in paths_to_add:
 118         if new_path not in existing_paths:
 119             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 120
 121
 122 def initialize(topsrcdir):
 123     # This directory was deleted in bug 1666345, but there may be some ignored
 124     # files here. We can safely just delete it for the user so they don't have
 125     # to clean the repo themselves.
 126     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 127     if os.path.exists(deleted_dir):
 128         shutil.rmtree(deleted_dir, ignore_errors=True)
 129
 130     # We need the "mach" module to access the logic to parse virtualenv
 131     # requirements. Since that depends on "packaging" (and, transitively,
 132     # "pyparsing"), we add those to the path too.
 133     sys.path[0:0] = [
 134         os.path.join(topsrcdir, module)
 135         for module in (
 136             os.path.join("python", "mach"),
 137             os.path.join("third_party", "python", "packaging"),
 138             os.path.join("third_party", "python", "pyparsing"),
 139         )
 140     ]
 141
 142     from mach.util import get_state_dir, setenv
 143
 144     state_dir = _create_state_dir()
 145
 146     # normpath state_dir to normalize msys-style slashes.
 147     _activate_python_environment(
 148         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 149     )
 150     _maybe_activate_mozillabuild_environment()
 151
 152     import mach.base
 153     import mach.main
 154     from mach.main import MachCommandReference
 155
 156     # Centralized registry of available mach commands
 157     MACH_COMMANDS = {
 158         "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"),
 159         "devtools-css-db": MachCommandReference(
 160             "devtools/shared/css/generated/mach_commands.py"
 161         ),
 162         "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"),
 163         "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"),
 164         "hazards": MachCommandReference(
 165             "js/src/devtools/rootAnalysis/mach_commands.py"
 166         ),
 167         "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 168         "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 169         "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 170         "android": MachCommandReference("mobile/android/mach_commands.py"),
 171         "gradle": MachCommandReference("mobile/android/mach_commands.py"),
 172         "gradle-install": MachCommandReference("mobile/android/mach_commands.py"),
 173         "mach-commands": MachCommandReference(
 174             "python/mach/mach/commands/commandinfo.py"
 175         ),
 176         "mach-debug-commands": MachCommandReference(
 177             "python/mach/mach/commands/commandinfo.py"
 178         ),
 179         "mach-completion": MachCommandReference(
 180             "python/mach/mach/commands/commandinfo.py"
 181         ),
 182         "settings": MachCommandReference("python/mach/mach/commands/settings.py"),
 183         "python": MachCommandReference("python/mach_commands.py"),
 184         "python-test": MachCommandReference("python/mach_commands.py"),
 185         "bootstrap": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 186         "vcs-setup": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 187         "artifact": MachCommandReference(
 188             "python/mozbuild/mozbuild/artifact_commands.py"
 189         ),
 190         "ide": MachCommandReference(
 191             "python/mozbuild/mozbuild/backend/mach_commands.py"
 192         ),
 193         "build": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 194         "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 195         "resource-usage": MachCommandReference(
 196             "python/mozbuild/mozbuild/build_commands.py"
 197         ),
 198         "build-backend": MachCommandReference(
 199             "python/mozbuild/mozbuild/build_commands.py"
 200         ),
 201         "clang-tidy": MachCommandReference(
 202             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 203         ),
 204         "static-analysis": MachCommandReference(
 205             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 206         ),
 207         "prettier-format": MachCommandReference(
 208             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 209         ),
 210         "clang-format": MachCommandReference(
 211             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 212         ),
 213         "compileflags": MachCommandReference(
 214             "python/mozbuild/mozbuild/compilation/codecomplete.py"
 215         ),
 216         "mozbuild-reference": MachCommandReference(
 217             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 218         ),
 219         "file-info": MachCommandReference(
 220             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 221         ),
 222         "vendor": MachCommandReference(
 223             "python/mozbuild/mozbuild/vendor/mach_commands.py"
 224         ),
 225         "watch": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 226         "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 227         "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 228         "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 229         "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 230         "warnings-summary": MachCommandReference(
 231             "python/mozbuild/mozbuild/mach_commands.py"
 232         ),
 233         "warnings-list": MachCommandReference(
 234             "python/mozbuild/mozbuild/mach_commands.py"
 235         ),
 236         "gtest": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 237         "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 238         "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 239         "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 240         "buildsymbols": MachCommandReference(
 241             "python/mozbuild/mozbuild/mach_commands.py"
 242         ),
 243         "environment": MachCommandReference(
 244             "python/mozbuild/mozbuild/mach_commands.py"
 245         ),
 246         "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 247         "package-multi-locale": MachCommandReference(
 248             "python/mozbuild/mozbuild/mach_commands.py"
 249         ),
 250         "perftest": MachCommandReference(
 251             "python/mozperftest/mozperftest/mach_commands.py"
 252         ),
 253         "perftest-test": MachCommandReference(
 254             "python/mozperftest/mozperftest/mach_commands.py"
 255         ),
 256         "release": MachCommandReference(
 257             "python/mozrelease/mozrelease/mach_commands.py"
 258         ),
 259         "remote": MachCommandReference("remote/mach_commands.py"),
 260         "puppeteer-test": MachCommandReference("remote/mach_commands.py"),
 261         "generate-test-certs": MachCommandReference(
 262             "security/manager/tools/mach_commands.py"
 263         ),
 264         "taskgraph": MachCommandReference("taskcluster/mach_commands.py"),
 265         "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"),
 266         "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"),
 267         "taskcluster-image-digest": MachCommandReference(
 268             "taskcluster/mach_commands.py"
 269         ),
 270         "release-history": MachCommandReference("taskcluster/mach_commands.py"),
 271         "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"),
 272         "fetch-condprofile": MachCommandReference(
 273             "testing/condprofile/mach_commands.py"
 274         ),
 275         "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
 276         "firefox-ui-functional": MachCommandReference(
 277             "testing/firefox-ui/mach_commands.py"
 278         ),
 279         "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"),
 280         "addtest": MachCommandReference("testing/mach_commands.py"),
 281         "test": MachCommandReference("testing/mach_commands.py"),
 282         "cppunittest": MachCommandReference("testing/mach_commands.py"),
 283         "jstests": MachCommandReference("testing/mach_commands.py"),
 284         "jit-test": MachCommandReference("testing/mach_commands.py"),
 285         "jsapi-tests": MachCommandReference("testing/mach_commands.py"),
 286         "jsshell-bench": MachCommandReference("testing/mach_commands.py"),
 287         "cramtest": MachCommandReference("testing/mach_commands.py"),
 288         "test-info": MachCommandReference("testing/mach_commands.py"),
 289         "rusttests": MachCommandReference("testing/mach_commands.py"),
 290         "fluent-migration-test": MachCommandReference("testing/mach_commands.py"),
 291         "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
 292         "mochitest": MachCommandReference("testing/mochitest/mach_commands.py"),
 293         "geckoview-junit": MachCommandReference("testing/mochitest/mach_commands.py"),
 294         "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"),
 295         "raptor": MachCommandReference("testing/raptor/mach_commands.py"),
 296         "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"),
 297         "talos-test": MachCommandReference("testing/talos/mach_commands.py"),
 298         "tps-build": MachCommandReference("testing/tps/mach_commands.py"),
 299         "web-platform-tests": MachCommandReference(
 300             "testing/web-platform/mach_commands.py"
 301         ),
 302         "wpt": MachCommandReference("testing/web-platform/mach_commands.py"),
 303         "web-platform-tests-update": MachCommandReference(
 304             "testing/web-platform/mach_commands.py"
 305         ),
 306         "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"),
 307         "wpt-manifest-update": MachCommandReference(
 308             "testing/web-platform/mach_commands.py"
 309         ),
 310         "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"),
 311         "wpt-metadata-summary": MachCommandReference(
 312             "testing/web-platform/mach_commands.py"
 313         ),
 314         "wpt-metadata-merge": MachCommandReference(
 315             "testing/web-platform/mach_commands.py"
 316         ),
 317         "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"),
 318         "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"),
 319         "wpt-fission-regressions": MachCommandReference(
 320             "testing/web-platform/mach_commands.py"
 321         ),
 322         "xpcshell-test": MachCommandReference("testing/xpcshell/mach_commands.py"),
 323         "telemetry-tests-client": MachCommandReference(
 324             "toolkit/components/telemetry/tests/marionette/mach_commands.py"
 325         ),
 326         "data-review": MachCommandReference(
 327             "toolkit/components/glean/build_scripts/mach_commands.py"
 328         ),
 329         "perf-data-review": MachCommandReference(
 330             "toolkit/components/glean/build_scripts/mach_commands.py"
 331         ),
 332         "update-glean-tags": MachCommandReference(
 333             "toolkit/components/glean/build_scripts/mach_commands.py"
 334         ),
 335         "update-glean": MachCommandReference(
 336             "toolkit/components/glean/build_scripts/mach_commands.py"
 337         ),
 338         "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"),
 339         "compare-locales": MachCommandReference(
 340             "tools/compare-locales/mach_commands.py"
 341         ),
 342         "l10n-cross-channel": MachCommandReference("tools/lint/mach_commands.py"),
 343         "busted": MachCommandReference("tools/mach_commands.py"),
 344         "pastebin": MachCommandReference("tools/mach_commands.py"),
 345         "mozregression": MachCommandReference("tools/mach_commands.py"),
 346         "node": MachCommandReference("tools/mach_commands.py"),
 347         "npm": MachCommandReference("tools/mach_commands.py"),
 348         "logspam": MachCommandReference("tools/mach_commands.py"),
 349         "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
 350         "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"),
 351         "power": MachCommandReference("tools/power/mach_commands.py"),
 352         "try": MachCommandReference("tools/tryselect/mach_commands.py"),
 353         "import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
 354         "test-interventions": MachCommandReference(
 355             "testing/webcompat/mach_commands.py"
 356         ),
 357         "esmify": MachCommandReference("tools/esmify/mach_commands.py"),
 358         "xpcshell": MachCommandReference("js/xpconnect/mach_commands.py"),
 359         "uniffi": MachCommandReference(
 360             "toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py"
 361         ),
 362         "storybook": MachCommandReference(
 363             "browser/components/storybook/mach_commands.py"
 364         ),
 365         "widgets": MachCommandReference("toolkit/content/widgets/mach_commands.py"),
 366     }
 367
 368     # Set a reasonable limit to the number of open files.
 369     #
 370     # Some linux systems set `ulimit -n` to a very high number, which works
 371     # well for systems that run servers, but this setting causes performance
 372     # problems when programs close file descriptors before forking, like
 373     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 374     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 375     # same thing when spawning processes.  We would prefer to lower this limit
 376     # to avoid such performance problems; processes spawned by `mach` will
 377     # inherit the limit set here.
 378     #
 379     # The Firefox build defaults the soft limit to 1024, except for builds that
 380     # do LTO, where the soft limit is 8192.  We're going to default to the
 381     # latter, since people do occasionally do LTO builds on their local
 382     # machines, and requiring them to discover another magical setting after
 383     # setting up an LTO build in the first place doesn't seem good.
 384     #
 385     # This code mimics the code in taskcluster/scripts/run-task.
 386     try:
 387         import resource
 388
 389         # Keep the hard limit the same, though, allowing processes to change
 390         # their soft limit if they need to (Firefox does, for instance).
 391         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 392         # Permit people to override our default limit if necessary via
 393         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 394         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 395         if limit:
 396             limit = int(limit)
 397         else:
 398             # If no explicit limit is given, use our default if it's less than
 399             # the current soft limit.  For instance, the default on macOS is
 400             # 256, so we'd pick that rather than our default.
 401             limit = min(soft, 8192)
 402         # Now apply the limit, if it's different from the original one.
 403         if limit != soft:
 404             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 405     except ImportError:
 406         # The resource module is UNIX only.
 407         pass
 408
 409     def resolve_repository():
 410         import mozversioncontrol
 411
 412         try:
 413             # This API doesn't respect the vcs binary choices from configure.
 414             # If we ever need to use the VCS binary here, consider something
 415             # more robust.
 416             return mozversioncontrol.get_repository_object(path=topsrcdir)
 417         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 418             return None
 419
 420     def pre_dispatch_handler(context, handler, args):
 421         # If --disable-tests flag was enabled in the mozconfig used to compile
 422         # the build, tests will be disabled. Instead of trying to run
 423         # nonexistent tests then reporting a failure, this will prevent mach
 424         # from progressing beyond this point.
 425         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 426             from mozbuild.base import BuildEnvironmentNotFoundException
 427
 428             try:
 429                 from mozbuild.base import MozbuildObject
 430
 431                 # all environments should have an instance of build object.
 432                 build = MozbuildObject.from_environment()
 433                 if build is not None and not getattr(
 434                     build, "substs", {"ENABLE_TESTS": True}
 435                 ).get("ENABLE_TESTS"):
 436                     print(
 437                         "Tests have been disabled with --disable-tests.\n"
 438                         + "Remove the flag, and re-compile to enable tests."
 439                     )
 440                     sys.exit(1)
 441             except BuildEnvironmentNotFoundException:
 442                 # likely automation environment, so do nothing.
 443                 pass
 444
 445     def post_dispatch_handler(
 446         context, handler, instance, success, start_time, end_time, depth, args
 447     ):
 448         """Perform global operations after command dispatch.
 449
 450
 451         For now,  we will use this to handle build system telemetry.
 452         """
 453
 454         # Don't finalize telemetry data if this mach command was invoked as part of
 455         # another mach command.
 456         if depth != 1:
 457             return
 458
 459         _finalize_telemetry_glean(
 460             context.telemetry, handler.name == "bootstrap", success
 461         )
 462
 463     def populate_context(key=None):
 464         if key is None:
 465             return
 466         if key == "state_dir":
 467             return state_dir
 468
 469         if key == "local_state_dir":
 470             return get_state_dir(specific_to_topsrcdir=True)
 471
 472         if key == "topdir":
 473             return topsrcdir
 474
 475         if key == "pre_dispatch_handler":
 476             return pre_dispatch_handler
 477
 478         if key == "post_dispatch_handler":
 479             return post_dispatch_handler
 480
 481         if key == "repository":
 482             return resolve_repository()
 483
 484         raise AttributeError(key)
 485
 486     # Note which process is top-level so that recursive mach invocations can avoid writing
 487     # telemetry data.
 488     if "MACH_MAIN_PID" not in os.environ:
 489         setenv("MACH_MAIN_PID", str(os.getpid()))
 490
 491     driver = mach.main.Mach(os.getcwd())
 492     driver.populate_context_handler = populate_context
 493
 494     if not driver.settings_paths:
 495         # default global machrc location
 496         driver.settings_paths.append(state_dir)
 497     # always load local repository configuration
 498     driver.settings_paths.append(topsrcdir)
 499
 500     for category, meta in CATEGORIES.items():
 501         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 502
 503     # Sparse checkouts may not have all mach_commands.py files. Ignore
 504     # errors from missing files. Same for spidermonkey tarballs.
 505     repo = resolve_repository()
 506     missing_ok = (
 507         repo is not None and repo.sparse_checkout_present()
 508     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 509
 510     driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
 511
 512     return driver
 513
 514
 515 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 516     """Submit telemetry collected by Glean.
 517
 518     Finalizes some metrics (command success state and duration, system information) and
 519     requests Glean to send the collected data.
 520     """
 521
 522     from mach.telemetry import MACH_METRICS_PATH
 523     from mozbuild.telemetry import (
 524         get_cpu_brand,
 525         get_distro_and_version,
 526         get_psutil_stats,
 527         get_shell_info,
 528         get_vscode_running,
 529     )
 530
 531     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 532     mach_metrics.mach.duration.stop()
 533     mach_metrics.mach.success.set(success)
 534     system_metrics = mach_metrics.mach.system
 535     cpu_brand = get_cpu_brand()
 536     if cpu_brand:
 537         system_metrics.cpu_brand.set(cpu_brand)
 538     distro, version = get_distro_and_version()
 539     system_metrics.distro.set(distro)
 540     system_metrics.distro_version.set(version)
 541
 542     vscode_terminal, ssh_connection = get_shell_info()
 543     system_metrics.vscode_terminal.set(vscode_terminal)
 544     system_metrics.ssh_connection.set(ssh_connection)
 545     system_metrics.vscode_running.set(get_vscode_running())
 546
 547     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 548     if has_psutil:
 549         # psutil may not be available (we may not have been able to download
 550         # a wheel or build it from source).
 551         system_metrics.logical_cores.add(logical_cores)
 552         if physical_cores is not None:
 553             system_metrics.physical_cores.add(physical_cores)
 554         if memory_total is not None:
 555             system_metrics.memory.accumulate(
 556                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 557             )
 558     telemetry.submit(is_bootstrap)
 559
 560
 561 def _create_state_dir():
 562     # Global build system and mach state is stored in a central directory. By
 563     # default, this is ~/.mozbuild. However, it can be defined via an
 564     # environment variable. We detect first run (by lack of this directory
 565     # existing) and notify the user that it will be created. The logic for
 566     # creation is much simpler for the "advanced" environment variable use
 567     # case. For default behavior, we educate users and give them an opportunity
 568     # to react.
 569     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 570     if state_dir:
 571         if not os.path.exists(state_dir):
 572             print(
 573                 "Creating global state directory from environment variable: {}".format(
 574                     state_dir
 575                 )
 576             )
 577     else:
 578         state_dir = os.path.expanduser("~/.mozbuild")
 579         if not os.path.exists(state_dir):
 580             if not os.environ.get("MOZ_AUTOMATION"):
 581                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 582
 583             print("Creating default state directory: {}".format(state_dir))
 584
 585     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 586     return state_dir
 587
 588
 589 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 590 # the source directory are essentially ignored. See further below for details
 591 # and caveats.
 592 # Objdirs outside the source directory are ignored because in most cases, if
 593 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 594 class ImportHook(object):
 595     def __init__(self, original_import):
 596         self._original_import = original_import
 597         # Assume the source directory is the parent directory of the one
 598         # containing this file.
 599         self._source_dir = (
 600             os.path.normcase(
 601                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 602             )
 603             + os.sep
 604         )
 605         self._modules = set()
 606
 607     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 608         if sys.version_info[0] >= 3 and level < 0:
 609             level = 0
 610
 611         # name might be a relative import. Instead of figuring out what that
 612         # resolves to, which is complex, just rely on the real import.
 613         # Since we don't know the full module name, we can't check sys.modules,
 614         # so we need to keep track of which modules we've already seen to avoid
 615         # to stat() them again when they are imported multiple times.
 616         module = self._original_import(name, globals, locals, fromlist, level)
 617
 618         # Some tests replace modules in sys.modules with non-module instances.
 619         if not isinstance(module, ModuleType):
 620             return module
 621
 622         resolved_name = module.__name__
 623         if resolved_name in self._modules:
 624             return module
 625         self._modules.add(resolved_name)
 626
 627         # Builtin modules don't have a __file__ attribute.
 628         if not getattr(module, "__file__", None):
 629             return module
 630
 631         # Note: module.__file__ is not always absolute.
 632         path = os.path.normcase(os.path.abspath(module.__file__))
 633         # Note: we could avoid normcase and abspath above for non pyc/pyo
 634         # files, but those are actually rare, so it doesn't really matter.
 635         if not path.endswith((".pyc", ".pyo")):
 636             return module
 637
 638         # Ignore modules outside our source directory
 639         if not path.startswith(self._source_dir):
 640             return module
 641
 642         # If there is no .py corresponding to the .pyc/.pyo module we're
 643         # loading, remove the .pyc/.pyo file, and reload the module.
 644         # Since we already loaded the .pyc/.pyo module, if it had side
 645         # effects, they will have happened already, and loading the module
 646         # with the same name, from another directory may have the same side
 647         # effects (or different ones). We assume it's not a problem for the
 648         # python modules under our source directory (either because it
 649         # doesn't happen or because it doesn't matter).
 650         if not os.path.exists(module.__file__[:-1]):
 651             if os.path.exists(module.__file__):
 652                 os.remove(module.__file__)
 653             del sys.modules[module.__name__]
 654             module = self(name, globals, locals, fromlist, level)
 655
 656         return module
 657
 658
 659 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 660 # the source directory are essentially ignored. See further below for details
 661 # and caveats.
 662 # Objdirs outside the source directory are ignored because in most cases, if
 663 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 664 class FinderHook(MetaPathFinder):
 665     def __init__(self, klass):
 666         # Assume the source directory is the parent directory of the one
 667         # containing this file.
 668         self._source_dir = (
 669             os.path.normcase(
 670                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 671             )
 672             + os.sep
 673         )
 674         self.finder_class = klass
 675
 676     def find_spec(self, full_name, paths=None, target=None):
 677         spec = self.finder_class.find_spec(full_name, paths, target)
 678
 679         # Some modules don't have an origin.
 680         if spec is None or spec.origin is None:
 681             return spec
 682
 683         # Normalize the origin path.
 684         path = os.path.normcase(os.path.abspath(spec.origin))
 685         # Note: we could avoid normcase and abspath above for non pyc/pyo
 686         # files, but those are actually rare, so it doesn't really matter.
 687         if not path.endswith((".pyc", ".pyo")):
 688             return spec
 689
 690         # Ignore modules outside our source directory
 691         if not path.startswith(self._source_dir):
 692             return spec
 693
 694         # If there is no .py corresponding to the .pyc/.pyo module we're
 695         # resolving, remove the .pyc/.pyo file, and try again.
 696         if not os.path.exists(spec.origin[:-1]):
 697             if os.path.exists(spec.origin):
 698                 os.remove(spec.origin)
 699             spec = self.finder_class.find_spec(full_name, paths, target)
 700
 701         return spec
 702
 703
 704 # Additional hook for python >= 3.8's importlib.metadata.
 705 class MetadataHook(FinderHook):
 706     def find_distributions(self, *args, **kwargs):
 707         return self.finder_class.find_distributions(*args, **kwargs)
 708
 709
 710 def hook(finder):
 711     has_find_spec = hasattr(finder, "find_spec")
 712     has_find_distributions = hasattr(finder, "find_distributions")
 713     if has_find_spec and has_find_distributions:
 714         return MetadataHook(finder)
 715     elif has_find_spec:
 716         return FinderHook(finder)
 717     return finder
 718
 719
 720 # Install our hook. This can be deleted when the Python 3 migration is complete.
 721 if sys.version_info[0] < 3:
 722     builtins.__import__ = ImportHook(builtins.__import__)
 723 else:
 724     sys.meta_path = [hook(c) for c in sys.meta_path]