build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import math
   8 import os
   9 import shutil
  10 import sys
  11 from pathlib import Path
  12
  13 if sys.version_info[0] < 3:
  14     import __builtin__ as builtins
  15
  16     class MetaPathFinder(object):
  17         pass
  18
  19
  20 else:
  21     from importlib.abc import MetaPathFinder
  22
  23
  24 from types import ModuleType
  25
  26
  27 STATE_DIR_FIRST_RUN = """
  28 Mach and the build system store shared state in a common directory
  29 on the filesystem. The following directory will be created:
  30
  31   {}
  32
  33 If you would like to use a different directory, hit CTRL+c, set the
  34 MOZBUILD_STATE_PATH environment variable to the directory you'd like to
  35 use, and run Mach again.
  36
  37 Press ENTER/RETURN to continue or CTRL+c to abort.
  38 """.strip()
  39
  40
  41 CATEGORIES = {
  42     "build": {
  43         "short": "Build Commands",
  44         "long": "Interact with the build system",
  45         "priority": 80,
  46     },
  47     "post-build": {
  48         "short": "Post-build Commands",
  49         "long": "Common actions performed after completing a build.",
  50         "priority": 70,
  51     },
  52     "testing": {
  53         "short": "Testing",
  54         "long": "Run tests.",
  55         "priority": 60,
  56     },
  57     "ci": {
  58         "short": "CI",
  59         "long": "Taskcluster commands",
  60         "priority": 59,
  61     },
  62     "devenv": {
  63         "short": "Development Environment",
  64         "long": "Set up and configure your development environment.",
  65         "priority": 50,
  66     },
  67     "build-dev": {
  68         "short": "Low-level Build System Interaction",
  69         "long": "Interact with specific parts of the build system.",
  70         "priority": 20,
  71     },
  72     "misc": {
  73         "short": "Potpourri",
  74         "long": "Potent potables and assorted snacks.",
  75         "priority": 10,
  76     },
  77     "release": {
  78         "short": "Release automation",
  79         "long": "Commands for used in release automation.",
  80         "priority": 5,
  81     },
  82     "disabled": {
  83         "short": "Disabled",
  84         "long": "The disabled commands are hidden by default. Use -v to display them. "
  85         "These commands are unavailable for your current context, "
  86         'run "mach <command>" to see why.',
  87         "priority": 0,
  88     },
  89 }
  90
  91
  92 def _activate_python_environment(topsrcdir, get_state_dir):
  93     from mach.site import MachSiteManager
  94
  95     mach_environment = MachSiteManager.from_environment(
  96         topsrcdir,
  97         get_state_dir,
  98     )
  99     mach_environment.activate()
 100
 101
 102 def _maybe_activate_mozillabuild_environment():
 103     if sys.platform != "win32":
 104         return
 105
 106     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 107     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 108     assert mozillabuild.exists(), (
 109         f'MozillaBuild was not found at "{mozillabuild}".\n'
 110         "If it's installed in a different location, please "
 111         'set the "MOZILLABUILD" environment variable '
 112         "accordingly."
 113     )
 114
 115     use_msys2 = (mozillabuild / "msys2").exists()
 116     if use_msys2:
 117         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 118     else:
 119         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 120
 121     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 122     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 123     for new_path in paths_to_add:
 124         if new_path not in existing_paths:
 125             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 126
 127
 128 def initialize(topsrcdir):
 129     # This directory was deleted in bug 1666345, but there may be some ignored
 130     # files here. We can safely just delete it for the user so they don't have
 131     # to clean the repo themselves.
 132     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 133     if os.path.exists(deleted_dir):
 134         shutil.rmtree(deleted_dir, ignore_errors=True)
 135
 136     # We need the "mach" module to access the logic to parse virtualenv
 137     # requirements. Since that depends on "packaging" (and, transitively,
 138     # "pyparsing"), we add those to the path too.
 139     sys.path[0:0] = [
 140         os.path.join(topsrcdir, module)
 141         for module in (
 142             os.path.join("python", "mach"),
 143             os.path.join("third_party", "python", "packaging"),
 144             os.path.join("third_party", "python", "pyparsing"),
 145         )
 146     ]
 147
 148     from mach.util import setenv, get_state_dir
 149
 150     state_dir = _create_state_dir()
 151
 152     # normpath state_dir to normalize msys-style slashes.
 153     _activate_python_environment(
 154         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 155     )
 156     _maybe_activate_mozillabuild_environment()
 157
 158     import mach.base
 159     import mach.main
 160
 161     from mach.main import MachCommandReference
 162
 163     # Centralized registry of available mach commands
 164     MACH_COMMANDS = {
 165         "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"),
 166         "devtools-css-db": MachCommandReference(
 167             "devtools/shared/css/generated/mach_commands.py"
 168         ),
 169         "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"),
 170         "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"),
 171         "hazards": MachCommandReference(
 172             "js/src/devtools/rootAnalysis/mach_commands.py"
 173         ),
 174         "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 175         "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 176         "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 177         "android": MachCommandReference("mobile/android/mach_commands.py"),
 178         "gradle": MachCommandReference("mobile/android/mach_commands.py"),
 179         "gradle-install": MachCommandReference("mobile/android/mach_commands.py"),
 180         "mach-commands": MachCommandReference(
 181             "python/mach/mach/commands/commandinfo.py"
 182         ),
 183         "mach-debug-commands": MachCommandReference(
 184             "python/mach/mach/commands/commandinfo.py"
 185         ),
 186         "mach-completion": MachCommandReference(
 187             "python/mach/mach/commands/commandinfo.py"
 188         ),
 189         "settings": MachCommandReference("python/mach/mach/commands/settings.py"),
 190         "python": MachCommandReference("python/mach_commands.py"),
 191         "python-test": MachCommandReference("python/mach_commands.py"),
 192         "bootstrap": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 193         "vcs-setup": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 194         "artifact": MachCommandReference(
 195             "python/mozbuild/mozbuild/artifact_commands.py"
 196         ),
 197         "ide": MachCommandReference(
 198             "python/mozbuild/mozbuild/backend/mach_commands.py"
 199         ),
 200         "build": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 201         "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 202         "resource-usage": MachCommandReference(
 203             "python/mozbuild/mozbuild/build_commands.py"
 204         ),
 205         "build-backend": MachCommandReference(
 206             "python/mozbuild/mozbuild/build_commands.py"
 207         ),
 208         "clang-tidy": MachCommandReference(
 209             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 210         ),
 211         "static-analysis": MachCommandReference(
 212             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 213         ),
 214         "prettier-format": MachCommandReference(
 215             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 216         ),
 217         "clang-format": MachCommandReference(
 218             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 219         ),
 220         "compileflags": MachCommandReference(
 221             "python/mozbuild/mozbuild/compilation/codecomplete.py"
 222         ),
 223         "mozbuild-reference": MachCommandReference(
 224             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 225         ),
 226         "file-info": MachCommandReference(
 227             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 228         ),
 229         "vendor": MachCommandReference(
 230             "python/mozbuild/mozbuild/vendor/mach_commands.py"
 231         ),
 232         "watch": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 233         "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 234         "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 235         "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 236         "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 237         "warnings-summary": MachCommandReference(
 238             "python/mozbuild/mozbuild/mach_commands.py"
 239         ),
 240         "warnings-list": MachCommandReference(
 241             "python/mozbuild/mozbuild/mach_commands.py"
 242         ),
 243         "gtest": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 244         "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 245         "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 246         "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 247         "buildsymbols": MachCommandReference(
 248             "python/mozbuild/mozbuild/mach_commands.py"
 249         ),
 250         "environment": MachCommandReference(
 251             "python/mozbuild/mozbuild/mach_commands.py"
 252         ),
 253         "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 254         "package-multi-locale": MachCommandReference(
 255             "python/mozbuild/mozbuild/mach_commands.py"
 256         ),
 257         "perftest": MachCommandReference(
 258             "python/mozperftest/mozperftest/mach_commands.py"
 259         ),
 260         "perftest-test": MachCommandReference(
 261             "python/mozperftest/mozperftest/mach_commands.py"
 262         ),
 263         "release": MachCommandReference(
 264             "python/mozrelease/mozrelease/mach_commands.py"
 265         ),
 266         "remote": MachCommandReference("remote/mach_commands.py"),
 267         "puppeteer-test": MachCommandReference("remote/mach_commands.py"),
 268         "generate-test-certs": MachCommandReference(
 269             "security/manager/tools/mach_commands.py"
 270         ),
 271         "taskgraph": MachCommandReference("taskcluster/mach_commands.py"),
 272         "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"),
 273         "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"),
 274         "taskcluster-image-digest": MachCommandReference(
 275             "taskcluster/mach_commands.py"
 276         ),
 277         "release-history": MachCommandReference("taskcluster/mach_commands.py"),
 278         "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"),
 279         "fetch-condprofile": MachCommandReference(
 280             "testing/condprofile/mach_commands.py"
 281         ),
 282         "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
 283         "firefox-ui-functional": MachCommandReference(
 284             "testing/firefox-ui/mach_commands.py"
 285         ),
 286         "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"),
 287         "addtest": MachCommandReference("testing/mach_commands.py"),
 288         "test": MachCommandReference("testing/mach_commands.py"),
 289         "cppunittest": MachCommandReference("testing/mach_commands.py"),
 290         "jstests": MachCommandReference("testing/mach_commands.py"),
 291         "jit-test": MachCommandReference("testing/mach_commands.py"),
 292         "jsapi-tests": MachCommandReference("testing/mach_commands.py"),
 293         "jsshell-bench": MachCommandReference("testing/mach_commands.py"),
 294         "cramtest": MachCommandReference("testing/mach_commands.py"),
 295         "test-info": MachCommandReference("testing/mach_commands.py"),
 296         "rusttests": MachCommandReference("testing/mach_commands.py"),
 297         "fluent-migration-test": MachCommandReference("testing/mach_commands.py"),
 298         "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
 299         "mochitest": MachCommandReference("testing/mochitest/mach_commands.py"),
 300         "geckoview-junit": MachCommandReference("testing/mochitest/mach_commands.py"),
 301         "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"),
 302         "raptor": MachCommandReference("testing/raptor/mach_commands.py"),
 303         "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"),
 304         "talos-test": MachCommandReference("testing/talos/mach_commands.py"),
 305         "tps-build": MachCommandReference("testing/tps/mach_commands.py"),
 306         "web-platform-tests": MachCommandReference(
 307             "testing/web-platform/mach_commands.py"
 308         ),
 309         "wpt": MachCommandReference("testing/web-platform/mach_commands.py"),
 310         "web-platform-tests-update": MachCommandReference(
 311             "testing/web-platform/mach_commands.py"
 312         ),
 313         "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"),
 314         "wpt-manifest-update": MachCommandReference(
 315             "testing/web-platform/mach_commands.py"
 316         ),
 317         "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"),
 318         "wpt-metadata-summary": MachCommandReference(
 319             "testing/web-platform/mach_commands.py"
 320         ),
 321         "wpt-metadata-merge": MachCommandReference(
 322             "testing/web-platform/mach_commands.py"
 323         ),
 324         "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"),
 325         "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"),
 326         "wpt-fission-regressions": MachCommandReference(
 327             "testing/web-platform/mach_commands.py"
 328         ),
 329         "xpcshell-test": MachCommandReference("testing/xpcshell/mach_commands.py"),
 330         "telemetry-tests-client": MachCommandReference(
 331             "toolkit/components/telemetry/tests/marionette/mach_commands.py"
 332         ),
 333         "data-review": MachCommandReference(
 334             "toolkit/components/glean/build_scripts/mach_commands.py"
 335         ),
 336         "perf-data-review": MachCommandReference(
 337             "toolkit/components/glean/build_scripts/mach_commands.py"
 338         ),
 339         "update-glean-tags": MachCommandReference(
 340             "toolkit/components/glean/build_scripts/mach_commands.py"
 341         ),
 342         "update-glean": MachCommandReference(
 343             "toolkit/components/glean/build_scripts/mach_commands.py"
 344         ),
 345         "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"),
 346         "compare-locales": MachCommandReference(
 347             "tools/compare-locales/mach_commands.py"
 348         ),
 349         "l10n-cross-channel": MachCommandReference("tools/lint/mach_commands.py"),
 350         "busted": MachCommandReference("tools/mach_commands.py"),
 351         "pastebin": MachCommandReference("tools/mach_commands.py"),
 352         "mozregression": MachCommandReference("tools/mach_commands.py"),
 353         "node": MachCommandReference("tools/mach_commands.py"),
 354         "npm": MachCommandReference("tools/mach_commands.py"),
 355         "logspam": MachCommandReference("tools/mach_commands.py"),
 356         "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
 357         "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"),
 358         "power": MachCommandReference("tools/power/mach_commands.py"),
 359         "try": MachCommandReference("tools/tryselect/mach_commands.py"),
 360         "import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
 361         "test-interventions": MachCommandReference(
 362             "testing/webcompat/mach_commands.py"
 363         ),
 364         "esmify": MachCommandReference("tools/esmify/mach_commands.py"),
 365         "xpcshell": MachCommandReference("js/xpconnect/mach_commands.py"),
 366         "uniffi": MachCommandReference(
 367             "toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py"
 368         ),
 369         "storybook": MachCommandReference(
 370             "browser/components/storybook/mach_commands.py"
 371         ),
 372     }
 373
 374     # Set a reasonable limit to the number of open files.
 375     #
 376     # Some linux systems set `ulimit -n` to a very high number, which works
 377     # well for systems that run servers, but this setting causes performance
 378     # problems when programs close file descriptors before forking, like
 379     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 380     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 381     # same thing when spawning processes.  We would prefer to lower this limit
 382     # to avoid such performance problems; processes spawned by `mach` will
 383     # inherit the limit set here.
 384     #
 385     # The Firefox build defaults the soft limit to 1024, except for builds that
 386     # do LTO, where the soft limit is 8192.  We're going to default to the
 387     # latter, since people do occasionally do LTO builds on their local
 388     # machines, and requiring them to discover another magical setting after
 389     # setting up an LTO build in the first place doesn't seem good.
 390     #
 391     # This code mimics the code in taskcluster/scripts/run-task.
 392     try:
 393         import resource
 394
 395         # Keep the hard limit the same, though, allowing processes to change
 396         # their soft limit if they need to (Firefox does, for instance).
 397         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 398         # Permit people to override our default limit if necessary via
 399         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 400         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 401         if limit:
 402             limit = int(limit)
 403         else:
 404             # If no explicit limit is given, use our default if it's less than
 405             # the current soft limit.  For instance, the default on macOS is
 406             # 256, so we'd pick that rather than our default.
 407             limit = min(soft, 8192)
 408         # Now apply the limit, if it's different from the original one.
 409         if limit != soft:
 410             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 411     except ImportError:
 412         # The resource module is UNIX only.
 413         pass
 414
 415     def resolve_repository():
 416         import mozversioncontrol
 417
 418         try:
 419             # This API doesn't respect the vcs binary choices from configure.
 420             # If we ever need to use the VCS binary here, consider something
 421             # more robust.
 422             return mozversioncontrol.get_repository_object(path=topsrcdir)
 423         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 424             return None
 425
 426     def pre_dispatch_handler(context, handler, args):
 427         # If --disable-tests flag was enabled in the mozconfig used to compile
 428         # the build, tests will be disabled. Instead of trying to run
 429         # nonexistent tests then reporting a failure, this will prevent mach
 430         # from progressing beyond this point.
 431         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 432             from mozbuild.base import BuildEnvironmentNotFoundException
 433
 434             try:
 435                 from mozbuild.base import MozbuildObject
 436
 437                 # all environments should have an instance of build object.
 438                 build = MozbuildObject.from_environment()
 439                 if build is not None and not getattr(
 440                     build, "substs", {"ENABLE_TESTS": True}
 441                 ).get("ENABLE_TESTS"):
 442                     print(
 443                         "Tests have been disabled with --disable-tests.\n"
 444                         + "Remove the flag, and re-compile to enable tests."
 445                     )
 446                     sys.exit(1)
 447             except BuildEnvironmentNotFoundException:
 448                 # likely automation environment, so do nothing.
 449                 pass
 450
 451     def post_dispatch_handler(
 452         context, handler, instance, success, start_time, end_time, depth, args
 453     ):
 454         """Perform global operations after command dispatch.
 455
 456
 457         For now,  we will use this to handle build system telemetry.
 458         """
 459
 460         # Don't finalize telemetry data if this mach command was invoked as part of
 461         # another mach command.
 462         if depth != 1:
 463             return
 464
 465         _finalize_telemetry_glean(
 466             context.telemetry, handler.name == "bootstrap", success
 467         )
 468
 469     def populate_context(key=None):
 470         if key is None:
 471             return
 472         if key == "state_dir":
 473             return state_dir
 474
 475         if key == "local_state_dir":
 476             return get_state_dir(specific_to_topsrcdir=True)
 477
 478         if key == "topdir":
 479             return topsrcdir
 480
 481         if key == "pre_dispatch_handler":
 482             return pre_dispatch_handler
 483
 484         if key == "post_dispatch_handler":
 485             return post_dispatch_handler
 486
 487         if key == "repository":
 488             return resolve_repository()
 489
 490         raise AttributeError(key)
 491
 492     # Note which process is top-level so that recursive mach invocations can avoid writing
 493     # telemetry data.
 494     if "MACH_MAIN_PID" not in os.environ:
 495         setenv("MACH_MAIN_PID", str(os.getpid()))
 496
 497     driver = mach.main.Mach(os.getcwd())
 498     driver.populate_context_handler = populate_context
 499
 500     if not driver.settings_paths:
 501         # default global machrc location
 502         driver.settings_paths.append(state_dir)
 503     # always load local repository configuration
 504     driver.settings_paths.append(topsrcdir)
 505
 506     for category, meta in CATEGORIES.items():
 507         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 508
 509     # Sparse checkouts may not have all mach_commands.py files. Ignore
 510     # errors from missing files. Same for spidermonkey tarballs.
 511     repo = resolve_repository()
 512     missing_ok = (
 513         repo is not None and repo.sparse_checkout_present()
 514     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 515
 516     driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
 517
 518     return driver
 519
 520
 521 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 522     """Submit telemetry collected by Glean.
 523
 524     Finalizes some metrics (command success state and duration, system information) and
 525     requests Glean to send the collected data.
 526     """
 527
 528     from mach.telemetry import MACH_METRICS_PATH
 529     from mozbuild.telemetry import (
 530         get_cpu_brand,
 531         get_distro_and_version,
 532         get_psutil_stats,
 533         get_shell_info,
 534         get_vscode_running,
 535     )
 536
 537     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 538     mach_metrics.mach.duration.stop()
 539     mach_metrics.mach.success.set(success)
 540     system_metrics = mach_metrics.mach.system
 541     cpu_brand = get_cpu_brand()
 542     if cpu_brand:
 543         system_metrics.cpu_brand.set(cpu_brand)
 544     distro, version = get_distro_and_version()
 545     system_metrics.distro.set(distro)
 546     system_metrics.distro_version.set(version)
 547
 548     vscode_terminal, ssh_connection = get_shell_info()
 549     system_metrics.vscode_terminal.set(vscode_terminal)
 550     system_metrics.ssh_connection.set(ssh_connection)
 551     system_metrics.vscode_running.set(get_vscode_running())
 552
 553     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 554     if has_psutil:
 555         # psutil may not be available (we may not have been able to download
 556         # a wheel or build it from source).
 557         system_metrics.logical_cores.add(logical_cores)
 558         if physical_cores is not None:
 559             system_metrics.physical_cores.add(physical_cores)
 560         if memory_total is not None:
 561             system_metrics.memory.accumulate(
 562                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 563             )
 564     telemetry.submit(is_bootstrap)
 565
 566
 567 def _create_state_dir():
 568     # Global build system and mach state is stored in a central directory. By
 569     # default, this is ~/.mozbuild. However, it can be defined via an
 570     # environment variable. We detect first run (by lack of this directory
 571     # existing) and notify the user that it will be created. The logic for
 572     # creation is much simpler for the "advanced" environment variable use
 573     # case. For default behavior, we educate users and give them an opportunity
 574     # to react.
 575     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 576     if state_dir:
 577         if not os.path.exists(state_dir):
 578             print(
 579                 "Creating global state directory from environment variable: {}".format(
 580                     state_dir
 581                 )
 582             )
 583     else:
 584         state_dir = os.path.expanduser("~/.mozbuild")
 585         if not os.path.exists(state_dir):
 586             if not os.environ.get("MOZ_AUTOMATION"):
 587                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 588                 try:
 589                     sys.stdin.readline()
 590                     print("\n")
 591                 except KeyboardInterrupt:
 592                     sys.exit(1)
 593
 594             print("Creating default state directory: {}".format(state_dir))
 595
 596     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 597     return state_dir
 598
 599
 600 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 601 # the source directory are essentially ignored. See further below for details
 602 # and caveats.
 603 # Objdirs outside the source directory are ignored because in most cases, if
 604 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 605 class ImportHook(object):
 606     def __init__(self, original_import):
 607         self._original_import = original_import
 608         # Assume the source directory is the parent directory of the one
 609         # containing this file.
 610         self._source_dir = (
 611             os.path.normcase(
 612                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 613             )
 614             + os.sep
 615         )
 616         self._modules = set()
 617
 618     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 619         if sys.version_info[0] >= 3 and level < 0:
 620             level = 0
 621
 622         # name might be a relative import. Instead of figuring out what that
 623         # resolves to, which is complex, just rely on the real import.
 624         # Since we don't know the full module name, we can't check sys.modules,
 625         # so we need to keep track of which modules we've already seen to avoid
 626         # to stat() them again when they are imported multiple times.
 627         module = self._original_import(name, globals, locals, fromlist, level)
 628
 629         # Some tests replace modules in sys.modules with non-module instances.
 630         if not isinstance(module, ModuleType):
 631             return module
 632
 633         resolved_name = module.__name__
 634         if resolved_name in self._modules:
 635             return module
 636         self._modules.add(resolved_name)
 637
 638         # Builtin modules don't have a __file__ attribute.
 639         if not getattr(module, "__file__", None):
 640             return module
 641
 642         # Note: module.__file__ is not always absolute.
 643         path = os.path.normcase(os.path.abspath(module.__file__))
 644         # Note: we could avoid normcase and abspath above for non pyc/pyo
 645         # files, but those are actually rare, so it doesn't really matter.
 646         if not path.endswith((".pyc", ".pyo")):
 647             return module
 648
 649         # Ignore modules outside our source directory
 650         if not path.startswith(self._source_dir):
 651             return module
 652
 653         # If there is no .py corresponding to the .pyc/.pyo module we're
 654         # loading, remove the .pyc/.pyo file, and reload the module.
 655         # Since we already loaded the .pyc/.pyo module, if it had side
 656         # effects, they will have happened already, and loading the module
 657         # with the same name, from another directory may have the same side
 658         # effects (or different ones). We assume it's not a problem for the
 659         # python modules under our source directory (either because it
 660         # doesn't happen or because it doesn't matter).
 661         if not os.path.exists(module.__file__[:-1]):
 662             if os.path.exists(module.__file__):
 663                 os.remove(module.__file__)
 664             del sys.modules[module.__name__]
 665             module = self(name, globals, locals, fromlist, level)
 666
 667         return module
 668
 669
 670 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 671 # the source directory are essentially ignored. See further below for details
 672 # and caveats.
 673 # Objdirs outside the source directory are ignored because in most cases, if
 674 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 675 class FinderHook(MetaPathFinder):
 676     def __init__(self, klass):
 677         # Assume the source directory is the parent directory of the one
 678         # containing this file.
 679         self._source_dir = (
 680             os.path.normcase(
 681                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 682             )
 683             + os.sep
 684         )
 685         self.finder_class = klass
 686
 687     def find_spec(self, full_name, paths=None, target=None):
 688         spec = self.finder_class.find_spec(full_name, paths, target)
 689
 690         # Some modules don't have an origin.
 691         if spec is None or spec.origin is None:
 692             return spec
 693
 694         # Normalize the origin path.
 695         path = os.path.normcase(os.path.abspath(spec.origin))
 696         # Note: we could avoid normcase and abspath above for non pyc/pyo
 697         # files, but those are actually rare, so it doesn't really matter.
 698         if not path.endswith((".pyc", ".pyo")):
 699             return spec
 700
 701         # Ignore modules outside our source directory
 702         if not path.startswith(self._source_dir):
 703             return spec
 704
 705         # If there is no .py corresponding to the .pyc/.pyo module we're
 706         # resolving, remove the .pyc/.pyo file, and try again.
 707         if not os.path.exists(spec.origin[:-1]):
 708             if os.path.exists(spec.origin):
 709                 os.remove(spec.origin)
 710             spec = self.finder_class.find_spec(full_name, paths, target)
 711
 712         return spec
 713
 714
 715 # Additional hook for python >= 3.8's importlib.metadata.
 716 class MetadataHook(FinderHook):
 717     def find_distributions(self, *args, **kwargs):
 718         return self.finder_class.find_distributions(*args, **kwargs)
 719
 720
 721 def hook(finder):
 722     has_find_spec = hasattr(finder, "find_spec")
 723     has_find_distributions = hasattr(finder, "find_distributions")
 724     if has_find_spec and has_find_distributions:
 725         return MetadataHook(finder)
 726     elif has_find_spec:
 727         return FinderHook(finder)
 728     return finder
 729
 730
 731 # Install our hook. This can be deleted when the Python 3 migration is complete.
 732 if sys.version_info[0] < 3:
 733     builtins.__import__ = ImportHook(builtins.__import__)
 734 else:
 735     sys.meta_path = [hook(c) for c in sys.meta_path]