build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import math
   8 import os
   9 import shutil
  10 import sys
  11 from pathlib import Path
  12
  13 if sys.version_info[0] < 3:
  14     import __builtin__ as builtins
  15
  16     class MetaPathFinder(object):
  17         pass
  18
  19
  20 else:
  21     from importlib.abc import MetaPathFinder
  22
  23
  24 from types import ModuleType
  25
  26
  27 STATE_DIR_FIRST_RUN = """
  28 Mach and the build system store shared state in a common directory
  29 on the filesystem. The following directory will be created:
  30
  31   {}
  32
  33 If you would like to use a different directory, hit CTRL+c, set the
  34 MOZBUILD_STATE_PATH environment variable to the directory you'd like to
  35 use, and run Mach again.
  36
  37 Press ENTER/RETURN to continue or CTRL+c to abort.
  38 """.strip()
  39
  40
  41 CATEGORIES = {
  42     "build": {
  43         "short": "Build Commands",
  44         "long": "Interact with the build system",
  45         "priority": 80,
  46     },
  47     "post-build": {
  48         "short": "Post-build Commands",
  49         "long": "Common actions performed after completing a build.",
  50         "priority": 70,
  51     },
  52     "testing": {
  53         "short": "Testing",
  54         "long": "Run tests.",
  55         "priority": 60,
  56     },
  57     "ci": {
  58         "short": "CI",
  59         "long": "Taskcluster commands",
  60         "priority": 59,
  61     },
  62     "devenv": {
  63         "short": "Development Environment",
  64         "long": "Set up and configure your development environment.",
  65         "priority": 50,
  66     },
  67     "build-dev": {
  68         "short": "Low-level Build System Interaction",
  69         "long": "Interact with specific parts of the build system.",
  70         "priority": 20,
  71     },
  72     "misc": {
  73         "short": "Potpourri",
  74         "long": "Potent potables and assorted snacks.",
  75         "priority": 10,
  76     },
  77     "release": {
  78         "short": "Release automation",
  79         "long": "Commands for used in release automation.",
  80         "priority": 5,
  81     },
  82     "disabled": {
  83         "short": "Disabled",
  84         "long": "The disabled commands are hidden by default. Use -v to display them. "
  85         "These commands are unavailable for your current context, "
  86         'run "mach <command>" to see why.',
  87         "priority": 0,
  88     },
  89 }
  90
  91
  92 def _activate_python_environment(topsrcdir, get_state_dir):
  93     from mach.site import MachSiteManager
  94
  95     mach_environment = MachSiteManager.from_environment(
  96         topsrcdir,
  97         get_state_dir,
  98     )
  99     mach_environment.activate()
 100
 101
 102 def _maybe_activate_mozillabuild_environment():
 103     if sys.platform != "win32":
 104         return
 105
 106     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 107     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 108     assert mozillabuild.exists(), (
 109         f'MozillaBuild was not found at "{mozillabuild}".\n'
 110         "If it's installed in a different location, please "
 111         'set the "MOZILLABUILD" environment variable '
 112         "accordingly."
 113     )
 114
 115     use_msys2 = (mozillabuild / "msys2").exists()
 116     if use_msys2:
 117         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 118     else:
 119         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 120
 121     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 122     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 123     for new_path in paths_to_add:
 124         if new_path not in existing_paths:
 125             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 126
 127
 128 def initialize(topsrcdir):
 129     # This directory was deleted in bug 1666345, but there may be some ignored
 130     # files here. We can safely just delete it for the user so they don't have
 131     # to clean the repo themselves.
 132     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 133     if os.path.exists(deleted_dir):
 134         shutil.rmtree(deleted_dir, ignore_errors=True)
 135
 136     # We need the "mach" module to access the logic to parse virtualenv
 137     # requirements. Since that depends on "packaging" (and, transitively,
 138     # "pyparsing"), we add those to the path too.
 139     sys.path[0:0] = [
 140         os.path.join(topsrcdir, module)
 141         for module in (
 142             os.path.join("python", "mach"),
 143             os.path.join("third_party", "python", "packaging"),
 144             os.path.join("third_party", "python", "pyparsing"),
 145         )
 146     ]
 147
 148     from mach.util import setenv, get_state_dir
 149
 150     state_dir = _create_state_dir()
 151
 152     # normpath state_dir to normalize msys-style slashes.
 153     _activate_python_environment(
 154         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 155     )
 156     _maybe_activate_mozillabuild_environment()
 157
 158     import mach.base
 159     import mach.main
 160
 161     from mach.main import MachCommandReference
 162
 163     # Centralized registry of available mach commands
 164     MACH_COMMANDS = {
 165         "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"),
 166         "devtools-css-db": MachCommandReference(
 167             "devtools/shared/css/generated/mach_commands.py"
 168         ),
 169         "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"),
 170         "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"),
 171         "hazards": MachCommandReference(
 172             "js/src/devtools/rootAnalysis/mach_commands.py"
 173         ),
 174         "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 175         "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 176         "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 177         "android": MachCommandReference("mobile/android/mach_commands.py"),
 178         "gradle": MachCommandReference("mobile/android/mach_commands.py"),
 179         "gradle-install": MachCommandReference("mobile/android/mach_commands.py"),
 180         "mach-commands": MachCommandReference(
 181             "python/mach/mach/commands/commandinfo.py"
 182         ),
 183         "mach-debug-commands": MachCommandReference(
 184             "python/mach/mach/commands/commandinfo.py"
 185         ),
 186         "mach-completion": MachCommandReference(
 187             "python/mach/mach/commands/commandinfo.py"
 188         ),
 189         "settings": MachCommandReference("python/mach/mach/commands/settings.py"),
 190         "python": MachCommandReference("python/mach_commands.py"),
 191         "python-test": MachCommandReference("python/mach_commands.py"),
 192         "bootstrap": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 193         "vcs-setup": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 194         "artifact": MachCommandReference(
 195             "python/mozbuild/mozbuild/artifact_commands.py"
 196         ),
 197         "ide": MachCommandReference(
 198             "python/mozbuild/mozbuild/backend/mach_commands.py"
 199         ),
 200         "build": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 201         "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 202         "resource-usage": MachCommandReference(
 203             "python/mozbuild/mozbuild/build_commands.py"
 204         ),
 205         "build-backend": MachCommandReference(
 206             "python/mozbuild/mozbuild/build_commands.py"
 207         ),
 208         "clang-tidy": MachCommandReference(
 209             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 210         ),
 211         "static-analysis": MachCommandReference(
 212             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 213         ),
 214         "prettier-format": MachCommandReference(
 215             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 216         ),
 217         "clang-format": MachCommandReference(
 218             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 219         ),
 220         "compileflags": MachCommandReference(
 221             "python/mozbuild/mozbuild/compilation/codecomplete.py"
 222         ),
 223         "mozbuild-reference": MachCommandReference(
 224             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 225         ),
 226         "file-info": MachCommandReference(
 227             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 228         ),
 229         "vendor": MachCommandReference(
 230             "python/mozbuild/mozbuild/vendor/mach_commands.py"
 231         ),
 232         "watch": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 233         "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 234         "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 235         "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 236         "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 237         "warnings-summary": MachCommandReference(
 238             "python/mozbuild/mozbuild/mach_commands.py"
 239         ),
 240         "warnings-list": MachCommandReference(
 241             "python/mozbuild/mozbuild/mach_commands.py"
 242         ),
 243         "gtest": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 244         "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 245         "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 246         "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 247         "buildsymbols": MachCommandReference(
 248             "python/mozbuild/mozbuild/mach_commands.py"
 249         ),
 250         "environment": MachCommandReference(
 251             "python/mozbuild/mozbuild/mach_commands.py"
 252         ),
 253         "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 254         "package-multi-locale": MachCommandReference(
 255             "python/mozbuild/mozbuild/mach_commands.py"
 256         ),
 257         "perftest": MachCommandReference(
 258             "python/mozperftest/mozperftest/mach_commands.py"
 259         ),
 260         "perftest-test": MachCommandReference(
 261             "python/mozperftest/mozperftest/mach_commands.py"
 262         ),
 263         "release": MachCommandReference(
 264             "python/mozrelease/mozrelease/mach_commands.py"
 265         ),
 266         "remote": MachCommandReference("remote/mach_commands.py"),
 267         "puppeteer-test": MachCommandReference("remote/mach_commands.py"),
 268         "generate-test-certs": MachCommandReference(
 269             "security/manager/tools/mach_commands.py"
 270         ),
 271         "taskgraph": MachCommandReference("taskcluster/mach_commands.py"),
 272         "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"),
 273         "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"),
 274         "taskcluster-image-digest": MachCommandReference(
 275             "taskcluster/mach_commands.py"
 276         ),
 277         "release-history": MachCommandReference("taskcluster/mach_commands.py"),
 278         "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"),
 279         "fetch-condprofile": MachCommandReference(
 280             "testing/condprofile/mach_commands.py"
 281         ),
 282         "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
 283         "firefox-ui-functional": MachCommandReference(
 284             "testing/firefox-ui/mach_commands.py"
 285         ),
 286         "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"),
 287         "addtest": MachCommandReference("testing/mach_commands.py"),
 288         "test": MachCommandReference("testing/mach_commands.py"),
 289         "cppunittest": MachCommandReference("testing/mach_commands.py"),
 290         "jstests": MachCommandReference("testing/mach_commands.py"),
 291         "jit-test": MachCommandReference("testing/mach_commands.py"),
 292         "jsapi-tests": MachCommandReference("testing/mach_commands.py"),
 293         "jsshell-bench": MachCommandReference("testing/mach_commands.py"),
 294         "cramtest": MachCommandReference("testing/mach_commands.py"),
 295         "test-info": MachCommandReference("testing/mach_commands.py"),
 296         "rusttests": MachCommandReference("testing/mach_commands.py"),
 297         "fluent-migration-test": MachCommandReference("testing/mach_commands.py"),
 298         "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
 299         "mochitest": MachCommandReference("testing/mochitest/mach_commands.py"),
 300         "geckoview-junit": MachCommandReference("testing/mochitest/mach_commands.py"),
 301         "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"),
 302         "raptor": MachCommandReference("testing/raptor/mach_commands.py"),
 303         "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"),
 304         "talos-test": MachCommandReference("testing/talos/mach_commands.py"),
 305         "tps-build": MachCommandReference("testing/tps/mach_commands.py"),
 306         "web-platform-tests": MachCommandReference(
 307             "testing/web-platform/mach_commands.py"
 308         ),
 309         "wpt": MachCommandReference("testing/web-platform/mach_commands.py"),
 310         "web-platform-tests-update": MachCommandReference(
 311             "testing/web-platform/mach_commands.py"
 312         ),
 313         "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"),
 314         "wpt-manifest-update": MachCommandReference(
 315             "testing/web-platform/mach_commands.py"
 316         ),
 317         "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"),
 318         "wpt-metadata-summary": MachCommandReference(
 319             "testing/web-platform/mach_commands.py"
 320         ),
 321         "wpt-metadata-merge": MachCommandReference(
 322             "testing/web-platform/mach_commands.py"
 323         ),
 324         "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"),
 325         "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"),
 326         "wpt-fission-regressions": MachCommandReference(
 327             "testing/web-platform/mach_commands.py"
 328         ),
 329         "xpcshell-test": MachCommandReference("testing/xpcshell/mach_commands.py"),
 330         "telemetry-tests-client": MachCommandReference(
 331             "toolkit/components/telemetry/tests/marionette/mach_commands.py"
 332         ),
 333         "data-review": MachCommandReference(
 334             "toolkit/components/glean/build_scripts/mach_commands.py"
 335         ),
 336         "perf-data-review": MachCommandReference(
 337             "toolkit/components/glean/build_scripts/mach_commands.py"
 338         ),
 339         "update-glean-tags": MachCommandReference(
 340             "toolkit/components/glean/build_scripts/mach_commands.py"
 341         ),
 342         "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"),
 343         "compare-locales": MachCommandReference(
 344             "tools/compare-locales/mach_commands.py"
 345         ),
 346         "l10n-cross-channel": MachCommandReference("tools/lint/mach_commands.py"),
 347         "busted": MachCommandReference("tools/mach_commands.py"),
 348         "pastebin": MachCommandReference("tools/mach_commands.py"),
 349         "mozregression": MachCommandReference("tools/mach_commands.py"),
 350         "node": MachCommandReference("tools/mach_commands.py"),
 351         "npm": MachCommandReference("tools/mach_commands.py"),
 352         "logspam": MachCommandReference("tools/mach_commands.py"),
 353         "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
 354         "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"),
 355         "power": MachCommandReference("tools/power/mach_commands.py"),
 356         "try": MachCommandReference("tools/tryselect/mach_commands.py"),
 357         "import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
 358         "test-interventions": MachCommandReference(
 359             "testing/webcompat/mach_commands.py"
 360         ),
 361         "esmify": MachCommandReference("tools/esmify/mach_commands.py"),
 362     }
 363
 364     # Set a reasonable limit to the number of open files.
 365     #
 366     # Some linux systems set `ulimit -n` to a very high number, which works
 367     # well for systems that run servers, but this setting causes performance
 368     # problems when programs close file descriptors before forking, like
 369     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 370     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 371     # same thing when spawning processes.  We would prefer to lower this limit
 372     # to avoid such performance problems; processes spawned by `mach` will
 373     # inherit the limit set here.
 374     #
 375     # The Firefox build defaults the soft limit to 1024, except for builds that
 376     # do LTO, where the soft limit is 8192.  We're going to default to the
 377     # latter, since people do occasionally do LTO builds on their local
 378     # machines, and requiring them to discover another magical setting after
 379     # setting up an LTO build in the first place doesn't seem good.
 380     #
 381     # This code mimics the code in taskcluster/scripts/run-task.
 382     try:
 383         import resource
 384
 385         # Keep the hard limit the same, though, allowing processes to change
 386         # their soft limit if they need to (Firefox does, for instance).
 387         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 388         # Permit people to override our default limit if necessary via
 389         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 390         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 391         if limit:
 392             limit = int(limit)
 393         else:
 394             # If no explicit limit is given, use our default if it's less than
 395             # the current soft limit.  For instance, the default on macOS is
 396             # 256, so we'd pick that rather than our default.
 397             limit = min(soft, 8192)
 398         # Now apply the limit, if it's different from the original one.
 399         if limit != soft:
 400             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 401     except ImportError:
 402         # The resource module is UNIX only.
 403         pass
 404
 405     def resolve_repository():
 406         import mozversioncontrol
 407
 408         try:
 409             # This API doesn't respect the vcs binary choices from configure.
 410             # If we ever need to use the VCS binary here, consider something
 411             # more robust.
 412             return mozversioncontrol.get_repository_object(path=topsrcdir)
 413         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 414             return None
 415
 416     def pre_dispatch_handler(context, handler, args):
 417         # If --disable-tests flag was enabled in the mozconfig used to compile
 418         # the build, tests will be disabled. Instead of trying to run
 419         # nonexistent tests then reporting a failure, this will prevent mach
 420         # from progressing beyond this point.
 421         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 422             from mozbuild.base import BuildEnvironmentNotFoundException
 423
 424             try:
 425                 from mozbuild.base import MozbuildObject
 426
 427                 # all environments should have an instance of build object.
 428                 build = MozbuildObject.from_environment()
 429                 if build is not None and not getattr(
 430                     build, "substs", {"ENABLE_TESTS": True}
 431                 ).get("ENABLE_TESTS"):
 432                     print(
 433                         "Tests have been disabled with --disable-tests.\n"
 434                         + "Remove the flag, and re-compile to enable tests."
 435                     )
 436                     sys.exit(1)
 437             except BuildEnvironmentNotFoundException:
 438                 # likely automation environment, so do nothing.
 439                 pass
 440
 441     def post_dispatch_handler(
 442         context, handler, instance, success, start_time, end_time, depth, args
 443     ):
 444         """Perform global operations after command dispatch.
 445
 446
 447         For now,  we will use this to handle build system telemetry.
 448         """
 449
 450         # Don't finalize telemetry data if this mach command was invoked as part of
 451         # another mach command.
 452         if depth != 1:
 453             return
 454
 455         _finalize_telemetry_glean(
 456             context.telemetry, handler.name == "bootstrap", success
 457         )
 458
 459     def populate_context(key=None):
 460         if key is None:
 461             return
 462         if key == "state_dir":
 463             return state_dir
 464
 465         if key == "local_state_dir":
 466             return get_state_dir(specific_to_topsrcdir=True)
 467
 468         if key == "topdir":
 469             return topsrcdir
 470
 471         if key == "pre_dispatch_handler":
 472             return pre_dispatch_handler
 473
 474         if key == "post_dispatch_handler":
 475             return post_dispatch_handler
 476
 477         if key == "repository":
 478             return resolve_repository()
 479
 480         raise AttributeError(key)
 481
 482     # Note which process is top-level so that recursive mach invocations can avoid writing
 483     # telemetry data.
 484     if "MACH_MAIN_PID" not in os.environ:
 485         setenv("MACH_MAIN_PID", str(os.getpid()))
 486
 487     driver = mach.main.Mach(os.getcwd())
 488     driver.populate_context_handler = populate_context
 489
 490     if not driver.settings_paths:
 491         # default global machrc location
 492         driver.settings_paths.append(state_dir)
 493     # always load local repository configuration
 494     driver.settings_paths.append(topsrcdir)
 495
 496     for category, meta in CATEGORIES.items():
 497         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 498
 499     # Sparse checkouts may not have all mach_commands.py files. Ignore
 500     # errors from missing files. Same for spidermonkey tarballs.
 501     repo = resolve_repository()
 502     missing_ok = (
 503         repo is not None and repo.sparse_checkout_present()
 504     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 505
 506     driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
 507
 508     return driver
 509
 510
 511 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 512     """Submit telemetry collected by Glean.
 513
 514     Finalizes some metrics (command success state and duration, system information) and
 515     requests Glean to send the collected data.
 516     """
 517
 518     from mach.telemetry import MACH_METRICS_PATH
 519     from mozbuild.telemetry import (
 520         get_cpu_brand,
 521         get_distro_and_version,
 522         get_psutil_stats,
 523         get_shell_info,
 524         get_vscode_running,
 525     )
 526
 527     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 528     mach_metrics.mach.duration.stop()
 529     mach_metrics.mach.success.set(success)
 530     system_metrics = mach_metrics.mach.system
 531     cpu_brand = get_cpu_brand()
 532     if cpu_brand:
 533         system_metrics.cpu_brand.set(cpu_brand)
 534     distro, version = get_distro_and_version()
 535     system_metrics.distro.set(distro)
 536     system_metrics.distro_version.set(version)
 537
 538     vscode_terminal, ssh_connection = get_shell_info()
 539     system_metrics.vscode_terminal.set(vscode_terminal)
 540     system_metrics.ssh_connection.set(ssh_connection)
 541     system_metrics.vscode_running.set(get_vscode_running())
 542
 543     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 544     if has_psutil:
 545         # psutil may not be available (we may not have been able to download
 546         # a wheel or build it from source).
 547         system_metrics.logical_cores.add(logical_cores)
 548         if physical_cores is not None:
 549             system_metrics.physical_cores.add(physical_cores)
 550         if memory_total is not None:
 551             system_metrics.memory.accumulate(
 552                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 553             )
 554     telemetry.submit(is_bootstrap)
 555
 556
 557 def _create_state_dir():
 558     # Global build system and mach state is stored in a central directory. By
 559     # default, this is ~/.mozbuild. However, it can be defined via an
 560     # environment variable. We detect first run (by lack of this directory
 561     # existing) and notify the user that it will be created. The logic for
 562     # creation is much simpler for the "advanced" environment variable use
 563     # case. For default behavior, we educate users and give them an opportunity
 564     # to react.
 565     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 566     if state_dir:
 567         if not os.path.exists(state_dir):
 568             print(
 569                 "Creating global state directory from environment variable: {}".format(
 570                     state_dir
 571                 )
 572             )
 573     else:
 574         state_dir = os.path.expanduser("~/.mozbuild")
 575         if not os.path.exists(state_dir):
 576             if not os.environ.get("MOZ_AUTOMATION"):
 577                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 578                 try:
 579                     sys.stdin.readline()
 580                     print("\n")
 581                 except KeyboardInterrupt:
 582                     sys.exit(1)
 583
 584             print("Creating default state directory: {}".format(state_dir))
 585
 586     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 587     return state_dir
 588
 589
 590 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 591 # the source directory are essentially ignored. See further below for details
 592 # and caveats.
 593 # Objdirs outside the source directory are ignored because in most cases, if
 594 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 595 class ImportHook(object):
 596     def __init__(self, original_import):
 597         self._original_import = original_import
 598         # Assume the source directory is the parent directory of the one
 599         # containing this file.
 600         self._source_dir = (
 601             os.path.normcase(
 602                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 603             )
 604             + os.sep
 605         )
 606         self._modules = set()
 607
 608     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 609         if sys.version_info[0] >= 3 and level < 0:
 610             level = 0
 611
 612         # name might be a relative import. Instead of figuring out what that
 613         # resolves to, which is complex, just rely on the real import.
 614         # Since we don't know the full module name, we can't check sys.modules,
 615         # so we need to keep track of which modules we've already seen to avoid
 616         # to stat() them again when they are imported multiple times.
 617         module = self._original_import(name, globals, locals, fromlist, level)
 618
 619         # Some tests replace modules in sys.modules with non-module instances.
 620         if not isinstance(module, ModuleType):
 621             return module
 622
 623         resolved_name = module.__name__
 624         if resolved_name in self._modules:
 625             return module
 626         self._modules.add(resolved_name)
 627
 628         # Builtin modules don't have a __file__ attribute.
 629         if not getattr(module, "__file__", None):
 630             return module
 631
 632         # Note: module.__file__ is not always absolute.
 633         path = os.path.normcase(os.path.abspath(module.__file__))
 634         # Note: we could avoid normcase and abspath above for non pyc/pyo
 635         # files, but those are actually rare, so it doesn't really matter.
 636         if not path.endswith((".pyc", ".pyo")):
 637             return module
 638
 639         # Ignore modules outside our source directory
 640         if not path.startswith(self._source_dir):
 641             return module
 642
 643         # If there is no .py corresponding to the .pyc/.pyo module we're
 644         # loading, remove the .pyc/.pyo file, and reload the module.
 645         # Since we already loaded the .pyc/.pyo module, if it had side
 646         # effects, they will have happened already, and loading the module
 647         # with the same name, from another directory may have the same side
 648         # effects (or different ones). We assume it's not a problem for the
 649         # python modules under our source directory (either because it
 650         # doesn't happen or because it doesn't matter).
 651         if not os.path.exists(module.__file__[:-1]):
 652             if os.path.exists(module.__file__):
 653                 os.remove(module.__file__)
 654             del sys.modules[module.__name__]
 655             module = self(name, globals, locals, fromlist, level)
 656
 657         return module
 658
 659
 660 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 661 # the source directory are essentially ignored. See further below for details
 662 # and caveats.
 663 # Objdirs outside the source directory are ignored because in most cases, if
 664 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 665 class FinderHook(MetaPathFinder):
 666     def __init__(self, klass):
 667         # Assume the source directory is the parent directory of the one
 668         # containing this file.
 669         self._source_dir = (
 670             os.path.normcase(
 671                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 672             )
 673             + os.sep
 674         )
 675         self.finder_class = klass
 676
 677     def find_spec(self, full_name, paths=None, target=None):
 678         spec = self.finder_class.find_spec(full_name, paths, target)
 679
 680         # Some modules don't have an origin.
 681         if spec is None or spec.origin is None:
 682             return spec
 683
 684         # Normalize the origin path.
 685         path = os.path.normcase(os.path.abspath(spec.origin))
 686         # Note: we could avoid normcase and abspath above for non pyc/pyo
 687         # files, but those are actually rare, so it doesn't really matter.
 688         if not path.endswith((".pyc", ".pyo")):
 689             return spec
 690
 691         # Ignore modules outside our source directory
 692         if not path.startswith(self._source_dir):
 693             return spec
 694
 695         # If there is no .py corresponding to the .pyc/.pyo module we're
 696         # resolving, remove the .pyc/.pyo file, and try again.
 697         if not os.path.exists(spec.origin[:-1]):
 698             if os.path.exists(spec.origin):
 699                 os.remove(spec.origin)
 700             spec = self.finder_class.find_spec(full_name, paths, target)
 701
 702         return spec
 703
 704
 705 # Additional hook for python >= 3.8's importlib.metadata.
 706 class MetadataHook(FinderHook):
 707     def find_distributions(self, *args, **kwargs):
 708         return self.finder_class.find_distributions(*args, **kwargs)
 709
 710
 711 def hook(finder):
 712     has_find_spec = hasattr(finder, "find_spec")
 713     has_find_distributions = hasattr(finder, "find_distributions")
 714     if has_find_spec and has_find_distributions:
 715         return MetadataHook(finder)
 716     elif has_find_spec:
 717         return FinderHook(finder)
 718     return finder
 719
 720
 721 # Install our hook. This can be deleted when the Python 3 migration is complete.
 722 if sys.version_info[0] < 3:
 723     builtins.__import__ = ImportHook(builtins.__import__)
 724 else:
 725     sys.meta_path = [hook(c) for c in sys.meta_path]