build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import math
   8 import os
   9 import shutil
  10 import sys
  11 from pathlib import Path
  12
  13 if sys.version_info[0] < 3:
  14     import __builtin__ as builtins
  15
  16     class MetaPathFinder(object):
  17         pass
  18
  19
  20 else:
  21     from importlib.abc import MetaPathFinder
  22
  23
  24 from types import ModuleType
  25
  26
  27 STATE_DIR_FIRST_RUN = """
  28 Mach and the build system store shared state in a common directory
  29 on the filesystem. The following directory will be created:
  30
  31   {}
  32
  33 If you would like to use a different directory, hit CTRL+c, set the
  34 MOZBUILD_STATE_PATH environment variable to the directory you'd like to
  35 use, and run Mach again.
  36
  37 Press ENTER/RETURN to continue or CTRL+c to abort.
  38 """.strip()
  39
  40
  41 CATEGORIES = {
  42     "build": {
  43         "short": "Build Commands",
  44         "long": "Interact with the build system",
  45         "priority": 80,
  46     },
  47     "post-build": {
  48         "short": "Post-build Commands",
  49         "long": "Common actions performed after completing a build.",
  50         "priority": 70,
  51     },
  52     "testing": {
  53         "short": "Testing",
  54         "long": "Run tests.",
  55         "priority": 60,
  56     },
  57     "ci": {
  58         "short": "CI",
  59         "long": "Taskcluster commands",
  60         "priority": 59,
  61     },
  62     "devenv": {
  63         "short": "Development Environment",
  64         "long": "Set up and configure your development environment.",
  65         "priority": 50,
  66     },
  67     "build-dev": {
  68         "short": "Low-level Build System Interaction",
  69         "long": "Interact with specific parts of the build system.",
  70         "priority": 20,
  71     },
  72     "misc": {
  73         "short": "Potpourri",
  74         "long": "Potent potables and assorted snacks.",
  75         "priority": 10,
  76     },
  77     "release": {
  78         "short": "Release automation",
  79         "long": "Commands for used in release automation.",
  80         "priority": 5,
  81     },
  82     "disabled": {
  83         "short": "Disabled",
  84         "long": "The disabled commands are hidden by default. Use -v to display them. "
  85         "These commands are unavailable for your current context, "
  86         'run "mach <command>" to see why.',
  87         "priority": 0,
  88     },
  89 }
  90
  91
  92 def _activate_python_environment(topsrcdir, get_state_dir):
  93     from mach.site import MachSiteManager
  94
  95     mach_environment = MachSiteManager.from_environment(
  96         topsrcdir,
  97         get_state_dir,
  98     )
  99     mach_environment.activate()
 100
 101
 102 def _maybe_activate_mozillabuild_environment():
 103     if sys.platform != "win32":
 104         return
 105
 106     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 107     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 108     assert mozillabuild.exists(), (
 109         f'MozillaBuild was not found at "{mozillabuild}".\n'
 110         "If it's installed in a different location, please "
 111         'set the "MOZILLABUILD" environment variable '
 112         "accordingly."
 113     )
 114
 115     use_msys2 = (mozillabuild / "msys2").exists()
 116     if use_msys2:
 117         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 118     else:
 119         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 120
 121     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 122     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 123     for new_path in paths_to_add:
 124         if new_path not in existing_paths:
 125             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 126
 127
 128 def initialize(topsrcdir):
 129     # This directory was deleted in bug 1666345, but there may be some ignored
 130     # files here. We can safely just delete it for the user so they don't have
 131     # to clean the repo themselves.
 132     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 133     if os.path.exists(deleted_dir):
 134         shutil.rmtree(deleted_dir, ignore_errors=True)
 135
 136     # We need the "mach" module to access the logic to parse virtualenv
 137     # requirements. Since that depends on "packaging" (and, transitively,
 138     # "pyparsing"), we add those to the path too.
 139     sys.path[0:0] = [
 140         os.path.join(topsrcdir, module)
 141         for module in (
 142             os.path.join("python", "mach"),
 143             os.path.join("third_party", "python", "packaging"),
 144             os.path.join("third_party", "python", "pyparsing"),
 145         )
 146     ]
 147
 148     from mach.util import setenv, get_state_dir
 149
 150     state_dir = _create_state_dir()
 151
 152     # normpath state_dir to normalize msys-style slashes.
 153     _activate_python_environment(
 154         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 155     )
 156     _maybe_activate_mozillabuild_environment()
 157
 158     import mach.base
 159     import mach.main
 160
 161     from mach.main import MachCommandReference
 162
 163     # Centralized registry of available mach commands
 164     MACH_COMMANDS = {
 165         "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"),
 166         "devtools-css-db": MachCommandReference(
 167             "devtools/shared/css/generated/mach_commands.py"
 168         ),
 169         "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"),
 170         "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"),
 171         "hazards": MachCommandReference(
 172             "js/src/devtools/rootAnalysis/mach_commands.py"
 173         ),
 174         "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 175         "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 176         "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
 177         "android": MachCommandReference("mobile/android/mach_commands.py"),
 178         "gradle": MachCommandReference("mobile/android/mach_commands.py"),
 179         "gradle-install": MachCommandReference("mobile/android/mach_commands.py"),
 180         "mach-commands": MachCommandReference(
 181             "python/mach/mach/commands/commandinfo.py"
 182         ),
 183         "mach-debug-commands": MachCommandReference(
 184             "python/mach/mach/commands/commandinfo.py"
 185         ),
 186         "mach-completion": MachCommandReference(
 187             "python/mach/mach/commands/commandinfo.py"
 188         ),
 189         "settings": MachCommandReference("python/mach/mach/commands/settings.py"),
 190         "python": MachCommandReference("python/mach_commands.py"),
 191         "python-test": MachCommandReference("python/mach_commands.py"),
 192         "bootstrap": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 193         "vcs-setup": MachCommandReference("python/mozboot/mozboot/mach_commands.py"),
 194         "artifact": MachCommandReference(
 195             "python/mozbuild/mozbuild/artifact_commands.py"
 196         ),
 197         "ide": MachCommandReference(
 198             "python/mozbuild/mozbuild/backend/mach_commands.py"
 199         ),
 200         "build": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 201         "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
 202         "resource-usage": MachCommandReference(
 203             "python/mozbuild/mozbuild/build_commands.py"
 204         ),
 205         "build-backend": MachCommandReference(
 206             "python/mozbuild/mozbuild/build_commands.py"
 207         ),
 208         "clang-tidy": MachCommandReference(
 209             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 210         ),
 211         "static-analysis": MachCommandReference(
 212             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 213         ),
 214         "prettier-format": MachCommandReference(
 215             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 216         ),
 217         "clang-format": MachCommandReference(
 218             "python/mozbuild/mozbuild/code_analysis/mach_commands.py"
 219         ),
 220         "compileflags": MachCommandReference(
 221             "python/mozbuild/mozbuild/compilation/codecomplete.py"
 222         ),
 223         "mozbuild-reference": MachCommandReference(
 224             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 225         ),
 226         "file-info": MachCommandReference(
 227             "python/mozbuild/mozbuild/frontend/mach_commands.py"
 228         ),
 229         "vendor": MachCommandReference(
 230             "python/mozbuild/mozbuild/vendor/mach_commands.py"
 231         ),
 232         "watch": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 233         "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 234         "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 235         "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 236         "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 237         "warnings-summary": MachCommandReference(
 238             "python/mozbuild/mozbuild/mach_commands.py"
 239         ),
 240         "warnings-list": MachCommandReference(
 241             "python/mozbuild/mozbuild/mach_commands.py"
 242         ),
 243         "gtest": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 244         "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 245         "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 246         "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 247         "buildsymbols": MachCommandReference(
 248             "python/mozbuild/mozbuild/mach_commands.py"
 249         ),
 250         "environment": MachCommandReference(
 251             "python/mozbuild/mozbuild/mach_commands.py"
 252         ),
 253         "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
 254         "package-multi-locale": MachCommandReference(
 255             "python/mozbuild/mozbuild/mach_commands.py"
 256         ),
 257         "perftest": MachCommandReference(
 258             "python/mozperftest/mozperftest/mach_commands.py"
 259         ),
 260         "perftest-test": MachCommandReference(
 261             "python/mozperftest/mozperftest/mach_commands.py"
 262         ),
 263         "release": MachCommandReference(
 264             "python/mozrelease/mozrelease/mach_commands.py"
 265         ),
 266         "remote": MachCommandReference("remote/mach_commands.py"),
 267         "puppeteer-test": MachCommandReference("remote/mach_commands.py"),
 268         "generate-test-certs": MachCommandReference(
 269             "security/manager/tools/mach_commands.py"
 270         ),
 271         "taskgraph": MachCommandReference("taskcluster/mach_commands.py"),
 272         "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"),
 273         "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"),
 274         "taskcluster-image-digest": MachCommandReference(
 275             "taskcluster/mach_commands.py"
 276         ),
 277         "release-history": MachCommandReference("taskcluster/mach_commands.py"),
 278         "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"),
 279         "fetch-condprofile": MachCommandReference(
 280             "testing/condprofile/mach_commands.py"
 281         ),
 282         "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
 283         "firefox-ui-functional": MachCommandReference(
 284             "testing/firefox-ui/mach_commands.py"
 285         ),
 286         "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"),
 287         "addtest": MachCommandReference("testing/mach_commands.py"),
 288         "test": MachCommandReference("testing/mach_commands.py"),
 289         "cppunittest": MachCommandReference("testing/mach_commands.py"),
 290         "jstests": MachCommandReference("testing/mach_commands.py"),
 291         "jit-test": MachCommandReference("testing/mach_commands.py"),
 292         "jsapi-tests": MachCommandReference("testing/mach_commands.py"),
 293         "jsshell-bench": MachCommandReference("testing/mach_commands.py"),
 294         "cramtest": MachCommandReference("testing/mach_commands.py"),
 295         "test-info": MachCommandReference("testing/mach_commands.py"),
 296         "rusttests": MachCommandReference("testing/mach_commands.py"),
 297         "fluent-migration-test": MachCommandReference("testing/mach_commands.py"),
 298         "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
 299         "mochitest": MachCommandReference("testing/mochitest/mach_commands.py"),
 300         "geckoview-junit": MachCommandReference("testing/mochitest/mach_commands.py"),
 301         "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"),
 302         "raptor": MachCommandReference("testing/raptor/mach_commands.py"),
 303         "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"),
 304         "talos-test": MachCommandReference("testing/talos/mach_commands.py"),
 305         "tps-build": MachCommandReference("testing/tps/mach_commands.py"),
 306         "web-platform-tests": MachCommandReference(
 307             "testing/web-platform/mach_commands.py"
 308         ),
 309         "wpt": MachCommandReference("testing/web-platform/mach_commands.py"),
 310         "web-platform-tests-update": MachCommandReference(
 311             "testing/web-platform/mach_commands.py"
 312         ),
 313         "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"),
 314         "wpt-manifest-update": MachCommandReference(
 315             "testing/web-platform/mach_commands.py"
 316         ),
 317         "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"),
 318         "wpt-metadata-summary": MachCommandReference(
 319             "testing/web-platform/mach_commands.py"
 320         ),
 321         "wpt-metadata-merge": MachCommandReference(
 322             "testing/web-platform/mach_commands.py"
 323         ),
 324         "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"),
 325         "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"),
 326         "wpt-fission-regressions": MachCommandReference(
 327             "testing/web-platform/mach_commands.py"
 328         ),
 329         "xpcshell-test": MachCommandReference("testing/xpcshell/mach_commands.py"),
 330         "telemetry-tests-client": MachCommandReference(
 331             "toolkit/components/telemetry/tests/marionette/mach_commands.py"
 332         ),
 333         "data-review": MachCommandReference(
 334             "toolkit/components/glean/build_scripts/mach_commands.py"
 335         ),
 336         "update-glean-tags": MachCommandReference(
 337             "toolkit/components/glean/build_scripts/mach_commands.py"
 338         ),
 339         "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"),
 340         "compare-locales": MachCommandReference(
 341             "tools/compare-locales/mach_commands.py"
 342         ),
 343         "l10n-cross-channel": MachCommandReference("tools/lint/mach_commands.py"),
 344         "busted": MachCommandReference("tools/mach_commands.py"),
 345         "pastebin": MachCommandReference("tools/mach_commands.py"),
 346         "mozregression": MachCommandReference("tools/mach_commands.py"),
 347         "node": MachCommandReference("tools/mach_commands.py"),
 348         "npm": MachCommandReference("tools/mach_commands.py"),
 349         "logspam": MachCommandReference("tools/mach_commands.py"),
 350         "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
 351         "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"),
 352         "power": MachCommandReference("tools/power/mach_commands.py"),
 353         "try": MachCommandReference("tools/tryselect/mach_commands.py"),
 354         "import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
 355         "test-interventions": MachCommandReference(
 356             "testing/webcompat/mach_commands.py"
 357         ),
 358     }
 359
 360     # Set a reasonable limit to the number of open files.
 361     #
 362     # Some linux systems set `ulimit -n` to a very high number, which works
 363     # well for systems that run servers, but this setting causes performance
 364     # problems when programs close file descriptors before forking, like
 365     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 366     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 367     # same thing when spawning processes.  We would prefer to lower this limit
 368     # to avoid such performance problems; processes spawned by `mach` will
 369     # inherit the limit set here.
 370     #
 371     # The Firefox build defaults the soft limit to 1024, except for builds that
 372     # do LTO, where the soft limit is 8192.  We're going to default to the
 373     # latter, since people do occasionally do LTO builds on their local
 374     # machines, and requiring them to discover another magical setting after
 375     # setting up an LTO build in the first place doesn't seem good.
 376     #
 377     # This code mimics the code in taskcluster/scripts/run-task.
 378     try:
 379         import resource
 380
 381         # Keep the hard limit the same, though, allowing processes to change
 382         # their soft limit if they need to (Firefox does, for instance).
 383         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 384         # Permit people to override our default limit if necessary via
 385         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 386         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 387         if limit:
 388             limit = int(limit)
 389         else:
 390             # If no explicit limit is given, use our default if it's less than
 391             # the current soft limit.  For instance, the default on macOS is
 392             # 256, so we'd pick that rather than our default.
 393             limit = min(soft, 8192)
 394         # Now apply the limit, if it's different from the original one.
 395         if limit != soft:
 396             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 397     except ImportError:
 398         # The resource module is UNIX only.
 399         pass
 400
 401     def resolve_repository():
 402         import mozversioncontrol
 403
 404         try:
 405             # This API doesn't respect the vcs binary choices from configure.
 406             # If we ever need to use the VCS binary here, consider something
 407             # more robust.
 408             return mozversioncontrol.get_repository_object(path=topsrcdir)
 409         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 410             return None
 411
 412     def pre_dispatch_handler(context, handler, args):
 413         # If --disable-tests flag was enabled in the mozconfig used to compile
 414         # the build, tests will be disabled. Instead of trying to run
 415         # nonexistent tests then reporting a failure, this will prevent mach
 416         # from progressing beyond this point.
 417         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 418             from mozbuild.base import BuildEnvironmentNotFoundException
 419
 420             try:
 421                 from mozbuild.base import MozbuildObject
 422
 423                 # all environments should have an instance of build object.
 424                 build = MozbuildObject.from_environment()
 425                 if build is not None and not getattr(
 426                     build, "substs", {"ENABLE_TESTS": True}
 427                 ).get("ENABLE_TESTS"):
 428                     print(
 429                         "Tests have been disabled with --disable-tests.\n"
 430                         + "Remove the flag, and re-compile to enable tests."
 431                     )
 432                     sys.exit(1)
 433             except BuildEnvironmentNotFoundException:
 434                 # likely automation environment, so do nothing.
 435                 pass
 436
 437     def post_dispatch_handler(
 438         context, handler, instance, success, start_time, end_time, depth, args
 439     ):
 440         """Perform global operations after command dispatch.
 441
 442
 443         For now,  we will use this to handle build system telemetry.
 444         """
 445
 446         # Don't finalize telemetry data if this mach command was invoked as part of
 447         # another mach command.
 448         if depth != 1:
 449             return
 450
 451         _finalize_telemetry_glean(
 452             context.telemetry, handler.name == "bootstrap", success
 453         )
 454
 455     def populate_context(key=None):
 456         if key is None:
 457             return
 458         if key == "state_dir":
 459             return state_dir
 460
 461         if key == "local_state_dir":
 462             return get_state_dir(specific_to_topsrcdir=True)
 463
 464         if key == "topdir":
 465             return topsrcdir
 466
 467         if key == "pre_dispatch_handler":
 468             return pre_dispatch_handler
 469
 470         if key == "post_dispatch_handler":
 471             return post_dispatch_handler
 472
 473         if key == "repository":
 474             return resolve_repository()
 475
 476         raise AttributeError(key)
 477
 478     # Note which process is top-level so that recursive mach invocations can avoid writing
 479     # telemetry data.
 480     if "MACH_MAIN_PID" not in os.environ:
 481         setenv("MACH_MAIN_PID", str(os.getpid()))
 482
 483     driver = mach.main.Mach(os.getcwd())
 484     driver.populate_context_handler = populate_context
 485
 486     if not driver.settings_paths:
 487         # default global machrc location
 488         driver.settings_paths.append(state_dir)
 489     # always load local repository configuration
 490     driver.settings_paths.append(topsrcdir)
 491
 492     for category, meta in CATEGORIES.items():
 493         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 494
 495     # Sparse checkouts may not have all mach_commands.py files. Ignore
 496     # errors from missing files. Same for spidermonkey tarballs.
 497     repo = resolve_repository()
 498     missing_ok = (
 499         repo is not None and repo.sparse_checkout_present()
 500     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 501
 502     driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok)
 503
 504     return driver
 505
 506
 507 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 508     """Submit telemetry collected by Glean.
 509
 510     Finalizes some metrics (command success state and duration, system information) and
 511     requests Glean to send the collected data.
 512     """
 513
 514     from mach.telemetry import MACH_METRICS_PATH
 515     from mozbuild.telemetry import (
 516         get_cpu_brand,
 517         get_distro_and_version,
 518         get_psutil_stats,
 519         get_shell_info,
 520         get_vscode_running,
 521     )
 522
 523     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 524     mach_metrics.mach.duration.stop()
 525     mach_metrics.mach.success.set(success)
 526     system_metrics = mach_metrics.mach.system
 527     cpu_brand = get_cpu_brand()
 528     if cpu_brand:
 529         system_metrics.cpu_brand.set(cpu_brand)
 530     distro, version = get_distro_and_version()
 531     system_metrics.distro.set(distro)
 532     system_metrics.distro_version.set(version)
 533
 534     vscode_terminal, ssh_connection = get_shell_info()
 535     system_metrics.vscode_terminal.set(vscode_terminal)
 536     system_metrics.ssh_connection.set(ssh_connection)
 537     system_metrics.vscode_running.set(get_vscode_running())
 538
 539     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 540     if has_psutil:
 541         # psutil may not be available (we may not have been able to download
 542         # a wheel or build it from source).
 543         system_metrics.logical_cores.add(logical_cores)
 544         system_metrics.physical_cores.add(physical_cores)
 545         if memory_total is not None:
 546             system_metrics.memory.accumulate(
 547                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 548             )
 549     telemetry.submit(is_bootstrap)
 550
 551
 552 def _create_state_dir():
 553     # Global build system and mach state is stored in a central directory. By
 554     # default, this is ~/.mozbuild. However, it can be defined via an
 555     # environment variable. We detect first run (by lack of this directory
 556     # existing) and notify the user that it will be created. The logic for
 557     # creation is much simpler for the "advanced" environment variable use
 558     # case. For default behavior, we educate users and give them an opportunity
 559     # to react.
 560     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 561     if state_dir:
 562         if not os.path.exists(state_dir):
 563             print(
 564                 "Creating global state directory from environment variable: {}".format(
 565                     state_dir
 566                 )
 567             )
 568     else:
 569         state_dir = os.path.expanduser("~/.mozbuild")
 570         if not os.path.exists(state_dir):
 571             if not os.environ.get("MOZ_AUTOMATION"):
 572                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 573                 try:
 574                     sys.stdin.readline()
 575                     print("\n")
 576                 except KeyboardInterrupt:
 577                     sys.exit(1)
 578
 579             print("Creating default state directory: {}".format(state_dir))
 580
 581     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 582     return state_dir
 583
 584
 585 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 586 # the source directory are essentially ignored. See further below for details
 587 # and caveats.
 588 # Objdirs outside the source directory are ignored because in most cases, if
 589 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 590 class ImportHook(object):
 591     def __init__(self, original_import):
 592         self._original_import = original_import
 593         # Assume the source directory is the parent directory of the one
 594         # containing this file.
 595         self._source_dir = (
 596             os.path.normcase(
 597                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 598             )
 599             + os.sep
 600         )
 601         self._modules = set()
 602
 603     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 604         if sys.version_info[0] >= 3 and level < 0:
 605             level = 0
 606
 607         # name might be a relative import. Instead of figuring out what that
 608         # resolves to, which is complex, just rely on the real import.
 609         # Since we don't know the full module name, we can't check sys.modules,
 610         # so we need to keep track of which modules we've already seen to avoid
 611         # to stat() them again when they are imported multiple times.
 612         module = self._original_import(name, globals, locals, fromlist, level)
 613
 614         # Some tests replace modules in sys.modules with non-module instances.
 615         if not isinstance(module, ModuleType):
 616             return module
 617
 618         resolved_name = module.__name__
 619         if resolved_name in self._modules:
 620             return module
 621         self._modules.add(resolved_name)
 622
 623         # Builtin modules don't have a __file__ attribute.
 624         if not getattr(module, "__file__", None):
 625             return module
 626
 627         # Note: module.__file__ is not always absolute.
 628         path = os.path.normcase(os.path.abspath(module.__file__))
 629         # Note: we could avoid normcase and abspath above for non pyc/pyo
 630         # files, but those are actually rare, so it doesn't really matter.
 631         if not path.endswith((".pyc", ".pyo")):
 632             return module
 633
 634         # Ignore modules outside our source directory
 635         if not path.startswith(self._source_dir):
 636             return module
 637
 638         # If there is no .py corresponding to the .pyc/.pyo module we're
 639         # loading, remove the .pyc/.pyo file, and reload the module.
 640         # Since we already loaded the .pyc/.pyo module, if it had side
 641         # effects, they will have happened already, and loading the module
 642         # with the same name, from another directory may have the same side
 643         # effects (or different ones). We assume it's not a problem for the
 644         # python modules under our source directory (either because it
 645         # doesn't happen or because it doesn't matter).
 646         if not os.path.exists(module.__file__[:-1]):
 647             if os.path.exists(module.__file__):
 648                 os.remove(module.__file__)
 649             del sys.modules[module.__name__]
 650             module = self(name, globals, locals, fromlist, level)
 651
 652         return module
 653
 654
 655 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 656 # the source directory are essentially ignored. See further below for details
 657 # and caveats.
 658 # Objdirs outside the source directory are ignored because in most cases, if
 659 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 660 class FinderHook(MetaPathFinder):
 661     def __init__(self, klass):
 662         # Assume the source directory is the parent directory of the one
 663         # containing this file.
 664         self._source_dir = (
 665             os.path.normcase(
 666                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 667             )
 668             + os.sep
 669         )
 670         self.finder_class = klass
 671
 672     def find_spec(self, full_name, paths=None, target=None):
 673         spec = self.finder_class.find_spec(full_name, paths, target)
 674
 675         # Some modules don't have an origin.
 676         if spec is None or spec.origin is None:
 677             return spec
 678
 679         # Normalize the origin path.
 680         path = os.path.normcase(os.path.abspath(spec.origin))
 681         # Note: we could avoid normcase and abspath above for non pyc/pyo
 682         # files, but those are actually rare, so it doesn't really matter.
 683         if not path.endswith((".pyc", ".pyo")):
 684             return spec
 685
 686         # Ignore modules outside our source directory
 687         if not path.startswith(self._source_dir):
 688             return spec
 689
 690         # If there is no .py corresponding to the .pyc/.pyo module we're
 691         # resolving, remove the .pyc/.pyo file, and try again.
 692         if not os.path.exists(spec.origin[:-1]):
 693             if os.path.exists(spec.origin):
 694                 os.remove(spec.origin)
 695             spec = self.finder_class.find_spec(full_name, paths, target)
 696
 697         return spec
 698
 699
 700 # Additional hook for python >= 3.8's importlib.metadata.
 701 class MetadataHook(FinderHook):
 702     def find_distributions(self, *args, **kwargs):
 703         return self.finder_class.find_distributions(*args, **kwargs)
 704
 705
 706 def hook(finder):
 707     has_find_spec = hasattr(finder, "find_spec")
 708     has_find_distributions = hasattr(finder, "find_distributions")
 709     if has_find_spec and has_find_distributions:
 710         return MetadataHook(finder)
 711     elif has_find_spec:
 712         return FinderHook(finder)
 713     return finder
 714
 715
 716 # Install our hook. This can be deleted when the Python 3 migration is complete.
 717 if sys.version_info[0] < 3:
 718     builtins.__import__ = ImportHook(builtins.__import__)
 719 else:
 720     sys.meta_path = [hook(c) for c in sys.meta_path]