build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from pathlib import Path
  10
  11 if sys.version_info[0] < 3:
  12     import __builtin__ as builtins
  13
  14     class MetaPathFinder(object):
  15         pass
  16
  17
  18 else:
  19     from importlib.abc import MetaPathFinder
  20
  21 from types import ModuleType
  22
  23 STATE_DIR_FIRST_RUN = """
  24 Mach and the build system store shared state in a common directory
  25 on the filesystem. The following directory will be created:
  26
  27   {}
  28
  29 If you would like to use a different directory, rename or move it to your
  30 desired location, and set the MOZBUILD_STATE_PATH environment variable
  31 accordingly.
  32 """.strip()
  33
  34
  35 CATEGORIES = {
  36     "build": {
  37         "short": "Build Commands",
  38         "long": "Interact with the build system",
  39         "priority": 80,
  40     },
  41     "post-build": {
  42         "short": "Post-build Commands",
  43         "long": "Common actions performed after completing a build.",
  44         "priority": 70,
  45     },
  46     "testing": {
  47         "short": "Testing",
  48         "long": "Run tests.",
  49         "priority": 60,
  50     },
  51     "ci": {
  52         "short": "CI",
  53         "long": "Taskcluster commands",
  54         "priority": 59,
  55     },
  56     "devenv": {
  57         "short": "Development Environment",
  58         "long": "Set up and configure your development environment.",
  59         "priority": 50,
  60     },
  61     "build-dev": {
  62         "short": "Low-level Build System Interaction",
  63         "long": "Interact with specific parts of the build system.",
  64         "priority": 20,
  65     },
  66     "misc": {
  67         "short": "Potpourri",
  68         "long": "Potent potables and assorted snacks.",
  69         "priority": 10,
  70     },
  71     "release": {
  72         "short": "Release automation",
  73         "long": "Commands for used in release automation.",
  74         "priority": 5,
  75     },
  76     "disabled": {
  77         "short": "Disabled",
  78         "long": "The disabled commands are hidden by default. Use -v to display them. "
  79         "These commands are unavailable for your current context, "
  80         'run "mach <command>" to see why.',
  81         "priority": 0,
  82     },
  83 }
  84
  85
  86 def _activate_python_environment(topsrcdir, get_state_dir):
  87     from mach.site import MachSiteManager
  88
  89     mach_environment = MachSiteManager.from_environment(
  90         topsrcdir,
  91         get_state_dir,
  92     )
  93     mach_environment.activate()
  94
  95
  96 def _maybe_activate_mozillabuild_environment():
  97     if sys.platform != "win32":
  98         return
  99
 100     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 101     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 102     assert mozillabuild.exists(), (
 103         f'MozillaBuild was not found at "{mozillabuild}".\n'
 104         "If it's installed in a different location, please "
 105         'set the "MOZILLABUILD" environment variable '
 106         "accordingly."
 107     )
 108
 109     use_msys2 = (mozillabuild / "msys2").exists()
 110     if use_msys2:
 111         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 112     else:
 113         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 114
 115     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 116     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 117     for new_path in paths_to_add:
 118         if new_path not in existing_paths:
 119             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 120
 121
 122 def check_for_spaces(topsrcdir):
 123     if " " in topsrcdir:
 124         raise Exception(
 125             f"Your checkout at path '{topsrcdir}' contains a space, which "
 126             f"is not supported. Please move it to somewhere that does not "
 127             f"have a space in the path before rerunning mach."
 128         )
 129
 130     mozillabuild_dir = os.environ.get("MOZILLABUILD", "")
 131     if sys.platform == "win32" and " " in mozillabuild_dir:
 132         raise Exception(
 133             f"Your installation of MozillaBuild appears to be installed on a path that "
 134             f"contains a space ('{mozillabuild_dir}') which is not supported. Please "
 135             f"reinstall MozillaBuild on a path without a space and restart your shell"
 136             f"from the new installation."
 137         )
 138
 139
 140 def initialize(topsrcdir, args=()):
 141     # This directory was deleted in bug 1666345, but there may be some ignored
 142     # files here. We can safely just delete it for the user so they don't have
 143     # to clean the repo themselves.
 144     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 145     if os.path.exists(deleted_dir):
 146         shutil.rmtree(deleted_dir, ignore_errors=True)
 147
 148     # We need the "mach" module to access the logic to parse virtualenv
 149     # requirements. Since that depends on "packaging" (and, transitively,
 150     # "pyparsing"), we add those to the path too.
 151     sys.path[0:0] = [
 152         os.path.join(topsrcdir, module)
 153         for module in (
 154             os.path.join("python", "mach"),
 155             os.path.join("third_party", "python", "packaging"),
 156             os.path.join("third_party", "python", "pyparsing"),
 157         )
 158     ]
 159
 160     from mach.util import get_state_dir, get_virtualenv_base_dir, setenv
 161
 162     state_dir = _create_state_dir()
 163
 164     check_for_spaces(topsrcdir)
 165
 166     # normpath state_dir to normalize msys-style slashes.
 167     _activate_python_environment(
 168         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 169     )
 170     _maybe_activate_mozillabuild_environment()
 171
 172     import mach.main
 173     from mach.command_util import (
 174         MACH_COMMANDS,
 175         DetermineCommandVenvAction,
 176         load_commands_from_spec,
 177     )
 178     from mach.main import get_argument_parser
 179
 180     parser = get_argument_parser(
 181         action=DetermineCommandVenvAction,
 182         topsrcdir=topsrcdir,
 183     )
 184     namespace = parser.parse_args()
 185
 186     command_name = getattr(namespace, "command_name", None)
 187     site_name = getattr(namespace, "site_name", "common")
 188     command_site_manager = None
 189
 190     # the 'clobber' command needs to run in the 'mach' venv, so we
 191     # don't want to activate any other virtualenv for it.
 192     if command_name != "clobber":
 193         from mach.site import CommandSiteManager
 194
 195         command_site_manager = CommandSiteManager.from_environment(
 196             topsrcdir,
 197             lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
 198             site_name,
 199             get_virtualenv_base_dir(topsrcdir),
 200         )
 201
 202         command_site_manager.activate()
 203
 204     # Set a reasonable limit to the number of open files.
 205     #
 206     # Some linux systems set `ulimit -n` to a very high number, which works
 207     # well for systems that run servers, but this setting causes performance
 208     # problems when programs close file descriptors before forking, like
 209     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 210     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 211     # same thing when spawning processes.  We would prefer to lower this limit
 212     # to avoid such performance problems; processes spawned by `mach` will
 213     # inherit the limit set here.
 214     #
 215     # The Firefox build defaults the soft limit to 1024, except for builds that
 216     # do LTO, where the soft limit is 8192.  We're going to default to the
 217     # latter, since people do occasionally do LTO builds on their local
 218     # machines, and requiring them to discover another magical setting after
 219     # setting up an LTO build in the first place doesn't seem good.
 220     #
 221     # This code mimics the code in taskcluster/scripts/run-task.
 222     try:
 223         import resource
 224
 225         # Keep the hard limit the same, though, allowing processes to change
 226         # their soft limit if they need to (Firefox does, for instance).
 227         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 228         # Permit people to override our default limit if necessary via
 229         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 230         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 231         if limit:
 232             limit = int(limit)
 233         else:
 234             # If no explicit limit is given, use our default if it's less than
 235             # the current soft limit.  For instance, the default on macOS is
 236             # 256, so we'd pick that rather than our default.
 237             limit = min(soft, 8192)
 238         # Now apply the limit, if it's different from the original one.
 239         if limit != soft:
 240             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 241     except ImportError:
 242         # The resource module is UNIX only.
 243         pass
 244
 245     def resolve_repository():
 246         import mozversioncontrol
 247
 248         try:
 249             # This API doesn't respect the vcs binary choices from configure.
 250             # If we ever need to use the VCS binary here, consider something
 251             # more robust.
 252             return mozversioncontrol.get_repository_object(path=topsrcdir)
 253         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 254             return None
 255
 256     def pre_dispatch_handler(context, handler, args):
 257         # If --disable-tests flag was enabled in the mozconfig used to compile
 258         # the build, tests will be disabled. Instead of trying to run
 259         # nonexistent tests then reporting a failure, this will prevent mach
 260         # from progressing beyond this point.
 261         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 262             from mozbuild.base import BuildEnvironmentNotFoundException
 263
 264             try:
 265                 from mozbuild.base import MozbuildObject
 266
 267                 # all environments should have an instance of build object.
 268                 build = MozbuildObject.from_environment()
 269                 if build is not None and not getattr(
 270                     build, "substs", {"ENABLE_TESTS": True}
 271                 ).get("ENABLE_TESTS"):
 272                     print(
 273                         "Tests have been disabled with --disable-tests.\n"
 274                         + "Remove the flag, and re-compile to enable tests."
 275                     )
 276                     sys.exit(1)
 277             except BuildEnvironmentNotFoundException:
 278                 # likely automation environment, so do nothing.
 279                 pass
 280
 281     def post_dispatch_handler(
 282         context, handler, instance, success, start_time, end_time, depth, args
 283     ):
 284         """Perform global operations after command dispatch.
 285
 286
 287         For now,  we will use this to handle build system telemetry.
 288         """
 289
 290         # Don't finalize telemetry data if this mach command was invoked as part of
 291         # another mach command.
 292         if depth != 1:
 293             return
 294
 295         _finalize_telemetry_glean(
 296             context.telemetry, handler.name == "bootstrap", success
 297         )
 298
 299     def populate_context(key=None):
 300         if key is None:
 301             return
 302         if key == "state_dir":
 303             return state_dir
 304
 305         if key == "local_state_dir":
 306             return get_state_dir(specific_to_topsrcdir=True)
 307
 308         if key == "topdir":
 309             return topsrcdir
 310
 311         if key == "pre_dispatch_handler":
 312             return pre_dispatch_handler
 313
 314         if key == "post_dispatch_handler":
 315             return post_dispatch_handler
 316
 317         if key == "repository":
 318             return resolve_repository()
 319
 320         raise AttributeError(key)
 321
 322     # Note which process is top-level so that recursive mach invocations can avoid writing
 323     # telemetry data.
 324     if "MACH_MAIN_PID" not in os.environ:
 325         setenv("MACH_MAIN_PID", str(os.getpid()))
 326
 327     driver = mach.main.Mach(os.getcwd(), command_site_manager)
 328     driver.populate_context_handler = populate_context
 329
 330     if not driver.settings_paths:
 331         # default global machrc location
 332         driver.settings_paths.append(state_dir)
 333     # always load local repository configuration
 334     driver.settings_paths.append(topsrcdir)
 335
 336     for category, meta in CATEGORIES.items():
 337         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 338
 339     # Sparse checkouts may not have all mach_commands.py files. Ignore
 340     # errors from missing files. Same for spidermonkey tarballs.
 341     repo = resolve_repository()
 342     if repo != "SOURCE":
 343         missing_ok = (
 344             repo is not None and repo.sparse_checkout_present()
 345         ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 346     else:
 347         missing_ok = ()
 348
 349     commands_that_need_all_modules_loaded = [
 350         "busted",
 351         "help",
 352         "mach-commands",
 353         "mach-completion",
 354         "mach-debug-commands",
 355     ]
 356
 357     def commands_to_load(top_level_command: str):
 358         visited = set()
 359
 360         def find_downstream_commands_recursively(command: str):
 361             if not MACH_COMMANDS.get(command):
 362                 return
 363
 364             if command in visited:
 365                 return
 366
 367             visited.add(command)
 368
 369             for command_dependency in MACH_COMMANDS[command].command_dependencies:
 370                 find_downstream_commands_recursively(command_dependency)
 371
 372         find_downstream_commands_recursively(top_level_command)
 373
 374         return list(visited)
 375
 376     if (
 377         command_name not in MACH_COMMANDS
 378         or command_name in commands_that_need_all_modules_loaded
 379     ):
 380         command_modules_to_load = MACH_COMMANDS
 381     else:
 382         command_names_to_load = commands_to_load(command_name)
 383         command_modules_to_load = {
 384             command_name: MACH_COMMANDS[command_name]
 385             for command_name in command_names_to_load
 386         }
 387
 388     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 389
 390     return driver
 391
 392
 393 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 394     """Submit telemetry collected by Glean.
 395
 396     Finalizes some metrics (command success state and duration, system information) and
 397     requests Glean to send the collected data.
 398     """
 399
 400     from mach.telemetry import MACH_METRICS_PATH
 401     from mozbuild.telemetry import (
 402         get_cpu_brand,
 403         get_distro_and_version,
 404         get_psutil_stats,
 405         get_shell_info,
 406         get_vscode_running,
 407     )
 408
 409     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 410     mach_metrics.mach.duration.stop()
 411     mach_metrics.mach.success.set(success)
 412     system_metrics = mach_metrics.mach.system
 413     cpu_brand = get_cpu_brand()
 414     if cpu_brand:
 415         system_metrics.cpu_brand.set(cpu_brand)
 416     distro, version = get_distro_and_version()
 417     system_metrics.distro.set(distro)
 418     system_metrics.distro_version.set(version)
 419
 420     vscode_terminal, ssh_connection = get_shell_info()
 421     system_metrics.vscode_terminal.set(vscode_terminal)
 422     system_metrics.ssh_connection.set(ssh_connection)
 423     system_metrics.vscode_running.set(get_vscode_running())
 424
 425     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 426     if has_psutil:
 427         # psutil may not be available (we may not have been able to download
 428         # a wheel or build it from source).
 429         system_metrics.logical_cores.add(logical_cores)
 430         if physical_cores is not None:
 431             system_metrics.physical_cores.add(physical_cores)
 432         if memory_total is not None:
 433             system_metrics.memory.accumulate(
 434                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 435             )
 436     telemetry.submit(is_bootstrap)
 437
 438
 439 def _create_state_dir():
 440     # Global build system and mach state is stored in a central directory. By
 441     # default, this is ~/.mozbuild. However, it can be defined via an
 442     # environment variable. We detect first run (by lack of this directory
 443     # existing) and notify the user that it will be created. The logic for
 444     # creation is much simpler for the "advanced" environment variable use
 445     # case. For default behavior, we educate users and give them an opportunity
 446     # to react.
 447     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 448     if state_dir:
 449         if not os.path.exists(state_dir):
 450             print(
 451                 "Creating global state directory from environment variable: {}".format(
 452                     state_dir
 453                 )
 454             )
 455     else:
 456         state_dir = os.path.expanduser("~/.mozbuild")
 457         if not os.path.exists(state_dir):
 458             if not os.environ.get("MOZ_AUTOMATION"):
 459                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 460
 461             print("Creating default state directory: {}".format(state_dir))
 462
 463     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 464     return state_dir
 465
 466
 467 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 468 # the source directory are essentially ignored. See further below for details
 469 # and caveats.
 470 # Objdirs outside the source directory are ignored because in most cases, if
 471 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 472 class ImportHook(object):
 473     def __init__(self, original_import):
 474         self._original_import = original_import
 475         # Assume the source directory is the parent directory of the one
 476         # containing this file.
 477         self._source_dir = (
 478             os.path.normcase(
 479                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 480             )
 481             + os.sep
 482         )
 483         self._modules = set()
 484
 485     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 486         if sys.version_info[0] >= 3 and level < 0:
 487             level = 0
 488
 489         # name might be a relative import. Instead of figuring out what that
 490         # resolves to, which is complex, just rely on the real import.
 491         # Since we don't know the full module name, we can't check sys.modules,
 492         # so we need to keep track of which modules we've already seen to avoid
 493         # to stat() them again when they are imported multiple times.
 494         module = self._original_import(name, globals, locals, fromlist, level)
 495
 496         # Some tests replace modules in sys.modules with non-module instances.
 497         if not isinstance(module, ModuleType):
 498             return module
 499
 500         resolved_name = module.__name__
 501         if resolved_name in self._modules:
 502             return module
 503         self._modules.add(resolved_name)
 504
 505         # Builtin modules don't have a __file__ attribute.
 506         if not getattr(module, "__file__", None):
 507             return module
 508
 509         # Note: module.__file__ is not always absolute.
 510         path = os.path.normcase(os.path.abspath(module.__file__))
 511         # Note: we could avoid normcase and abspath above for non pyc/pyo
 512         # files, but those are actually rare, so it doesn't really matter.
 513         if not path.endswith((".pyc", ".pyo")):
 514             return module
 515
 516         # Ignore modules outside our source directory
 517         if not path.startswith(self._source_dir):
 518             return module
 519
 520         # If there is no .py corresponding to the .pyc/.pyo module we're
 521         # loading, remove the .pyc/.pyo file, and reload the module.
 522         # Since we already loaded the .pyc/.pyo module, if it had side
 523         # effects, they will have happened already, and loading the module
 524         # with the same name, from another directory may have the same side
 525         # effects (or different ones). We assume it's not a problem for the
 526         # python modules under our source directory (either because it
 527         # doesn't happen or because it doesn't matter).
 528         if not os.path.exists(module.__file__[:-1]):
 529             if os.path.exists(module.__file__):
 530                 os.remove(module.__file__)
 531             del sys.modules[module.__name__]
 532             module = self(name, globals, locals, fromlist, level)
 533
 534         return module
 535
 536
 537 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 538 # the source directory are essentially ignored. See further below for details
 539 # and caveats.
 540 # Objdirs outside the source directory are ignored because in most cases, if
 541 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 542 class FinderHook(MetaPathFinder):
 543     def __init__(self, klass):
 544         # Assume the source directory is the parent directory of the one
 545         # containing this file.
 546         self._source_dir = (
 547             os.path.normcase(
 548                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 549             )
 550             + os.sep
 551         )
 552         self.finder_class = klass
 553
 554     def find_spec(self, full_name, paths=None, target=None):
 555         spec = self.finder_class.find_spec(full_name, paths, target)
 556
 557         # Some modules don't have an origin.
 558         if spec is None or spec.origin is None:
 559             return spec
 560
 561         # Normalize the origin path.
 562         path = os.path.normcase(os.path.abspath(spec.origin))
 563         # Note: we could avoid normcase and abspath above for non pyc/pyo
 564         # files, but those are actually rare, so it doesn't really matter.
 565         if not path.endswith((".pyc", ".pyo")):
 566             return spec
 567
 568         # Ignore modules outside our source directory
 569         if not path.startswith(self._source_dir):
 570             return spec
 571
 572         # If there is no .py corresponding to the .pyc/.pyo module we're
 573         # resolving, remove the .pyc/.pyo file, and try again.
 574         if not os.path.exists(spec.origin[:-1]):
 575             if os.path.exists(spec.origin):
 576                 os.remove(spec.origin)
 577             spec = self.finder_class.find_spec(full_name, paths, target)
 578
 579         return spec
 580
 581
 582 # Additional hook for python >= 3.8's importlib.metadata.
 583 class MetadataHook(FinderHook):
 584     def find_distributions(self, *args, **kwargs):
 585         return self.finder_class.find_distributions(*args, **kwargs)
 586
 587
 588 def hook(finder):
 589     has_find_spec = hasattr(finder, "find_spec")
 590     has_find_distributions = hasattr(finder, "find_distributions")
 591     if has_find_spec and has_find_distributions:
 592         return MetadataHook(finder)
 593     elif has_find_spec:
 594         return FinderHook(finder)
 595     return finder
 596
 597
 598 # Install our hook. This can be deleted when the Python 3 migration is complete.
 599 if sys.version_info[0] < 3:
 600     builtins.__import__ = ImportHook(builtins.__import__)
 601 else:
 602     sys.meta_path = [hook(c) for c in sys.meta_path]