build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from pathlib import Path
  10
  11 if sys.version_info[0] < 3:
  12     import __builtin__ as builtins
  13
  14     class MetaPathFinder(object):
  15         pass
  16
  17 else:
  18     from importlib.abc import MetaPathFinder
  19
  20 from types import ModuleType
  21
  22 STATE_DIR_FIRST_RUN = """
  23 Mach and the build system store shared state in a common directory
  24 on the filesystem. The following directory will be created:
  25
  26   {}
  27
  28 If you would like to use a different directory, rename or move it to your
  29 desired location, and set the MOZBUILD_STATE_PATH environment variable
  30 accordingly.
  31 """.strip()
  32
  33
  34 CATEGORIES = {
  35     "build": {
  36         "short": "Build Commands",
  37         "long": "Interact with the build system",
  38         "priority": 80,
  39     },
  40     "post-build": {
  41         "short": "Post-build Commands",
  42         "long": "Common actions performed after completing a build.",
  43         "priority": 70,
  44     },
  45     "testing": {
  46         "short": "Testing",
  47         "long": "Run tests.",
  48         "priority": 60,
  49     },
  50     "ci": {
  51         "short": "CI",
  52         "long": "Taskcluster commands",
  53         "priority": 59,
  54     },
  55     "devenv": {
  56         "short": "Development Environment",
  57         "long": "Set up and configure your development environment.",
  58         "priority": 50,
  59     },
  60     "build-dev": {
  61         "short": "Low-level Build System Interaction",
  62         "long": "Interact with specific parts of the build system.",
  63         "priority": 20,
  64     },
  65     "misc": {
  66         "short": "Potpourri",
  67         "long": "Potent potables and assorted snacks.",
  68         "priority": 10,
  69     },
  70     "release": {
  71         "short": "Release automation",
  72         "long": "Commands for used in release automation.",
  73         "priority": 5,
  74     },
  75     "disabled": {
  76         "short": "Disabled",
  77         "long": "The disabled commands are hidden by default. Use -v to display them. "
  78         "These commands are unavailable for your current context, "
  79         'run "mach <command>" to see why.',
  80         "priority": 0,
  81     },
  82 }
  83
  84
  85 def _activate_python_environment(topsrcdir, get_state_dir):
  86     from mach.site import MachSiteManager
  87
  88     mach_environment = MachSiteManager.from_environment(
  89         topsrcdir,
  90         get_state_dir,
  91     )
  92     mach_environment.activate()
  93
  94
  95 def _maybe_activate_mozillabuild_environment():
  96     if sys.platform != "win32":
  97         return
  98
  99     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 100     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 101     assert mozillabuild.exists(), (
 102         f'MozillaBuild was not found at "{mozillabuild}".\n'
 103         "If it's installed in a different location, please "
 104         'set the "MOZILLABUILD" environment variable '
 105         "accordingly."
 106     )
 107
 108     use_msys2 = (mozillabuild / "msys2").exists()
 109     if use_msys2:
 110         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 111     else:
 112         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 113
 114     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 115     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 116     for new_path in paths_to_add:
 117         if new_path not in existing_paths:
 118             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 119
 120
 121 def check_for_spaces(topsrcdir):
 122     if " " in topsrcdir:
 123         raise Exception(
 124             f"Your checkout at path '{topsrcdir}' contains a space, which "
 125             f"is not supported. Please move it to somewhere that does not "
 126             f"have a space in the path before rerunning mach."
 127         )
 128
 129     mozillabuild_dir = os.environ.get("MOZILLABUILD", "")
 130     if sys.platform == "win32" and " " in mozillabuild_dir:
 131         raise Exception(
 132             f"Your installation of MozillaBuild appears to be installed on a path that "
 133             f"contains a space ('{mozillabuild_dir}') which is not supported. Please "
 134             f"reinstall MozillaBuild on a path without a space and restart your shell"
 135             f"from the new installation."
 136         )
 137
 138
 139 def initialize(topsrcdir, args=()):
 140     # This directory was deleted in bug 1666345, but there may be some ignored
 141     # files here. We can safely just delete it for the user so they don't have
 142     # to clean the repo themselves.
 143     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 144     if os.path.exists(deleted_dir):
 145         shutil.rmtree(deleted_dir, ignore_errors=True)
 146
 147     # We need the "mach" module to access the logic to parse virtualenv
 148     # requirements. Since that depends on "packaging", we add it to the path too.
 149     sys.path[0:0] = [
 150         os.path.join(topsrcdir, module)
 151         for module in (
 152             os.path.join("python", "mach"),
 153             os.path.join("third_party", "python", "packaging"),
 154         )
 155     ]
 156
 157     from mach.util import get_state_dir, get_virtualenv_base_dir, setenv
 158
 159     state_dir = _create_state_dir()
 160
 161     check_for_spaces(topsrcdir)
 162
 163     # normpath state_dir to normalize msys-style slashes.
 164     _activate_python_environment(
 165         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 166     )
 167     _maybe_activate_mozillabuild_environment()
 168
 169     import mach.main
 170     from mach.command_util import (
 171         MACH_COMMANDS,
 172         DetermineCommandVenvAction,
 173         load_commands_from_spec,
 174     )
 175     from mach.main import get_argument_parser
 176
 177     # Set a reasonable limit to the number of open files.
 178     #
 179     # Some linux systems set `ulimit -n` to a very high number, which works
 180     # well for systems that run servers, but this setting causes performance
 181     # problems when programs close file descriptors before forking, like
 182     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 183     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 184     # same thing when spawning processes.  We would prefer to lower this limit
 185     # to avoid such performance problems; processes spawned by `mach` will
 186     # inherit the limit set here.
 187     #
 188     # The Firefox build defaults the soft limit to 1024, except for builds that
 189     # do LTO, where the soft limit is 8192.  We're going to default to the
 190     # latter, since people do occasionally do LTO builds on their local
 191     # machines, and requiring them to discover another magical setting after
 192     # setting up an LTO build in the first place doesn't seem good.
 193     #
 194     # This code mimics the code in taskcluster/scripts/run-task.
 195     try:
 196         import resource
 197
 198         # Keep the hard limit the same, though, allowing processes to change
 199         # their soft limit if they need to (Firefox does, for instance).
 200         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 201         # Permit people to override our default limit if necessary via
 202         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 203         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 204         if limit:
 205             limit = int(limit)
 206         else:
 207             # If no explicit limit is given, use our default if it's less than
 208             # the current soft limit.  For instance, the default on macOS is
 209             # 256, so we'd pick that rather than our default.
 210             limit = min(soft, 8192)
 211         # Now apply the limit, if it's different from the original one.
 212         if limit != soft:
 213             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 214     except ImportError:
 215         # The resource module is UNIX only.
 216         pass
 217
 218     def resolve_repository():
 219         import mozversioncontrol
 220
 221         try:
 222             # This API doesn't respect the vcs binary choices from configure.
 223             # If we ever need to use the VCS binary here, consider something
 224             # more robust.
 225             return mozversioncontrol.get_repository_object(path=topsrcdir)
 226         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 227             return None
 228
 229     def pre_dispatch_handler(context, handler, args):
 230         # If --disable-tests flag was enabled in the mozconfig used to compile
 231         # the build, tests will be disabled. Instead of trying to run
 232         # nonexistent tests then reporting a failure, this will prevent mach
 233         # from progressing beyond this point.
 234         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 235             from mozbuild.base import BuildEnvironmentNotFoundException
 236
 237             try:
 238                 from mozbuild.base import MozbuildObject
 239
 240                 # all environments should have an instance of build object.
 241                 build = MozbuildObject.from_environment()
 242                 if build is not None and not getattr(
 243                     build, "substs", {"ENABLE_TESTS": True}
 244                 ).get("ENABLE_TESTS"):
 245                     print(
 246                         "Tests have been disabled with --disable-tests.\n"
 247                         + "Remove the flag, and re-compile to enable tests."
 248                     )
 249                     sys.exit(1)
 250             except BuildEnvironmentNotFoundException:
 251                 # likely automation environment, so do nothing.
 252                 pass
 253
 254     def post_dispatch_handler(
 255         context, handler, instance, success, start_time, end_time, depth, args
 256     ):
 257         """Perform global operations after command dispatch.
 258
 259
 260         For now,  we will use this to handle build system telemetry.
 261         """
 262
 263         # Don't finalize telemetry data if this mach command was invoked as part of
 264         # another mach command.
 265         if depth != 1:
 266             return
 267
 268         _finalize_telemetry_glean(
 269             context.telemetry, handler.name == "bootstrap", success
 270         )
 271
 272     def populate_context(key=None):
 273         if key is None:
 274             return
 275         if key == "state_dir":
 276             return state_dir
 277
 278         if key == "local_state_dir":
 279             return get_state_dir(specific_to_topsrcdir=True)
 280
 281         if key == "topdir":
 282             return topsrcdir
 283
 284         if key == "pre_dispatch_handler":
 285             return pre_dispatch_handler
 286
 287         if key == "post_dispatch_handler":
 288             return post_dispatch_handler
 289
 290         if key == "repository":
 291             return resolve_repository()
 292
 293         raise AttributeError(key)
 294
 295     # Note which process is top-level so that recursive mach invocations can avoid writing
 296     # telemetry data.
 297     if "MACH_MAIN_PID" not in os.environ:
 298         setenv("MACH_MAIN_PID", str(os.getpid()))
 299
 300     driver = mach.main.Mach(os.getcwd())
 301     driver.populate_context_handler = populate_context
 302
 303     if not driver.settings_paths:
 304         # default global machrc location
 305         driver.settings_paths.append(state_dir)
 306     # always load local repository configuration
 307     driver.settings_paths.append(topsrcdir)
 308     driver.load_settings()
 309
 310     aliases = driver.settings.alias
 311
 312     parser = get_argument_parser(
 313         action=DetermineCommandVenvAction,
 314         topsrcdir=topsrcdir,
 315     )
 316     from argparse import Namespace
 317
 318     namespace_in = Namespace()
 319     setattr(namespace_in, "mach_command_aliases", aliases)
 320     namespace = parser.parse_args(args, namespace_in)
 321
 322     command_name = getattr(namespace, "command_name", None)
 323     site_name = getattr(namespace, "site_name", "common")
 324     command_site_manager = None
 325
 326     # the 'clobber' command needs to run in the 'mach' venv, so we
 327     # don't want to activate any other virtualenv for it.
 328     if command_name != "clobber":
 329         from mach.site import CommandSiteManager
 330
 331         command_site_manager = CommandSiteManager.from_environment(
 332             topsrcdir,
 333             lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
 334             site_name,
 335             get_virtualenv_base_dir(topsrcdir),
 336         )
 337
 338         command_site_manager.activate()
 339
 340     for category, meta in CATEGORIES.items():
 341         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 342
 343     # Sparse checkouts may not have all mach_commands.py files. Ignore
 344     # errors from missing files. Same for spidermonkey tarballs.
 345     repo = resolve_repository()
 346     if repo != "SOURCE":
 347         missing_ok = (
 348             repo is not None and repo.sparse_checkout_present()
 349         ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 350     else:
 351         missing_ok = ()
 352
 353     commands_that_need_all_modules_loaded = [
 354         "busted",
 355         "help",
 356         "mach-commands",
 357         "mach-completion",
 358         "mach-debug-commands",
 359     ]
 360
 361     def commands_to_load(top_level_command: str):
 362         visited = set()
 363
 364         def find_downstream_commands_recursively(command: str):
 365             if not MACH_COMMANDS.get(command):
 366                 return
 367
 368             if command in visited:
 369                 return
 370
 371             visited.add(command)
 372
 373             for command_dependency in MACH_COMMANDS[command].command_dependencies:
 374                 find_downstream_commands_recursively(command_dependency)
 375
 376         find_downstream_commands_recursively(top_level_command)
 377
 378         return list(visited)
 379
 380     if (
 381         command_name not in MACH_COMMANDS
 382         or command_name in commands_that_need_all_modules_loaded
 383     ):
 384         command_modules_to_load = MACH_COMMANDS
 385     else:
 386         command_names_to_load = commands_to_load(command_name)
 387         command_modules_to_load = {
 388             command_name: MACH_COMMANDS[command_name]
 389             for command_name in command_names_to_load
 390         }
 391
 392     driver.command_site_manager = command_site_manager
 393     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 394
 395     return driver
 396
 397
 398 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 399     """Submit telemetry collected by Glean.
 400
 401     Finalizes some metrics (command success state and duration, system information) and
 402     requests Glean to send the collected data.
 403     """
 404
 405     from mach.telemetry import MACH_METRICS_PATH
 406     from mozbuild.telemetry import (
 407         get_cpu_brand,
 408         get_distro_and_version,
 409         get_psutil_stats,
 410         get_shell_info,
 411         get_vscode_running,
 412     )
 413
 414     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 415     mach_metrics.mach.duration.stop()
 416     mach_metrics.mach.success.set(success)
 417     system_metrics = mach_metrics.mach.system
 418     cpu_brand = get_cpu_brand()
 419     if cpu_brand:
 420         system_metrics.cpu_brand.set(cpu_brand)
 421     distro, version = get_distro_and_version()
 422     system_metrics.distro.set(distro)
 423     system_metrics.distro_version.set(version)
 424
 425     vscode_terminal, ssh_connection = get_shell_info()
 426     system_metrics.vscode_terminal.set(vscode_terminal)
 427     system_metrics.ssh_connection.set(ssh_connection)
 428     system_metrics.vscode_running.set(get_vscode_running())
 429
 430     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 431     if has_psutil:
 432         # psutil may not be available (we may not have been able to download
 433         # a wheel or build it from source).
 434         system_metrics.logical_cores.add(logical_cores)
 435         if physical_cores is not None:
 436             system_metrics.physical_cores.add(physical_cores)
 437         if memory_total is not None:
 438             system_metrics.memory.accumulate(
 439                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 440             )
 441     telemetry.submit(is_bootstrap)
 442
 443
 444 def _create_state_dir():
 445     # Global build system and mach state is stored in a central directory. By
 446     # default, this is ~/.mozbuild. However, it can be defined via an
 447     # environment variable. We detect first run (by lack of this directory
 448     # existing) and notify the user that it will be created. The logic for
 449     # creation is much simpler for the "advanced" environment variable use
 450     # case. For default behavior, we educate users and give them an opportunity
 451     # to react.
 452     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 453     if state_dir:
 454         if not os.path.exists(state_dir):
 455             print(
 456                 "Creating global state directory from environment variable: {}".format(
 457                     state_dir
 458                 )
 459             )
 460     else:
 461         state_dir = os.path.expanduser("~/.mozbuild")
 462         if not os.path.exists(state_dir):
 463             if not os.environ.get("MOZ_AUTOMATION"):
 464                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 465
 466             print("Creating default state directory: {}".format(state_dir))
 467
 468     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 469     return state_dir
 470
 471
 472 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 473 # the source directory are essentially ignored. See further below for details
 474 # and caveats.
 475 # Objdirs outside the source directory are ignored because in most cases, if
 476 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 477 class ImportHook(object):
 478     def __init__(self, original_import):
 479         self._original_import = original_import
 480         # Assume the source directory is the parent directory of the one
 481         # containing this file.
 482         self._source_dir = (
 483             os.path.normcase(
 484                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 485             )
 486             + os.sep
 487         )
 488         self._modules = set()
 489
 490     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 491         if sys.version_info[0] >= 3 and level < 0:
 492             level = 0
 493
 494         # name might be a relative import. Instead of figuring out what that
 495         # resolves to, which is complex, just rely on the real import.
 496         # Since we don't know the full module name, we can't check sys.modules,
 497         # so we need to keep track of which modules we've already seen to avoid
 498         # to stat() them again when they are imported multiple times.
 499         module = self._original_import(name, globals, locals, fromlist, level)
 500
 501         # Some tests replace modules in sys.modules with non-module instances.
 502         if not isinstance(module, ModuleType):
 503             return module
 504
 505         resolved_name = module.__name__
 506         if resolved_name in self._modules:
 507             return module
 508         self._modules.add(resolved_name)
 509
 510         # Builtin modules don't have a __file__ attribute.
 511         if not getattr(module, "__file__", None):
 512             return module
 513
 514         # Note: module.__file__ is not always absolute.
 515         path = os.path.normcase(os.path.abspath(module.__file__))
 516         # Note: we could avoid normcase and abspath above for non pyc/pyo
 517         # files, but those are actually rare, so it doesn't really matter.
 518         if not path.endswith((".pyc", ".pyo")):
 519             return module
 520
 521         # Ignore modules outside our source directory
 522         if not path.startswith(self._source_dir):
 523             return module
 524
 525         # If there is no .py corresponding to the .pyc/.pyo module we're
 526         # loading, remove the .pyc/.pyo file, and reload the module.
 527         # Since we already loaded the .pyc/.pyo module, if it had side
 528         # effects, they will have happened already, and loading the module
 529         # with the same name, from another directory may have the same side
 530         # effects (or different ones). We assume it's not a problem for the
 531         # python modules under our source directory (either because it
 532         # doesn't happen or because it doesn't matter).
 533         if not os.path.exists(module.__file__[:-1]):
 534             if os.path.exists(module.__file__):
 535                 os.remove(module.__file__)
 536             del sys.modules[module.__name__]
 537             module = self(name, globals, locals, fromlist, level)
 538
 539         return module
 540
 541
 542 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 543 # the source directory are essentially ignored. See further below for details
 544 # and caveats.
 545 # Objdirs outside the source directory are ignored because in most cases, if
 546 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 547 class FinderHook(MetaPathFinder):
 548     def __init__(self, klass):
 549         # Assume the source directory is the parent directory of the one
 550         # containing this file.
 551         self._source_dir = (
 552             os.path.normcase(
 553                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 554             )
 555             + os.sep
 556         )
 557         self.finder_class = klass
 558
 559     def find_spec(self, full_name, paths=None, target=None):
 560         spec = self.finder_class.find_spec(full_name, paths, target)
 561
 562         # Some modules don't have an origin.
 563         if spec is None or spec.origin is None:
 564             return spec
 565
 566         # Normalize the origin path.
 567         path = os.path.normcase(os.path.abspath(spec.origin))
 568         # Note: we could avoid normcase and abspath above for non pyc/pyo
 569         # files, but those are actually rare, so it doesn't really matter.
 570         if not path.endswith((".pyc", ".pyo")):
 571             return spec
 572
 573         # Ignore modules outside our source directory
 574         if not path.startswith(self._source_dir):
 575             return spec
 576
 577         # If there is no .py corresponding to the .pyc/.pyo module we're
 578         # resolving, remove the .pyc/.pyo file, and try again.
 579         if not os.path.exists(spec.origin[:-1]):
 580             if os.path.exists(spec.origin):
 581                 os.remove(spec.origin)
 582             spec = self.finder_class.find_spec(full_name, paths, target)
 583
 584         return spec
 585
 586
 587 # Additional hook for python >= 3.8's importlib.metadata.
 588 class MetadataHook(FinderHook):
 589     def find_distributions(self, *args, **kwargs):
 590         return self.finder_class.find_distributions(*args, **kwargs)
 591
 592
 593 def hook(finder):
 594     has_find_spec = hasattr(finder, "find_spec")
 595     has_find_distributions = hasattr(finder, "find_distributions")
 596     if has_find_spec and has_find_distributions:
 597         return MetadataHook(finder)
 598     elif has_find_spec:
 599         return FinderHook(finder)
 600     return finder
 601
 602
 603 # Install our hook. This can be deleted when the Python 3 migration is complete.
 604 if sys.version_info[0] < 3:
 605     builtins.__import__ = ImportHook(builtins.__import__)
 606 else:
 607     sys.meta_path = [hook(c) for c in sys.meta_path]