build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from pathlib import Path
  10
  11 if sys.version_info[0] < 3:
  12     import __builtin__ as builtins
  13
  14     class MetaPathFinder(object):
  15         pass
  16
  17 else:
  18     from importlib.abc import MetaPathFinder
  19
  20 from types import ModuleType
  21
  22 STATE_DIR_FIRST_RUN = """
  23 Mach and the build system store shared state in a common directory
  24 on the filesystem. The following directory will be created:
  25
  26   {}
  27
  28 If you would like to use a different directory, rename or move it to your
  29 desired location, and set the MOZBUILD_STATE_PATH environment variable
  30 accordingly.
  31 """.strip()
  32
  33
  34 CATEGORIES = {
  35     "build": {
  36         "short": "Build Commands",
  37         "long": "Interact with the build system",
  38         "priority": 80,
  39     },
  40     "post-build": {
  41         "short": "Post-build Commands",
  42         "long": "Common actions performed after completing a build.",
  43         "priority": 70,
  44     },
  45     "testing": {
  46         "short": "Testing",
  47         "long": "Run tests.",
  48         "priority": 60,
  49     },
  50     "ci": {
  51         "short": "CI",
  52         "long": "Taskcluster commands",
  53         "priority": 59,
  54     },
  55     "devenv": {
  56         "short": "Development Environment",
  57         "long": "Set up and configure your development environment.",
  58         "priority": 50,
  59     },
  60     "build-dev": {
  61         "short": "Low-level Build System Interaction",
  62         "long": "Interact with specific parts of the build system.",
  63         "priority": 20,
  64     },
  65     "misc": {
  66         "short": "Potpourri",
  67         "long": "Potent potables and assorted snacks.",
  68         "priority": 10,
  69     },
  70     "release": {
  71         "short": "Release automation",
  72         "long": "Commands for used in release automation.",
  73         "priority": 5,
  74     },
  75     "disabled": {
  76         "short": "Disabled",
  77         "long": "The disabled commands are hidden by default. Use -v to display them. "
  78         "These commands are unavailable for your current context, "
  79         'run "mach <command>" to see why.',
  80         "priority": 0,
  81     },
  82 }
  83
  84
  85 def _activate_python_environment(topsrcdir, get_state_dir):
  86     from mach.site import MachSiteManager
  87
  88     mach_environment = MachSiteManager.from_environment(
  89         topsrcdir,
  90         get_state_dir,
  91     )
  92     mach_environment.activate()
  93
  94
  95 def _maybe_activate_mozillabuild_environment():
  96     if sys.platform != "win32":
  97         return
  98
  99     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 100     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 101     assert mozillabuild.exists(), (
 102         f'MozillaBuild was not found at "{mozillabuild}".\n'
 103         "If it's installed in a different location, please "
 104         'set the "MOZILLABUILD" environment variable '
 105         "accordingly."
 106     )
 107
 108     use_msys2 = (mozillabuild / "msys2").exists()
 109     if use_msys2:
 110         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 111     else:
 112         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 113
 114     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 115     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 116     for new_path in paths_to_add:
 117         if new_path not in existing_paths:
 118             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 119
 120
 121 def check_for_spaces(topsrcdir):
 122     if " " in topsrcdir:
 123         raise Exception(
 124             f"Your checkout at path '{topsrcdir}' contains a space, which "
 125             f"is not supported. Please move it to somewhere that does not "
 126             f"have a space in the path before rerunning mach."
 127         )
 128
 129     mozillabuild_dir = os.environ.get("MOZILLABUILD", "")
 130     if sys.platform == "win32" and " " in mozillabuild_dir:
 131         raise Exception(
 132             f"Your installation of MozillaBuild appears to be installed on a path that "
 133             f"contains a space ('{mozillabuild_dir}') which is not supported. Please "
 134             f"reinstall MozillaBuild on a path without a space and restart your shell"
 135             f"from the new installation."
 136         )
 137
 138
 139 def initialize(topsrcdir, args=()):
 140     # This directory was deleted in bug 1666345, but there may be some ignored
 141     # files here. We can safely just delete it for the user so they don't have
 142     # to clean the repo themselves.
 143     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 144     if os.path.exists(deleted_dir):
 145         shutil.rmtree(deleted_dir, ignore_errors=True)
 146
 147     # We need the "mach" module to access the logic to parse virtualenv
 148     # requirements. Since that depends on "packaging", we add it to the path too.
 149     sys.path[0:0] = [
 150         os.path.join(topsrcdir, module)
 151         for module in (
 152             os.path.join("python", "mach"),
 153             os.path.join("third_party", "python", "packaging"),
 154         )
 155     ]
 156
 157     from mach.util import get_state_dir, get_virtualenv_base_dir, setenv
 158
 159     state_dir = _create_state_dir()
 160
 161     check_for_spaces(topsrcdir)
 162
 163     # normpath state_dir to normalize msys-style slashes.
 164     _activate_python_environment(
 165         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 166     )
 167     _maybe_activate_mozillabuild_environment()
 168
 169     import mach.main
 170     from mach.command_util import (
 171         MACH_COMMANDS,
 172         DetermineCommandVenvAction,
 173         load_commands_from_spec,
 174     )
 175     from mach.main import get_argument_parser
 176
 177     # Set a reasonable limit to the number of open files.
 178     #
 179     # Some linux systems set `ulimit -n` to a very high number, which works
 180     # well for systems that run servers, but this setting causes performance
 181     # problems when programs close file descriptors before forking, like
 182     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 183     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 184     # same thing when spawning processes.  We would prefer to lower this limit
 185     # to avoid such performance problems; processes spawned by `mach` will
 186     # inherit the limit set here.
 187     #
 188     # The Firefox build defaults the soft limit to 1024, except for builds that
 189     # do LTO, where the soft limit is 8192.  We're going to default to the
 190     # latter, since people do occasionally do LTO builds on their local
 191     # machines, and requiring them to discover another magical setting after
 192     # setting up an LTO build in the first place doesn't seem good.
 193     #
 194     # This code mimics the code in taskcluster/scripts/run-task.
 195     try:
 196         import resource
 197
 198         # Keep the hard limit the same, though, allowing processes to change
 199         # their soft limit if they need to (Firefox does, for instance).
 200         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 201         # Permit people to override our default limit if necessary via
 202         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 203         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 204         if limit:
 205             limit = int(limit)
 206         else:
 207             # If no explicit limit is given, use our default if it's less than
 208             # the current soft limit.  For instance, the default on macOS is
 209             # 256, so we'd pick that rather than our default.
 210             limit = min(soft, 8192)
 211         # Now apply the limit, if it's different from the original one.
 212         if limit != soft:
 213             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 214     except ImportError:
 215         # The resource module is UNIX only.
 216         pass
 217
 218     def resolve_repository():
 219         import mozversioncontrol
 220
 221         try:
 222             # This API doesn't respect the vcs binary choices from configure.
 223             # If we ever need to use the VCS binary here, consider something
 224             # more robust.
 225             return mozversioncontrol.get_repository_object(path=topsrcdir)
 226         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 227             return None
 228
 229     def pre_dispatch_handler(context, handler, args):
 230         # If --disable-tests flag was enabled in the mozconfig used to compile
 231         # the build, tests will be disabled. Instead of trying to run
 232         # nonexistent tests then reporting a failure, this will prevent mach
 233         # from progressing beyond this point.
 234         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 235             from mozbuild.base import BuildEnvironmentNotFoundException
 236
 237             try:
 238                 from mozbuild.base import MozbuildObject
 239
 240                 # all environments should have an instance of build object.
 241                 build = MozbuildObject.from_environment()
 242                 if build is not None and not getattr(
 243                     build, "substs", {"ENABLE_TESTS": True}
 244                 ).get("ENABLE_TESTS"):
 245                     print(
 246                         "Tests have been disabled with --disable-tests.\n"
 247                         + "Remove the flag, and re-compile to enable tests."
 248                     )
 249                     sys.exit(1)
 250             except BuildEnvironmentNotFoundException:
 251                 # likely automation environment, so do nothing.
 252                 pass
 253
 254     def post_dispatch_handler(
 255         context, handler, instance, success, start_time, end_time, depth, args
 256     ):
 257         """Perform global operations after command dispatch.
 258
 259
 260         For now,  we will use this to handle build system telemetry.
 261         """
 262
 263         # Don't finalize telemetry data if this mach command was invoked as part of
 264         # another mach command.
 265         if depth != 1:
 266             return
 267
 268         _finalize_telemetry_glean(
 269             context.telemetry, handler.name == "bootstrap", success
 270         )
 271
 272     def populate_context(key=None):
 273         if key is None:
 274             return
 275         if key == "state_dir":
 276             return state_dir
 277
 278         if key == "local_state_dir":
 279             return get_state_dir(specific_to_topsrcdir=True)
 280
 281         if key == "topdir":
 282             return topsrcdir
 283
 284         if key == "pre_dispatch_handler":
 285             return pre_dispatch_handler
 286
 287         if key == "post_dispatch_handler":
 288             return post_dispatch_handler
 289
 290         if key == "repository":
 291             return resolve_repository()
 292
 293         raise AttributeError(key)
 294
 295     # Note which process is top-level so that recursive mach invocations can avoid writing
 296     # telemetry data.
 297     if "MACH_MAIN_PID" not in os.environ:
 298         setenv("MACH_MAIN_PID", str(os.getpid()))
 299
 300     driver = mach.main.Mach(os.getcwd())
 301     driver.populate_context_handler = populate_context
 302
 303     if not driver.settings_paths:
 304         # default global machrc location
 305         driver.settings_paths.append(state_dir)
 306     # always load local repository configuration
 307     driver.settings_paths.append(topsrcdir)
 308     driver.load_settings()
 309
 310     aliases = driver.settings.alias
 311
 312     parser = get_argument_parser(
 313         action=DetermineCommandVenvAction,
 314         topsrcdir=topsrcdir,
 315     )
 316     from argparse import Namespace
 317
 318     from mach.main import (
 319         SUGGESTED_COMMANDS_MESSAGE,
 320         UNKNOWN_COMMAND_ERROR,
 321         UnknownCommandError,
 322     )
 323
 324     namespace_in = Namespace()
 325     setattr(namespace_in, "mach_command_aliases", aliases)
 326
 327     try:
 328         namespace = parser.parse_args(args, namespace_in)
 329     except UnknownCommandError as e:
 330         suggestion_message = (
 331             SUGGESTED_COMMANDS_MESSAGE % (e.verb, ", ".join(e.suggested_commands))
 332             if e.suggested_commands
 333             else ""
 334         )
 335         print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message))
 336         sys.exit(1)
 337
 338     command_name = getattr(namespace, "command_name", None)
 339     site_name = getattr(namespace, "site_name", "common")
 340     command_site_manager = None
 341
 342     # the 'clobber' command needs to run in the 'mach' venv, so we
 343     # don't want to activate any other virtualenv for it.
 344     if command_name != "clobber":
 345         from mach.site import CommandSiteManager
 346
 347         command_site_manager = CommandSiteManager.from_environment(
 348             topsrcdir,
 349             lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
 350             site_name,
 351             get_virtualenv_base_dir(topsrcdir),
 352         )
 353
 354         command_site_manager.activate()
 355
 356     for category, meta in CATEGORIES.items():
 357         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 358
 359     # Sparse checkouts may not have all mach_commands.py files. Ignore
 360     # errors from missing files. Same for spidermonkey tarballs.
 361     repo = resolve_repository()
 362     if repo != "SOURCE":
 363         missing_ok = (
 364             repo is not None and repo.sparse_checkout_present()
 365         ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 366     else:
 367         missing_ok = ()
 368
 369     commands_that_need_all_modules_loaded = [
 370         "busted",
 371         "help",
 372         "mach-commands",
 373         "mach-completion",
 374         "mach-debug-commands",
 375     ]
 376
 377     def commands_to_load(top_level_command: str):
 378         visited = set()
 379
 380         def find_downstream_commands_recursively(command: str):
 381             if not MACH_COMMANDS.get(command):
 382                 return
 383
 384             if command in visited:
 385                 return
 386
 387             visited.add(command)
 388
 389             for command_dependency in MACH_COMMANDS[command].command_dependencies:
 390                 find_downstream_commands_recursively(command_dependency)
 391
 392         find_downstream_commands_recursively(top_level_command)
 393
 394         return list(visited)
 395
 396     if (
 397         command_name not in MACH_COMMANDS
 398         or command_name in commands_that_need_all_modules_loaded
 399     ):
 400         command_modules_to_load = MACH_COMMANDS
 401     else:
 402         command_names_to_load = commands_to_load(command_name)
 403         command_modules_to_load = {
 404             command_name: MACH_COMMANDS[command_name]
 405             for command_name in command_names_to_load
 406         }
 407
 408     driver.command_site_manager = command_site_manager
 409     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 410
 411     return driver
 412
 413
 414 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 415     """Submit telemetry collected by Glean.
 416
 417     Finalizes some metrics (command success state and duration, system information) and
 418     requests Glean to send the collected data.
 419     """
 420
 421     from mach.telemetry import MACH_METRICS_PATH
 422     from mozbuild.telemetry import (
 423         get_cpu_brand,
 424         get_distro_and_version,
 425         get_psutil_stats,
 426         get_shell_info,
 427         get_vscode_running,
 428     )
 429
 430     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 431     mach_metrics.mach.duration.stop()
 432     mach_metrics.mach.success.set(success)
 433     system_metrics = mach_metrics.mach.system
 434     cpu_brand = get_cpu_brand()
 435     if cpu_brand:
 436         system_metrics.cpu_brand.set(cpu_brand)
 437     distro, version = get_distro_and_version()
 438     system_metrics.distro.set(distro)
 439     system_metrics.distro_version.set(version)
 440
 441     vscode_terminal, ssh_connection = get_shell_info()
 442     system_metrics.vscode_terminal.set(vscode_terminal)
 443     system_metrics.ssh_connection.set(ssh_connection)
 444     system_metrics.vscode_running.set(get_vscode_running())
 445
 446     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 447     if has_psutil:
 448         # psutil may not be available (we may not have been able to download
 449         # a wheel or build it from source).
 450         system_metrics.logical_cores.add(logical_cores)
 451         if physical_cores is not None:
 452             system_metrics.physical_cores.add(physical_cores)
 453         if memory_total is not None:
 454             system_metrics.memory.accumulate(
 455                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 456             )
 457     telemetry.submit(is_bootstrap)
 458
 459
 460 def _create_state_dir():
 461     # Global build system and mach state is stored in a central directory. By
 462     # default, this is ~/.mozbuild. However, it can be defined via an
 463     # environment variable. We detect first run (by lack of this directory
 464     # existing) and notify the user that it will be created. The logic for
 465     # creation is much simpler for the "advanced" environment variable use
 466     # case. For default behavior, we educate users and give them an opportunity
 467     # to react.
 468     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 469     if state_dir:
 470         if not os.path.exists(state_dir):
 471             print(
 472                 "Creating global state directory from environment variable: {}".format(
 473                     state_dir
 474                 )
 475             )
 476     else:
 477         state_dir = os.path.expanduser("~/.mozbuild")
 478         if not os.path.exists(state_dir):
 479             if not os.environ.get("MOZ_AUTOMATION"):
 480                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 481
 482             print("Creating default state directory: {}".format(state_dir))
 483
 484     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 485     return state_dir
 486
 487
 488 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 489 # the source directory are essentially ignored. See further below for details
 490 # and caveats.
 491 # Objdirs outside the source directory are ignored because in most cases, if
 492 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 493 class ImportHook(object):
 494     def __init__(self, original_import):
 495         self._original_import = original_import
 496         # Assume the source directory is the parent directory of the one
 497         # containing this file.
 498         self._source_dir = (
 499             os.path.normcase(
 500                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 501             )
 502             + os.sep
 503         )
 504         self._modules = set()
 505
 506     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 507         if sys.version_info[0] >= 3 and level < 0:
 508             level = 0
 509
 510         # name might be a relative import. Instead of figuring out what that
 511         # resolves to, which is complex, just rely on the real import.
 512         # Since we don't know the full module name, we can't check sys.modules,
 513         # so we need to keep track of which modules we've already seen to avoid
 514         # to stat() them again when they are imported multiple times.
 515         module = self._original_import(name, globals, locals, fromlist, level)
 516
 517         # Some tests replace modules in sys.modules with non-module instances.
 518         if not isinstance(module, ModuleType):
 519             return module
 520
 521         resolved_name = module.__name__
 522         if resolved_name in self._modules:
 523             return module
 524         self._modules.add(resolved_name)
 525
 526         # Builtin modules don't have a __file__ attribute.
 527         if not getattr(module, "__file__", None):
 528             return module
 529
 530         # Note: module.__file__ is not always absolute.
 531         path = os.path.normcase(os.path.abspath(module.__file__))
 532         # Note: we could avoid normcase and abspath above for non pyc/pyo
 533         # files, but those are actually rare, so it doesn't really matter.
 534         if not path.endswith((".pyc", ".pyo")):
 535             return module
 536
 537         # Ignore modules outside our source directory
 538         if not path.startswith(self._source_dir):
 539             return module
 540
 541         # If there is no .py corresponding to the .pyc/.pyo module we're
 542         # loading, remove the .pyc/.pyo file, and reload the module.
 543         # Since we already loaded the .pyc/.pyo module, if it had side
 544         # effects, they will have happened already, and loading the module
 545         # with the same name, from another directory may have the same side
 546         # effects (or different ones). We assume it's not a problem for the
 547         # python modules under our source directory (either because it
 548         # doesn't happen or because it doesn't matter).
 549         if not os.path.exists(module.__file__[:-1]):
 550             if os.path.exists(module.__file__):
 551                 os.remove(module.__file__)
 552             del sys.modules[module.__name__]
 553             module = self(name, globals, locals, fromlist, level)
 554
 555         return module
 556
 557
 558 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 559 # the source directory are essentially ignored. See further below for details
 560 # and caveats.
 561 # Objdirs outside the source directory are ignored because in most cases, if
 562 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 563 class FinderHook(MetaPathFinder):
 564     def __init__(self, klass):
 565         # Assume the source directory is the parent directory of the one
 566         # containing this file.
 567         self._source_dir = (
 568             os.path.normcase(
 569                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 570             )
 571             + os.sep
 572         )
 573         self.finder_class = klass
 574
 575     def find_spec(self, full_name, paths=None, target=None):
 576         spec = self.finder_class.find_spec(full_name, paths, target)
 577
 578         # Some modules don't have an origin.
 579         if spec is None or spec.origin is None:
 580             return spec
 581
 582         # Normalize the origin path.
 583         path = os.path.normcase(os.path.abspath(spec.origin))
 584         # Note: we could avoid normcase and abspath above for non pyc/pyo
 585         # files, but those are actually rare, so it doesn't really matter.
 586         if not path.endswith((".pyc", ".pyo")):
 587             return spec
 588
 589         # Ignore modules outside our source directory
 590         if not path.startswith(self._source_dir):
 591             return spec
 592
 593         # If there is no .py corresponding to the .pyc/.pyo module we're
 594         # resolving, remove the .pyc/.pyo file, and try again.
 595         if not os.path.exists(spec.origin[:-1]):
 596             if os.path.exists(spec.origin):
 597                 os.remove(spec.origin)
 598             spec = self.finder_class.find_spec(full_name, paths, target)
 599
 600         return spec
 601
 602
 603 # Additional hook for python >= 3.8's importlib.metadata.
 604 class MetadataHook(FinderHook):
 605     def find_distributions(self, *args, **kwargs):
 606         return self.finder_class.find_distributions(*args, **kwargs)
 607
 608
 609 def hook(finder):
 610     has_find_spec = hasattr(finder, "find_spec")
 611     has_find_distributions = hasattr(finder, "find_distributions")
 612     if has_find_spec and has_find_distributions:
 613         return MetadataHook(finder)
 614     elif has_find_spec:
 615         return FinderHook(finder)
 616     return finder
 617
 618
 619 # Install our hook. This can be deleted when the Python 3 migration is complete.
 620 if sys.version_info[0] < 3:
 621     builtins.__import__ = ImportHook(builtins.__import__)
 622 else:
 623     sys.meta_path = [hook(c) for c in sys.meta_path]