build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from pathlib import Path
  10
  11 if sys.version_info[0] < 3:
  12     import __builtin__ as builtins
  13
  14     class MetaPathFinder(object):
  15         pass
  16
  17 else:
  18     from importlib.abc import MetaPathFinder
  19
  20 from types import ModuleType
  21
  22 STATE_DIR_FIRST_RUN = """
  23 Mach and the build system store shared state in a common directory
  24 on the filesystem. The following directory will be created:
  25
  26   {}
  27
  28 If you would like to use a different directory, rename or move it to your
  29 desired location, and set the MOZBUILD_STATE_PATH environment variable
  30 accordingly.
  31 """.strip()
  32
  33
  34 CATEGORIES = {
  35     "build": {
  36         "short": "Build Commands",
  37         "long": "Interact with the build system",
  38         "priority": 80,
  39     },
  40     "post-build": {
  41         "short": "Post-build Commands",
  42         "long": "Common actions performed after completing a build.",
  43         "priority": 70,
  44     },
  45     "testing": {
  46         "short": "Testing",
  47         "long": "Run tests.",
  48         "priority": 60,
  49     },
  50     "ci": {
  51         "short": "CI",
  52         "long": "Taskcluster commands",
  53         "priority": 59,
  54     },
  55     "devenv": {
  56         "short": "Development Environment",
  57         "long": "Set up and configure your development environment.",
  58         "priority": 50,
  59     },
  60     "build-dev": {
  61         "short": "Low-level Build System Interaction",
  62         "long": "Interact with specific parts of the build system.",
  63         "priority": 20,
  64     },
  65     "misc": {
  66         "short": "Potpourri",
  67         "long": "Potent potables and assorted snacks.",
  68         "priority": 10,
  69     },
  70     "release": {
  71         "short": "Release automation",
  72         "long": "Commands for used in release automation.",
  73         "priority": 5,
  74     },
  75     "disabled": {
  76         "short": "Disabled",
  77         "long": "The disabled commands are hidden by default. Use -v to display them. "
  78         "These commands are unavailable for your current context, "
  79         'run "mach <command>" to see why.',
  80         "priority": 0,
  81     },
  82 }
  83
  84
  85 def _activate_python_environment(topsrcdir, get_state_dir):
  86     from mach.site import MachSiteManager
  87
  88     mach_environment = MachSiteManager.from_environment(
  89         topsrcdir,
  90         get_state_dir,
  91     )
  92     mach_environment.activate()
  93
  94
  95 def _maybe_activate_mozillabuild_environment():
  96     if sys.platform != "win32":
  97         return
  98
  99     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
 100     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
 101     assert mozillabuild.exists(), (
 102         f'MozillaBuild was not found at "{mozillabuild}".\n'
 103         "If it's installed in a different location, please "
 104         'set the "MOZILLABUILD" environment variable '
 105         "accordingly."
 106     )
 107
 108     use_msys2 = (mozillabuild / "msys2").exists()
 109     if use_msys2:
 110         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 111     else:
 112         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 113
 114     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 115     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 116     for new_path in paths_to_add:
 117         if new_path not in existing_paths:
 118             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 119
 120
 121 def check_for_spaces(topsrcdir):
 122     if " " in topsrcdir:
 123         raise Exception(
 124             f"Your checkout at path '{topsrcdir}' contains a space, which "
 125             f"is not supported. Please move it to somewhere that does not "
 126             f"have a space in the path before rerunning mach."
 127         )
 128
 129     mozillabuild_dir = os.environ.get("MOZILLABUILD", "")
 130     if sys.platform == "win32" and " " in mozillabuild_dir:
 131         raise Exception(
 132             f"Your installation of MozillaBuild appears to be installed on a path that "
 133             f"contains a space ('{mozillabuild_dir}') which is not supported. Please "
 134             f"reinstall MozillaBuild on a path without a space and restart your shell"
 135             f"from the new installation."
 136         )
 137
 138
 139 def initialize(topsrcdir, args=()):
 140     # This directory was deleted in bug 1666345, but there may be some ignored
 141     # files here. We can safely just delete it for the user so they don't have
 142     # to clean the repo themselves.
 143     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 144     if os.path.exists(deleted_dir):
 145         shutil.rmtree(deleted_dir, ignore_errors=True)
 146
 147     # We need the "mach" module to access the logic to parse virtualenv
 148     # requirements. Since that depends on "packaging", we add it to the path too.
 149     sys.path[0:0] = [
 150         os.path.join(topsrcdir, module)
 151         for module in (
 152             os.path.join("python", "mach"),
 153             os.path.join("third_party", "python", "packaging"),
 154         )
 155     ]
 156
 157     from mach.util import get_state_dir, get_virtualenv_base_dir, setenv
 158
 159     state_dir = _create_state_dir()
 160
 161     check_for_spaces(topsrcdir)
 162
 163     # normpath state_dir to normalize msys-style slashes.
 164     _activate_python_environment(
 165         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 166     )
 167     _maybe_activate_mozillabuild_environment()
 168
 169     import mach.main
 170     from mach.command_util import (
 171         MACH_COMMANDS,
 172         DetermineCommandVenvAction,
 173         load_commands_from_spec,
 174     )
 175     from mach.main import get_argument_parser
 176
 177     parser = get_argument_parser(
 178         action=DetermineCommandVenvAction,
 179         topsrcdir=topsrcdir,
 180     )
 181     namespace = parser.parse_args()
 182
 183     command_name = getattr(namespace, "command_name", None)
 184     site_name = getattr(namespace, "site_name", "common")
 185     command_site_manager = None
 186
 187     # the 'clobber' command needs to run in the 'mach' venv, so we
 188     # don't want to activate any other virtualenv for it.
 189     if command_name != "clobber":
 190         from mach.site import CommandSiteManager
 191
 192         command_site_manager = CommandSiteManager.from_environment(
 193             topsrcdir,
 194             lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
 195             site_name,
 196             get_virtualenv_base_dir(topsrcdir),
 197         )
 198
 199         command_site_manager.activate()
 200
 201     # Set a reasonable limit to the number of open files.
 202     #
 203     # Some linux systems set `ulimit -n` to a very high number, which works
 204     # well for systems that run servers, but this setting causes performance
 205     # problems when programs close file descriptors before forking, like
 206     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 207     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 208     # same thing when spawning processes.  We would prefer to lower this limit
 209     # to avoid such performance problems; processes spawned by `mach` will
 210     # inherit the limit set here.
 211     #
 212     # The Firefox build defaults the soft limit to 1024, except for builds that
 213     # do LTO, where the soft limit is 8192.  We're going to default to the
 214     # latter, since people do occasionally do LTO builds on their local
 215     # machines, and requiring them to discover another magical setting after
 216     # setting up an LTO build in the first place doesn't seem good.
 217     #
 218     # This code mimics the code in taskcluster/scripts/run-task.
 219     try:
 220         import resource
 221
 222         # Keep the hard limit the same, though, allowing processes to change
 223         # their soft limit if they need to (Firefox does, for instance).
 224         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 225         # Permit people to override our default limit if necessary via
 226         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 227         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 228         if limit:
 229             limit = int(limit)
 230         else:
 231             # If no explicit limit is given, use our default if it's less than
 232             # the current soft limit.  For instance, the default on macOS is
 233             # 256, so we'd pick that rather than our default.
 234             limit = min(soft, 8192)
 235         # Now apply the limit, if it's different from the original one.
 236         if limit != soft:
 237             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 238     except ImportError:
 239         # The resource module is UNIX only.
 240         pass
 241
 242     def resolve_repository():
 243         import mozversioncontrol
 244
 245         try:
 246             # This API doesn't respect the vcs binary choices from configure.
 247             # If we ever need to use the VCS binary here, consider something
 248             # more robust.
 249             return mozversioncontrol.get_repository_object(path=topsrcdir)
 250         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 251             return None
 252
 253     def pre_dispatch_handler(context, handler, args):
 254         # If --disable-tests flag was enabled in the mozconfig used to compile
 255         # the build, tests will be disabled. Instead of trying to run
 256         # nonexistent tests then reporting a failure, this will prevent mach
 257         # from progressing beyond this point.
 258         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 259             from mozbuild.base import BuildEnvironmentNotFoundException
 260
 261             try:
 262                 from mozbuild.base import MozbuildObject
 263
 264                 # all environments should have an instance of build object.
 265                 build = MozbuildObject.from_environment()
 266                 if build is not None and not getattr(
 267                     build, "substs", {"ENABLE_TESTS": True}
 268                 ).get("ENABLE_TESTS"):
 269                     print(
 270                         "Tests have been disabled with --disable-tests.\n"
 271                         + "Remove the flag, and re-compile to enable tests."
 272                     )
 273                     sys.exit(1)
 274             except BuildEnvironmentNotFoundException:
 275                 # likely automation environment, so do nothing.
 276                 pass
 277
 278     def post_dispatch_handler(
 279         context, handler, instance, success, start_time, end_time, depth, args
 280     ):
 281         """Perform global operations after command dispatch.
 282
 283
 284         For now,  we will use this to handle build system telemetry.
 285         """
 286
 287         # Don't finalize telemetry data if this mach command was invoked as part of
 288         # another mach command.
 289         if depth != 1:
 290             return
 291
 292         _finalize_telemetry_glean(
 293             context.telemetry, handler.name == "bootstrap", success
 294         )
 295
 296     def populate_context(key=None):
 297         if key is None:
 298             return
 299         if key == "state_dir":
 300             return state_dir
 301
 302         if key == "local_state_dir":
 303             return get_state_dir(specific_to_topsrcdir=True)
 304
 305         if key == "topdir":
 306             return topsrcdir
 307
 308         if key == "pre_dispatch_handler":
 309             return pre_dispatch_handler
 310
 311         if key == "post_dispatch_handler":
 312             return post_dispatch_handler
 313
 314         if key == "repository":
 315             return resolve_repository()
 316
 317         raise AttributeError(key)
 318
 319     # Note which process is top-level so that recursive mach invocations can avoid writing
 320     # telemetry data.
 321     if "MACH_MAIN_PID" not in os.environ:
 322         setenv("MACH_MAIN_PID", str(os.getpid()))
 323
 324     driver = mach.main.Mach(os.getcwd(), command_site_manager)
 325     driver.populate_context_handler = populate_context
 326
 327     if not driver.settings_paths:
 328         # default global machrc location
 329         driver.settings_paths.append(state_dir)
 330     # always load local repository configuration
 331     driver.settings_paths.append(topsrcdir)
 332
 333     for category, meta in CATEGORIES.items():
 334         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 335
 336     # Sparse checkouts may not have all mach_commands.py files. Ignore
 337     # errors from missing files. Same for spidermonkey tarballs.
 338     repo = resolve_repository()
 339     if repo != "SOURCE":
 340         missing_ok = (
 341             repo is not None and repo.sparse_checkout_present()
 342         ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 343     else:
 344         missing_ok = ()
 345
 346     commands_that_need_all_modules_loaded = [
 347         "busted",
 348         "help",
 349         "mach-commands",
 350         "mach-completion",
 351         "mach-debug-commands",
 352     ]
 353
 354     def commands_to_load(top_level_command: str):
 355         visited = set()
 356
 357         def find_downstream_commands_recursively(command: str):
 358             if not MACH_COMMANDS.get(command):
 359                 return
 360
 361             if command in visited:
 362                 return
 363
 364             visited.add(command)
 365
 366             for command_dependency in MACH_COMMANDS[command].command_dependencies:
 367                 find_downstream_commands_recursively(command_dependency)
 368
 369         find_downstream_commands_recursively(top_level_command)
 370
 371         return list(visited)
 372
 373     if (
 374         command_name not in MACH_COMMANDS
 375         or command_name in commands_that_need_all_modules_loaded
 376     ):
 377         command_modules_to_load = MACH_COMMANDS
 378     else:
 379         command_names_to_load = commands_to_load(command_name)
 380         command_modules_to_load = {
 381             command_name: MACH_COMMANDS[command_name]
 382             for command_name in command_names_to_load
 383         }
 384
 385     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 386
 387     return driver
 388
 389
 390 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 391     """Submit telemetry collected by Glean.
 392
 393     Finalizes some metrics (command success state and duration, system information) and
 394     requests Glean to send the collected data.
 395     """
 396
 397     from mach.telemetry import MACH_METRICS_PATH
 398     from mozbuild.telemetry import (
 399         get_cpu_brand,
 400         get_distro_and_version,
 401         get_psutil_stats,
 402         get_shell_info,
 403         get_vscode_running,
 404     )
 405
 406     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 407     mach_metrics.mach.duration.stop()
 408     mach_metrics.mach.success.set(success)
 409     system_metrics = mach_metrics.mach.system
 410     cpu_brand = get_cpu_brand()
 411     if cpu_brand:
 412         system_metrics.cpu_brand.set(cpu_brand)
 413     distro, version = get_distro_and_version()
 414     system_metrics.distro.set(distro)
 415     system_metrics.distro_version.set(version)
 416
 417     vscode_terminal, ssh_connection = get_shell_info()
 418     system_metrics.vscode_terminal.set(vscode_terminal)
 419     system_metrics.ssh_connection.set(ssh_connection)
 420     system_metrics.vscode_running.set(get_vscode_running())
 421
 422     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 423     if has_psutil:
 424         # psutil may not be available (we may not have been able to download
 425         # a wheel or build it from source).
 426         system_metrics.logical_cores.add(logical_cores)
 427         if physical_cores is not None:
 428             system_metrics.physical_cores.add(physical_cores)
 429         if memory_total is not None:
 430             system_metrics.memory.accumulate(
 431                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 432             )
 433     telemetry.submit(is_bootstrap)
 434
 435
 436 def _create_state_dir():
 437     # Global build system and mach state is stored in a central directory. By
 438     # default, this is ~/.mozbuild. However, it can be defined via an
 439     # environment variable. We detect first run (by lack of this directory
 440     # existing) and notify the user that it will be created. The logic for
 441     # creation is much simpler for the "advanced" environment variable use
 442     # case. For default behavior, we educate users and give them an opportunity
 443     # to react.
 444     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 445     if state_dir:
 446         if not os.path.exists(state_dir):
 447             print(
 448                 "Creating global state directory from environment variable: {}".format(
 449                     state_dir
 450                 )
 451             )
 452     else:
 453         state_dir = os.path.expanduser("~/.mozbuild")
 454         if not os.path.exists(state_dir):
 455             if not os.environ.get("MOZ_AUTOMATION"):
 456                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 457
 458             print("Creating default state directory: {}".format(state_dir))
 459
 460     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 461     return state_dir
 462
 463
 464 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 465 # the source directory are essentially ignored. See further below for details
 466 # and caveats.
 467 # Objdirs outside the source directory are ignored because in most cases, if
 468 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 469 class ImportHook(object):
 470     def __init__(self, original_import):
 471         self._original_import = original_import
 472         # Assume the source directory is the parent directory of the one
 473         # containing this file.
 474         self._source_dir = (
 475             os.path.normcase(
 476                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 477             )
 478             + os.sep
 479         )
 480         self._modules = set()
 481
 482     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 483         if sys.version_info[0] >= 3 and level < 0:
 484             level = 0
 485
 486         # name might be a relative import. Instead of figuring out what that
 487         # resolves to, which is complex, just rely on the real import.
 488         # Since we don't know the full module name, we can't check sys.modules,
 489         # so we need to keep track of which modules we've already seen to avoid
 490         # to stat() them again when they are imported multiple times.
 491         module = self._original_import(name, globals, locals, fromlist, level)
 492
 493         # Some tests replace modules in sys.modules with non-module instances.
 494         if not isinstance(module, ModuleType):
 495             return module
 496
 497         resolved_name = module.__name__
 498         if resolved_name in self._modules:
 499             return module
 500         self._modules.add(resolved_name)
 501
 502         # Builtin modules don't have a __file__ attribute.
 503         if not getattr(module, "__file__", None):
 504             return module
 505
 506         # Note: module.__file__ is not always absolute.
 507         path = os.path.normcase(os.path.abspath(module.__file__))
 508         # Note: we could avoid normcase and abspath above for non pyc/pyo
 509         # files, but those are actually rare, so it doesn't really matter.
 510         if not path.endswith((".pyc", ".pyo")):
 511             return module
 512
 513         # Ignore modules outside our source directory
 514         if not path.startswith(self._source_dir):
 515             return module
 516
 517         # If there is no .py corresponding to the .pyc/.pyo module we're
 518         # loading, remove the .pyc/.pyo file, and reload the module.
 519         # Since we already loaded the .pyc/.pyo module, if it had side
 520         # effects, they will have happened already, and loading the module
 521         # with the same name, from another directory may have the same side
 522         # effects (or different ones). We assume it's not a problem for the
 523         # python modules under our source directory (either because it
 524         # doesn't happen or because it doesn't matter).
 525         if not os.path.exists(module.__file__[:-1]):
 526             if os.path.exists(module.__file__):
 527                 os.remove(module.__file__)
 528             del sys.modules[module.__name__]
 529             module = self(name, globals, locals, fromlist, level)
 530
 531         return module
 532
 533
 534 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 535 # the source directory are essentially ignored. See further below for details
 536 # and caveats.
 537 # Objdirs outside the source directory are ignored because in most cases, if
 538 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 539 class FinderHook(MetaPathFinder):
 540     def __init__(self, klass):
 541         # Assume the source directory is the parent directory of the one
 542         # containing this file.
 543         self._source_dir = (
 544             os.path.normcase(
 545                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 546             )
 547             + os.sep
 548         )
 549         self.finder_class = klass
 550
 551     def find_spec(self, full_name, paths=None, target=None):
 552         spec = self.finder_class.find_spec(full_name, paths, target)
 553
 554         # Some modules don't have an origin.
 555         if spec is None or spec.origin is None:
 556             return spec
 557
 558         # Normalize the origin path.
 559         path = os.path.normcase(os.path.abspath(spec.origin))
 560         # Note: we could avoid normcase and abspath above for non pyc/pyo
 561         # files, but those are actually rare, so it doesn't really matter.
 562         if not path.endswith((".pyc", ".pyo")):
 563             return spec
 564
 565         # Ignore modules outside our source directory
 566         if not path.startswith(self._source_dir):
 567             return spec
 568
 569         # If there is no .py corresponding to the .pyc/.pyo module we're
 570         # resolving, remove the .pyc/.pyo file, and try again.
 571         if not os.path.exists(spec.origin[:-1]):
 572             if os.path.exists(spec.origin):
 573                 os.remove(spec.origin)
 574             spec = self.finder_class.find_spec(full_name, paths, target)
 575
 576         return spec
 577
 578
 579 # Additional hook for python >= 3.8's importlib.metadata.
 580 class MetadataHook(FinderHook):
 581     def find_distributions(self, *args, **kwargs):
 582         return self.finder_class.find_distributions(*args, **kwargs)
 583
 584
 585 def hook(finder):
 586     has_find_spec = hasattr(finder, "find_spec")
 587     has_find_distributions = hasattr(finder, "find_distributions")
 588     if has_find_spec and has_find_distributions:
 589         return MetadataHook(finder)
 590     elif has_find_spec:
 591         return FinderHook(finder)
 592     return finder
 593
 594
 595 # Install our hook. This can be deleted when the Python 3 migration is complete.
 596 if sys.version_info[0] < 3:
 597     builtins.__import__ = ImportHook(builtins.__import__)
 598 else:
 599     sys.meta_path = [hook(c) for c in sys.meta_path]