build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import math
   6 import os
   7 import shutil
   8 import sys
   9 from importlib.abc import MetaPathFinder
  10 from pathlib import Path
  11
  12 STATE_DIR_FIRST_RUN = """
  13 Mach and the build system store shared state in a common directory
  14 on the filesystem. The following directory will be created:
  15
  16   {}
  17
  18 If you would like to use a different directory, rename or move it to your
  19 desired location, and set the MOZBUILD_STATE_PATH environment variable
  20 accordingly.
  21 """.strip()
  22
  23
  24 CATEGORIES = {
  25     "build": {
  26         "short": "Build Commands",
  27         "long": "Interact with the build system",
  28         "priority": 80,
  29     },
  30     "post-build": {
  31         "short": "Post-build Commands",
  32         "long": "Common actions performed after completing a build.",
  33         "priority": 70,
  34     },
  35     "testing": {
  36         "short": "Testing",
  37         "long": "Run tests.",
  38         "priority": 60,
  39     },
  40     "ci": {
  41         "short": "CI",
  42         "long": "Taskcluster commands",
  43         "priority": 59,
  44     },
  45     "devenv": {
  46         "short": "Development Environment",
  47         "long": "Set up and configure your development environment.",
  48         "priority": 50,
  49     },
  50     "build-dev": {
  51         "short": "Low-level Build System Interaction",
  52         "long": "Interact with specific parts of the build system.",
  53         "priority": 20,
  54     },
  55     "misc": {
  56         "short": "Potpourri",
  57         "long": "Potent potables and assorted snacks.",
  58         "priority": 10,
  59     },
  60     "release": {
  61         "short": "Release automation",
  62         "long": "Commands for used in release automation.",
  63         "priority": 5,
  64     },
  65     "disabled": {
  66         "short": "Disabled",
  67         "long": "The disabled commands are hidden by default. Use -v to display them. "
  68         "These commands are unavailable for your current context, "
  69         'run "mach <command>" to see why.',
  70         "priority": 0,
  71     },
  72 }
  73
  74
  75 def _activate_python_environment(topsrcdir, get_state_dir):
  76     from mach.site import MachSiteManager
  77
  78     mach_environment = MachSiteManager.from_environment(
  79         topsrcdir,
  80         get_state_dir,
  81     )
  82     mach_environment.activate()
  83
  84
  85 def _maybe_activate_mozillabuild_environment():
  86     if sys.platform != "win32":
  87         return
  88
  89     mozillabuild = Path(os.environ.get("MOZILLABUILD", r"C:\mozilla-build"))
  90     os.environ.setdefault("MOZILLABUILD", str(mozillabuild))
  91     assert mozillabuild.exists(), (
  92         f'MozillaBuild was not found at "{mozillabuild}".\n'
  93         "If it's installed in a different location, please "
  94         'set the "MOZILLABUILD" environment variable '
  95         "accordingly."
  96     )
  97
  98     use_msys2 = (mozillabuild / "msys2").exists()
  99     if use_msys2:
 100         mozillabuild_msys_tools_path = mozillabuild / "msys2" / "usr" / "bin"
 101     else:
 102         mozillabuild_msys_tools_path = mozillabuild / "msys" / "bin"
 103
 104     paths_to_add = [mozillabuild_msys_tools_path, mozillabuild / "bin"]
 105     existing_paths = [Path(p) for p in os.environ.get("PATH", "").split(os.pathsep)]
 106     for new_path in paths_to_add:
 107         if new_path not in existing_paths:
 108             os.environ["PATH"] += f"{os.pathsep}{new_path}"
 109
 110
 111 def check_for_spaces(topsrcdir):
 112     if " " in topsrcdir:
 113         raise Exception(
 114             f"Your checkout at path '{topsrcdir}' contains a space, which "
 115             f"is not supported. Please move it to somewhere that does not "
 116             f"have a space in the path before rerunning mach."
 117         )
 118
 119     mozillabuild_dir = os.environ.get("MOZILLABUILD", "")
 120     if sys.platform == "win32" and " " in mozillabuild_dir:
 121         raise Exception(
 122             f"Your installation of MozillaBuild appears to be installed on a path that "
 123             f"contains a space ('{mozillabuild_dir}') which is not supported. Please "
 124             f"reinstall MozillaBuild on a path without a space and restart your shell"
 125             f"from the new installation."
 126         )
 127
 128
 129 def initialize(topsrcdir, args=()):
 130     # This directory was deleted in bug 1666345, but there may be some ignored
 131     # files here. We can safely just delete it for the user so they don't have
 132     # to clean the repo themselves.
 133     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 134     if os.path.exists(deleted_dir):
 135         shutil.rmtree(deleted_dir, ignore_errors=True)
 136
 137     # We need the "mach" module to access the logic to parse virtualenv
 138     # requirements. Since that depends on "packaging", we add it to the path too.
 139     sys.path[0:0] = [
 140         os.path.join(topsrcdir, module)
 141         for module in (
 142             os.path.join("python", "mach"),
 143             os.path.join("third_party", "python", "packaging"),
 144         )
 145     ]
 146
 147     from mach.util import get_state_dir, get_virtualenv_base_dir, setenv
 148
 149     state_dir = _create_state_dir()
 150
 151     check_for_spaces(topsrcdir)
 152
 153     # normpath state_dir to normalize msys-style slashes.
 154     _activate_python_environment(
 155         topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir))
 156     )
 157     _maybe_activate_mozillabuild_environment()
 158
 159     import mach.main
 160     from mach.command_util import (
 161         MACH_COMMANDS,
 162         DetermineCommandVenvAction,
 163         load_commands_from_spec,
 164     )
 165     from mach.main import get_argument_parser
 166
 167     # Set a reasonable limit to the number of open files.
 168     #
 169     # Some linux systems set `ulimit -n` to a very high number, which works
 170     # well for systems that run servers, but this setting causes performance
 171     # problems when programs close file descriptors before forking, like
 172     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 173     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 174     # same thing when spawning processes.  We would prefer to lower this limit
 175     # to avoid such performance problems; processes spawned by `mach` will
 176     # inherit the limit set here.
 177     #
 178     # The Firefox build defaults the soft limit to 1024, except for builds that
 179     # do LTO, where the soft limit is 8192.  We're going to default to the
 180     # latter, since people do occasionally do LTO builds on their local
 181     # machines, and requiring them to discover another magical setting after
 182     # setting up an LTO build in the first place doesn't seem good.
 183     #
 184     # This code mimics the code in taskcluster/scripts/run-task.
 185     try:
 186         import resource
 187
 188         # Keep the hard limit the same, though, allowing processes to change
 189         # their soft limit if they need to (Firefox does, for instance).
 190         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 191         # Permit people to override our default limit if necessary via
 192         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 193         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 194         if limit:
 195             limit = int(limit)
 196         else:
 197             # If no explicit limit is given, use our default if it's less than
 198             # the current soft limit.  For instance, the default on macOS is
 199             # 256, so we'd pick that rather than our default.
 200             limit = min(soft, 8192)
 201         # Now apply the limit, if it's different from the original one.
 202         if limit != soft:
 203             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 204     except ImportError:
 205         # The resource module is UNIX only.
 206         pass
 207
 208     def resolve_repository():
 209         import mozversioncontrol
 210
 211         try:
 212             # This API doesn't respect the vcs binary choices from configure.
 213             # If we ever need to use the VCS binary here, consider something
 214             # more robust.
 215             return mozversioncontrol.get_repository_object(path=topsrcdir)
 216         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 217             return None
 218
 219     def pre_dispatch_handler(context, handler, args):
 220         # If --disable-tests flag was enabled in the mozconfig used to compile
 221         # the build, tests will be disabled. Instead of trying to run
 222         # nonexistent tests then reporting a failure, this will prevent mach
 223         # from progressing beyond this point.
 224         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 225             from mozbuild.base import BuildEnvironmentNotFoundException
 226
 227             try:
 228                 from mozbuild.base import MozbuildObject
 229
 230                 # all environments should have an instance of build object.
 231                 build = MozbuildObject.from_environment()
 232                 if build is not None and not getattr(
 233                     build, "substs", {"ENABLE_TESTS": True}
 234                 ).get("ENABLE_TESTS"):
 235                     print(
 236                         "Tests have been disabled with --disable-tests.\n"
 237                         + "Remove the flag, and re-compile to enable tests."
 238                     )
 239                     sys.exit(1)
 240             except BuildEnvironmentNotFoundException:
 241                 # likely automation environment, so do nothing.
 242                 pass
 243
 244     def post_dispatch_handler(
 245         context, handler, instance, success, start_time, end_time, depth, args
 246     ):
 247         """Perform global operations after command dispatch.
 248
 249
 250         For now,  we will use this to handle build system telemetry.
 251         """
 252
 253         # Don't finalize telemetry data if this mach command was invoked as part of
 254         # another mach command.
 255         if depth != 1:
 256             return
 257
 258         _finalize_telemetry_glean(
 259             context.telemetry, handler.name == "bootstrap", success
 260         )
 261
 262     def populate_context(key=None):
 263         if key is None:
 264             return
 265         if key == "state_dir":
 266             return state_dir
 267
 268         if key == "local_state_dir":
 269             return get_state_dir(specific_to_topsrcdir=True)
 270
 271         if key == "topdir":
 272             return topsrcdir
 273
 274         if key == "pre_dispatch_handler":
 275             return pre_dispatch_handler
 276
 277         if key == "post_dispatch_handler":
 278             return post_dispatch_handler
 279
 280         if key == "repository":
 281             return resolve_repository()
 282
 283         raise AttributeError(key)
 284
 285     # Note which process is top-level so that recursive mach invocations can avoid writing
 286     # telemetry data.
 287     if "MACH_MAIN_PID" not in os.environ:
 288         setenv("MACH_MAIN_PID", str(os.getpid()))
 289
 290     driver = mach.main.Mach(os.getcwd())
 291     driver.populate_context_handler = populate_context
 292
 293     if not driver.settings_paths:
 294         # default global machrc location
 295         driver.settings_paths.append(state_dir)
 296     # always load local repository configuration
 297     driver.settings_paths.append(topsrcdir)
 298     driver.load_settings()
 299
 300     aliases = driver.settings.alias
 301
 302     parser = get_argument_parser(
 303         action=DetermineCommandVenvAction,
 304         topsrcdir=topsrcdir,
 305     )
 306     from argparse import Namespace
 307
 308     from mach.main import (
 309         SUGGESTED_COMMANDS_MESSAGE,
 310         UNKNOWN_COMMAND_ERROR,
 311         UnknownCommandError,
 312     )
 313
 314     namespace_in = Namespace()
 315     setattr(namespace_in, "mach_command_aliases", aliases)
 316
 317     try:
 318         namespace = parser.parse_args(args, namespace_in)
 319     except UnknownCommandError as e:
 320         suggestion_message = (
 321             SUGGESTED_COMMANDS_MESSAGE % (e.verb, ", ".join(e.suggested_commands))
 322             if e.suggested_commands
 323             else ""
 324         )
 325         print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message))
 326         sys.exit(1)
 327
 328     command_name = getattr(namespace, "command_name", None)
 329     site_name = getattr(namespace, "site_name", "common")
 330     command_site_manager = None
 331
 332     # the 'clobber' command needs to run in the 'mach' venv, so we
 333     # don't want to activate any other virtualenv for it.
 334     if command_name != "clobber":
 335         from mach.site import CommandSiteManager
 336
 337         command_site_manager = CommandSiteManager.from_environment(
 338             topsrcdir,
 339             lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)),
 340             site_name,
 341             get_virtualenv_base_dir(topsrcdir),
 342         )
 343
 344         command_site_manager.activate()
 345
 346     for category, meta in CATEGORIES.items():
 347         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 348
 349     # Sparse checkouts may not have all mach_commands.py files. Ignore
 350     # errors from missing files. Same for spidermonkey tarballs.
 351     repo = resolve_repository()
 352     if repo != "SOURCE":
 353         missing_ok = (
 354             repo is not None and repo.sparse_checkout_present()
 355         ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 356     else:
 357         missing_ok = ()
 358
 359     commands_that_need_all_modules_loaded = [
 360         "busted",
 361         "help",
 362         "mach-commands",
 363         "mach-completion",
 364         "mach-debug-commands",
 365     ]
 366
 367     def commands_to_load(top_level_command: str):
 368         visited = set()
 369
 370         def find_downstream_commands_recursively(command: str):
 371             if not MACH_COMMANDS.get(command):
 372                 return
 373
 374             if command in visited:
 375                 return
 376
 377             visited.add(command)
 378
 379             for command_dependency in MACH_COMMANDS[command].command_dependencies:
 380                 find_downstream_commands_recursively(command_dependency)
 381
 382         find_downstream_commands_recursively(top_level_command)
 383
 384         return list(visited)
 385
 386     if (
 387         command_name not in MACH_COMMANDS
 388         or command_name in commands_that_need_all_modules_loaded
 389     ):
 390         command_modules_to_load = MACH_COMMANDS
 391     else:
 392         command_names_to_load = commands_to_load(command_name)
 393         command_modules_to_load = {
 394             command_name: MACH_COMMANDS[command_name]
 395             for command_name in command_names_to_load
 396         }
 397
 398     driver.command_site_manager = command_site_manager
 399     load_commands_from_spec(command_modules_to_load, topsrcdir, missing_ok=missing_ok)
 400
 401     return driver
 402
 403
 404 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 405     """Submit telemetry collected by Glean.
 406
 407     Finalizes some metrics (command success state and duration, system information) and
 408     requests Glean to send the collected data.
 409     """
 410
 411     from mach.telemetry import MACH_METRICS_PATH
 412     from mozbuild.telemetry import (
 413         get_cpu_brand,
 414         get_distro_and_version,
 415         get_psutil_stats,
 416         get_shell_info,
 417         get_vscode_running,
 418     )
 419
 420     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 421     mach_metrics.mach.duration.stop()
 422     mach_metrics.mach.success.set(success)
 423     system_metrics = mach_metrics.mach.system
 424     cpu_brand = get_cpu_brand()
 425     if cpu_brand:
 426         system_metrics.cpu_brand.set(cpu_brand)
 427     distro, version = get_distro_and_version()
 428     system_metrics.distro.set(distro)
 429     system_metrics.distro_version.set(version)
 430
 431     vscode_terminal, ssh_connection = get_shell_info()
 432     system_metrics.vscode_terminal.set(vscode_terminal)
 433     system_metrics.ssh_connection.set(ssh_connection)
 434     system_metrics.vscode_running.set(get_vscode_running())
 435
 436     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 437     if has_psutil:
 438         # psutil may not be available (we may not have been able to download
 439         # a wheel or build it from source).
 440         system_metrics.logical_cores.add(logical_cores)
 441         if physical_cores is not None:
 442             system_metrics.physical_cores.add(physical_cores)
 443         if memory_total is not None:
 444             system_metrics.memory.accumulate(
 445                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 446             )
 447     telemetry.submit(is_bootstrap)
 448
 449
 450 def _create_state_dir():
 451     # Global build system and mach state is stored in a central directory. By
 452     # default, this is ~/.mozbuild. However, it can be defined via an
 453     # environment variable. We detect first run (by lack of this directory
 454     # existing) and notify the user that it will be created. The logic for
 455     # creation is much simpler for the "advanced" environment variable use
 456     # case. For default behavior, we educate users and give them an opportunity
 457     # to react.
 458     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 459     if state_dir:
 460         if not os.path.exists(state_dir):
 461             print(
 462                 "Creating global state directory from environment variable: {}".format(
 463                     state_dir
 464                 )
 465             )
 466     else:
 467         state_dir = os.path.expanduser("~/.mozbuild")
 468         if not os.path.exists(state_dir):
 469             if not os.environ.get("MOZ_AUTOMATION"):
 470                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 471
 472             print("Creating default state directory: {}".format(state_dir))
 473
 474     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 475     return state_dir
 476
 477
 478 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 479 # the source directory are essentially ignored. See further below for details
 480 # and caveats.
 481 # Objdirs outside the source directory are ignored because in most cases, if
 482 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 483 class FinderHook(MetaPathFinder):
 484     def __init__(self, klass):
 485         # Assume the source directory is the parent directory of the one
 486         # containing this file.
 487         self._source_dir = (
 488             os.path.normcase(
 489                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 490             )
 491             + os.sep
 492         )
 493         self.finder_class = klass
 494
 495     def find_spec(self, full_name, paths=None, target=None):
 496         spec = self.finder_class.find_spec(full_name, paths, target)
 497
 498         # Some modules don't have an origin.
 499         if spec is None or spec.origin is None:
 500             return spec
 501
 502         # Normalize the origin path.
 503         path = os.path.normcase(os.path.abspath(spec.origin))
 504         # Note: we could avoid normcase and abspath above for non pyc/pyo
 505         # files, but those are actually rare, so it doesn't really matter.
 506         if not path.endswith((".pyc", ".pyo")):
 507             return spec
 508
 509         # Ignore modules outside our source directory
 510         if not path.startswith(self._source_dir):
 511             return spec
 512
 513         # If there is no .py corresponding to the .pyc/.pyo module we're
 514         # resolving, remove the .pyc/.pyo file, and try again.
 515         if not os.path.exists(spec.origin[:-1]):
 516             if os.path.exists(spec.origin):
 517                 os.remove(spec.origin)
 518             spec = self.finder_class.find_spec(full_name, paths, target)
 519
 520         return spec
 521
 522
 523 # Additional hook for python >= 3.8's importlib.metadata.
 524 class MetadataHook(FinderHook):
 525     def find_distributions(self, *args, **kwargs):
 526         return self.finder_class.find_distributions(*args, **kwargs)
 527
 528
 529 def hook(finder):
 530     has_find_spec = hasattr(finder, "find_spec")
 531     has_find_distributions = hasattr(finder, "find_distributions")
 532     if has_find_spec and has_find_distributions:
 533         return MetadataHook(finder)
 534     elif has_find_spec:
 535         return FinderHook(finder)
 536     return finder
 537
 538
 539 sys.meta_path = [hook(c) for c in sys.meta_path]