build/mach_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import math
   8 import os
   9 import platform
  10 import shutil
  11 import site
  12 import sys
  13
  14 if sys.version_info[0] < 3:
  15     import __builtin__ as builtins
  16
  17     class MetaPathFinder(object):
  18         pass
  19
  20
  21 else:
  22     from importlib.abc import MetaPathFinder
  23
  24
  25 from types import ModuleType
  26
  27
  28 STATE_DIR_FIRST_RUN = """
  29 Mach and the build system store shared state in a common directory
  30 on the filesystem. The following directory will be created:
  31
  32   {}
  33
  34 If you would like to use a different directory, hit CTRL+c, set the
  35 MOZBUILD_STATE_PATH environment variable to the directory you'd like to
  36 use, and run Mach again.
  37
  38 Press ENTER/RETURN to continue or CTRL+c to abort.
  39 """.strip()
  40
  41
  42 # Individual files providing mach commands.
  43 MACH_MODULES = [
  44     "build/valgrind/mach_commands.py",
  45     "devtools/shared/css/generated/mach_commands.py",
  46     "dom/bindings/mach_commands.py",
  47     "js/src/devtools/rootAnalysis/mach_commands.py",
  48     "layout/tools/reftest/mach_commands.py",
  49     "mobile/android/mach_commands.py",
  50     "python/mach/mach/commands/commandinfo.py",
  51     "python/mach/mach/commands/settings.py",
  52     "python/mach_commands.py",
  53     "python/mozboot/mozboot/mach_commands.py",
  54     "python/mozbuild/mozbuild/artifact_commands.py",
  55     "python/mozbuild/mozbuild/backend/mach_commands.py",
  56     "python/mozbuild/mozbuild/build_commands.py",
  57     "python/mozbuild/mozbuild/code_analysis/mach_commands.py",
  58     "python/mozbuild/mozbuild/compilation/codecomplete.py",
  59     "python/mozbuild/mozbuild/frontend/mach_commands.py",
  60     "python/mozbuild/mozbuild/vendor/mach_commands.py",
  61     "python/mozbuild/mozbuild/mach_commands.py",
  62     "python/mozperftest/mozperftest/mach_commands.py",
  63     "python/mozrelease/mozrelease/mach_commands.py",
  64     "remote/mach_commands.py",
  65     "security/manager/tools/mach_commands.py",
  66     "taskcluster/mach_commands.py",
  67     "testing/awsy/mach_commands.py",
  68     "testing/condprofile/mach_commands.py",
  69     "testing/firefox-ui/mach_commands.py",
  70     "testing/geckodriver/mach_commands.py",
  71     "testing/mach_commands.py",
  72     "testing/marionette/mach_commands.py",
  73     "testing/mochitest/mach_commands.py",
  74     "testing/mozharness/mach_commands.py",
  75     "testing/raptor/mach_commands.py",
  76     "testing/talos/mach_commands.py",
  77     "testing/tps/mach_commands.py",
  78     "testing/web-platform/mach_commands.py",
  79     "testing/xpcshell/mach_commands.py",
  80     "toolkit/components/telemetry/tests/marionette/mach_commands.py",
  81     "toolkit/components/glean/build_scripts/mach_commands.py",
  82     "tools/browsertime/mach_commands.py",
  83     "tools/compare-locales/mach_commands.py",
  84     "tools/lint/mach_commands.py",
  85     "tools/mach_commands.py",
  86     "tools/moztreedocs/mach_commands.py",
  87     "tools/phabricator/mach_commands.py",
  88     "tools/power/mach_commands.py",
  89     "tools/tryselect/mach_commands.py",
  90     "tools/vcs/mach_commands.py",
  91 ]
  92
  93
  94 CATEGORIES = {
  95     "build": {
  96         "short": "Build Commands",
  97         "long": "Interact with the build system",
  98         "priority": 80,
  99     },
 100     "post-build": {
 101         "short": "Post-build Commands",
 102         "long": "Common actions performed after completing a build.",
 103         "priority": 70,
 104     },
 105     "testing": {
 106         "short": "Testing",
 107         "long": "Run tests.",
 108         "priority": 60,
 109     },
 110     "ci": {
 111         "short": "CI",
 112         "long": "Taskcluster commands",
 113         "priority": 59,
 114     },
 115     "devenv": {
 116         "short": "Development Environment",
 117         "long": "Set up and configure your development environment.",
 118         "priority": 50,
 119     },
 120     "build-dev": {
 121         "short": "Low-level Build System Interaction",
 122         "long": "Interact with specific parts of the build system.",
 123         "priority": 20,
 124     },
 125     "misc": {
 126         "short": "Potpourri",
 127         "long": "Potent potables and assorted snacks.",
 128         "priority": 10,
 129     },
 130     "release": {
 131         "short": "Release automation",
 132         "long": "Commands for used in release automation.",
 133         "priority": 5,
 134     },
 135     "disabled": {
 136         "short": "Disabled",
 137         "long": "The disabled commands are hidden by default. Use -v to display them. "
 138         "These commands are unavailable for your current context, "
 139         'run "mach <command>" to see why.',
 140         "priority": 0,
 141     },
 142 }
 143
 144 INSTALL_PYTHON_GUIDANCE_LINUX = """
 145 See https://firefox-source-docs.mozilla.org/setup/linux_build.html#installingpython
 146 for guidance on how to install Python on your system.
 147 """.strip()
 148
 149 INSTALL_PYTHON_GUIDANCE_OSX = """
 150 See https://firefox-source-docs.mozilla.org/setup/macos_build.html
 151 for guidance on how to prepare your system to build Firefox. Perhaps
 152 you need to update Xcode, or install Python using brew?
 153 """.strip()
 154
 155 INSTALL_PYTHON_GUIDANCE_MOZILLABUILD = """
 156 Python is provided by MozillaBuild; ensure your MozillaBuild
 157 installation is up to date.
 158 See https://firefox-source-docs.mozilla.org/setup/windows_build.html#install-mozillabuild
 159 for details.
 160 """.strip()
 161
 162 INSTALL_PYTHON_GUIDANCE_OTHER = """
 163 We do not have specific instructions for your platform on how to
 164 install Python. You may find Pyenv (https://github.com/pyenv/pyenv)
 165 helpful, if your system package manager does not provide a way to
 166 install a recent enough Python 3.
 167 """.strip()
 168
 169
 170 def _activate_python_environment(topsrcdir):
 171     # We need the "mach" module to access the logic to parse virtualenv
 172     # requirements. Since that depends on "packaging" (and, transitively,
 173     # "pyparsing"), we add those to the path too.
 174     sys.path[0:0] = [
 175         os.path.join(topsrcdir, module)
 176         for module in (
 177             os.path.join("python", "mach"),
 178             os.path.join("third_party", "python", "packaging"),
 179             os.path.join("third_party", "python", "pyparsing"),
 180         )
 181     ]
 182
 183     from mach.requirements import MachEnvRequirements
 184
 185     thunderbird_dir = os.path.join(topsrcdir, "comm")
 186     is_thunderbird = os.path.exists(thunderbird_dir) and bool(
 187         os.listdir(thunderbird_dir)
 188     )
 189
 190     requirements = MachEnvRequirements.from_requirements_definition(
 191         topsrcdir,
 192         is_thunderbird,
 193         True,
 194         os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
 195     )
 196     sys.path[0:0] = [
 197         os.path.join(topsrcdir, pth.path)
 198         for pth in requirements.pth_requirements + requirements.vendored_requirements
 199     ]
 200
 201
 202 def initialize(topsrcdir):
 203     # Ensure we are running Python 3.6+. We run this check as soon as
 204     # possible to avoid a cryptic import/usage error.
 205     if sys.version_info < (3, 6):
 206         print("Python 3.6+ is required to run mach.")
 207         print("You are running Python", platform.python_version())
 208         if sys.platform.startswith("linux"):
 209             print(INSTALL_PYTHON_GUIDANCE_LINUX)
 210         elif sys.platform.startswith("darwin"):
 211             print(INSTALL_PYTHON_GUIDANCE_OSX)
 212         elif "MOZILLABUILD" in os.environ:
 213             print(INSTALL_PYTHON_GUIDANCE_MOZILLABUILD)
 214         else:
 215             print(INSTALL_PYTHON_GUIDANCE_OTHER)
 216         sys.exit(1)
 217
 218     # This directory was deleted in bug 1666345, but there may be some ignored
 219     # files here. We can safely just delete it for the user so they don't have
 220     # to clean the repo themselves.
 221     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 222     if os.path.exists(deleted_dir):
 223         shutil.rmtree(deleted_dir, ignore_errors=True)
 224
 225     if sys.prefix == sys.base_prefix:
 226         # We are not in a virtualenv. Remove global site packages
 227         # from sys.path.
 228         site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
 229         sys.path = [path for path in sys.path if path not in site_paths]
 230
 231     state_dir = _create_state_dir()
 232     _activate_python_environment(topsrcdir)
 233
 234     import mach.base
 235     import mach.main
 236     from mach.util import setenv
 237     from mozboot.util import get_state_dir
 238
 239     # Set a reasonable limit to the number of open files.
 240     #
 241     # Some linux systems set `ulimit -n` to a very high number, which works
 242     # well for systems that run servers, but this setting causes performance
 243     # problems when programs close file descriptors before forking, like
 244     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 245     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 246     # same thing when spawning processes.  We would prefer to lower this limit
 247     # to avoid such performance problems; processes spawned by `mach` will
 248     # inherit the limit set here.
 249     #
 250     # The Firefox build defaults the soft limit to 1024, except for builds that
 251     # do LTO, where the soft limit is 8192.  We're going to default to the
 252     # latter, since people do occasionally do LTO builds on their local
 253     # machines, and requiring them to discover another magical setting after
 254     # setting up an LTO build in the first place doesn't seem good.
 255     #
 256     # This code mimics the code in taskcluster/scripts/run-task.
 257     try:
 258         import resource
 259
 260         # Keep the hard limit the same, though, allowing processes to change
 261         # their soft limit if they need to (Firefox does, for instance).
 262         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 263         # Permit people to override our default limit if necessary via
 264         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 265         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 266         if limit:
 267             limit = int(limit)
 268         else:
 269             # If no explicit limit is given, use our default if it's less than
 270             # the current soft limit.  For instance, the default on macOS is
 271             # 256, so we'd pick that rather than our default.
 272             limit = min(soft, 8192)
 273         # Now apply the limit, if it's different from the original one.
 274         if limit != soft:
 275             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 276     except ImportError:
 277         # The resource module is UNIX only.
 278         pass
 279
 280     def resolve_repository():
 281         import mozversioncontrol
 282
 283         try:
 284             # This API doesn't respect the vcs binary choices from configure.
 285             # If we ever need to use the VCS binary here, consider something
 286             # more robust.
 287             return mozversioncontrol.get_repository_object(path=topsrcdir)
 288         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 289             return None
 290
 291     def pre_dispatch_handler(context, handler, args):
 292         # If --disable-tests flag was enabled in the mozconfig used to compile
 293         # the build, tests will be disabled. Instead of trying to run
 294         # nonexistent tests then reporting a failure, this will prevent mach
 295         # from progressing beyond this point.
 296         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 297             from mozbuild.base import BuildEnvironmentNotFoundException
 298
 299             try:
 300                 from mozbuild.base import MozbuildObject
 301
 302                 # all environments should have an instance of build object.
 303                 build = MozbuildObject.from_environment()
 304                 if build is not None and hasattr(build, "mozconfig"):
 305                     ac_options = build.mozconfig["configure_args"]
 306                     if ac_options and "--disable-tests" in ac_options:
 307                         print(
 308                             "Tests have been disabled by mozconfig with the flag "
 309                             + '"ac_add_options --disable-tests".\n'
 310                             + "Remove the flag, and re-compile to enable tests."
 311                         )
 312                         sys.exit(1)
 313             except BuildEnvironmentNotFoundException:
 314                 # likely automation environment, so do nothing.
 315                 pass
 316
 317     def post_dispatch_handler(
 318         context, handler, instance, success, start_time, end_time, depth, args
 319     ):
 320         """Perform global operations after command dispatch.
 321
 322
 323         For now,  we will use this to handle build system telemetry.
 324         """
 325
 326         # Don't finalize telemetry data if this mach command was invoked as part of
 327         # another mach command.
 328         if depth != 1:
 329             return
 330
 331         _finalize_telemetry_glean(
 332             context.telemetry, handler.name == "bootstrap", success
 333         )
 334
 335     def populate_context(key=None):
 336         if key is None:
 337             return
 338         if key == "state_dir":
 339             return state_dir
 340
 341         if key == "local_state_dir":
 342             return get_state_dir(srcdir=True)
 343
 344         if key == "topdir":
 345             return topsrcdir
 346
 347         if key == "pre_dispatch_handler":
 348             return pre_dispatch_handler
 349
 350         if key == "post_dispatch_handler":
 351             return post_dispatch_handler
 352
 353         if key == "repository":
 354             return resolve_repository()
 355
 356         raise AttributeError(key)
 357
 358     # Note which process is top-level so that recursive mach invocations can avoid writing
 359     # telemetry data.
 360     if "MACH_MAIN_PID" not in os.environ:
 361         setenv("MACH_MAIN_PID", str(os.getpid()))
 362
 363     driver = mach.main.Mach(os.getcwd())
 364     driver.populate_context_handler = populate_context
 365
 366     if not driver.settings_paths:
 367         # default global machrc location
 368         driver.settings_paths.append(state_dir)
 369     # always load local repository configuration
 370     driver.settings_paths.append(topsrcdir)
 371
 372     for category, meta in CATEGORIES.items():
 373         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 374
 375     # Sparse checkouts may not have all mach_commands.py files. Ignore
 376     # errors from missing files. Same for spidermonkey tarballs.
 377     repo = resolve_repository()
 378     missing_ok = (
 379         repo is not None and repo.sparse_checkout_present()
 380     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 381
 382     for path in MACH_MODULES:
 383         try:
 384             driver.load_commands_from_file(os.path.join(topsrcdir, path))
 385         except mach.base.MissingFileError:
 386             if not missing_ok:
 387                 raise
 388
 389     return driver
 390
 391
 392 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 393     """Submit telemetry collected by Glean.
 394
 395     Finalizes some metrics (command success state and duration, system information) and
 396     requests Glean to send the collected data.
 397     """
 398
 399     from mach.telemetry import MACH_METRICS_PATH
 400     from mozbuild.telemetry import (
 401         get_cpu_brand,
 402         get_distro_and_version,
 403         get_psutil_stats,
 404         get_shell_info,
 405         get_vscode_running,
 406     )
 407
 408     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 409     mach_metrics.mach.duration.stop()
 410     mach_metrics.mach.success.set(success)
 411     system_metrics = mach_metrics.mach.system
 412     cpu_brand = get_cpu_brand()
 413     if cpu_brand:
 414         system_metrics.cpu_brand.set(cpu_brand)
 415     distro, version = get_distro_and_version()
 416     system_metrics.distro.set(distro)
 417     system_metrics.distro_version.set(version)
 418
 419     vscode_terminal, ssh_connection = get_shell_info()
 420     system_metrics.vscode_terminal.set(vscode_terminal)
 421     system_metrics.ssh_connection.set(ssh_connection)
 422     system_metrics.vscode_running.set(get_vscode_running())
 423
 424     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 425     if has_psutil:
 426         # psutil may not be available (we allow `mach create-mach-environment`
 427         # to fail to install it).
 428         system_metrics.logical_cores.add(logical_cores)
 429         system_metrics.physical_cores.add(physical_cores)
 430         if memory_total is not None:
 431             system_metrics.memory.accumulate(
 432                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 433             )
 434     telemetry.submit(is_bootstrap)
 435
 436
 437 def _create_state_dir():
 438     # Global build system and mach state is stored in a central directory. By
 439     # default, this is ~/.mozbuild. However, it can be defined via an
 440     # environment variable. We detect first run (by lack of this directory
 441     # existing) and notify the user that it will be created. The logic for
 442     # creation is much simpler for the "advanced" environment variable use
 443     # case. For default behavior, we educate users and give them an opportunity
 444     # to react.
 445     state_dir = os.environ.get("MOZBUILD_STATE_PATH")
 446     if state_dir:
 447         if not os.path.exists(state_dir):
 448             print(
 449                 "Creating global state directory from environment variable: {}".format(
 450                     state_dir
 451                 )
 452             )
 453     else:
 454         state_dir = os.path.expanduser("~/.mozbuild")
 455         if not os.path.exists(state_dir):
 456             if not os.environ.get("MOZ_AUTOMATION"):
 457                 print(STATE_DIR_FIRST_RUN.format(state_dir))
 458                 try:
 459                     sys.stdin.readline()
 460                     print("\n")
 461                 except KeyboardInterrupt:
 462                     sys.exit(1)
 463
 464             print("Creating default state directory: {}".format(state_dir))
 465
 466     os.makedirs(state_dir, mode=0o770, exist_ok=True)
 467     return state_dir
 468
 469
 470 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 471 # the source directory are essentially ignored. See further below for details
 472 # and caveats.
 473 # Objdirs outside the source directory are ignored because in most cases, if
 474 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 475 class ImportHook(object):
 476     def __init__(self, original_import):
 477         self._original_import = original_import
 478         # Assume the source directory is the parent directory of the one
 479         # containing this file.
 480         self._source_dir = (
 481             os.path.normcase(
 482                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 483             )
 484             + os.sep
 485         )
 486         self._modules = set()
 487
 488     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 489         if sys.version_info[0] >= 3 and level < 0:
 490             level = 0
 491
 492         # name might be a relative import. Instead of figuring out what that
 493         # resolves to, which is complex, just rely on the real import.
 494         # Since we don't know the full module name, we can't check sys.modules,
 495         # so we need to keep track of which modules we've already seen to avoid
 496         # to stat() them again when they are imported multiple times.
 497         module = self._original_import(name, globals, locals, fromlist, level)
 498
 499         # Some tests replace modules in sys.modules with non-module instances.
 500         if not isinstance(module, ModuleType):
 501             return module
 502
 503         resolved_name = module.__name__
 504         if resolved_name in self._modules:
 505             return module
 506         self._modules.add(resolved_name)
 507
 508         # Builtin modules don't have a __file__ attribute.
 509         if not getattr(module, "__file__", None):
 510             return module
 511
 512         # Note: module.__file__ is not always absolute.
 513         path = os.path.normcase(os.path.abspath(module.__file__))
 514         # Note: we could avoid normcase and abspath above for non pyc/pyo
 515         # files, but those are actually rare, so it doesn't really matter.
 516         if not path.endswith((".pyc", ".pyo")):
 517             return module
 518
 519         # Ignore modules outside our source directory
 520         if not path.startswith(self._source_dir):
 521             return module
 522
 523         # If there is no .py corresponding to the .pyc/.pyo module we're
 524         # loading, remove the .pyc/.pyo file, and reload the module.
 525         # Since we already loaded the .pyc/.pyo module, if it had side
 526         # effects, they will have happened already, and loading the module
 527         # with the same name, from another directory may have the same side
 528         # effects (or different ones). We assume it's not a problem for the
 529         # python modules under our source directory (either because it
 530         # doesn't happen or because it doesn't matter).
 531         if not os.path.exists(module.__file__[:-1]):
 532             if os.path.exists(module.__file__):
 533                 os.remove(module.__file__)
 534             del sys.modules[module.__name__]
 535             module = self(name, globals, locals, fromlist, level)
 536
 537         return module
 538
 539
 540 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 541 # the source directory are essentially ignored. See further below for details
 542 # and caveats.
 543 # Objdirs outside the source directory are ignored because in most cases, if
 544 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 545 class FinderHook(MetaPathFinder):
 546     def __init__(self, klass):
 547         # Assume the source directory is the parent directory of the one
 548         # containing this file.
 549         self._source_dir = (
 550             os.path.normcase(
 551                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 552             )
 553             + os.sep
 554         )
 555         self.finder_class = klass
 556
 557     def find_spec(self, full_name, paths=None, target=None):
 558         spec = self.finder_class.find_spec(full_name, paths, target)
 559
 560         # Some modules don't have an origin.
 561         if spec is None or spec.origin is None:
 562             return spec
 563
 564         # Normalize the origin path.
 565         path = os.path.normcase(os.path.abspath(spec.origin))
 566         # Note: we could avoid normcase and abspath above for non pyc/pyo
 567         # files, but those are actually rare, so it doesn't really matter.
 568         if not path.endswith((".pyc", ".pyo")):
 569             return spec
 570
 571         # Ignore modules outside our source directory
 572         if not path.startswith(self._source_dir):
 573             return spec
 574
 575         # If there is no .py corresponding to the .pyc/.pyo module we're
 576         # resolving, remove the .pyc/.pyo file, and try again.
 577         if not os.path.exists(spec.origin[:-1]):
 578             if os.path.exists(spec.origin):
 579                 os.remove(spec.origin)
 580             spec = self.finder_class.find_spec(full_name, paths, target)
 581
 582         return spec
 583
 584
 585 # Additional hook for python >= 3.8's importlib.metadata.
 586 class MetadataHook(FinderHook):
 587     def find_distributions(self, *args, **kwargs):
 588         return self.finder_class.find_distributions(*args, **kwargs)
 589
 590
 591 def hook(finder):
 592     has_find_spec = hasattr(finder, "find_spec")
 593     has_find_distributions = hasattr(finder, "find_distributions")
 594     if has_find_spec and has_find_distributions:
 595         return MetadataHook(finder)
 596     elif has_find_spec:
 597         return FinderHook(finder)
 598     return finder
 599
 600
 601 # Install our hook. This can be deleted when the Python 3 migration is complete.
 602 if sys.version_info[0] < 3:
 603     builtins.__import__ = ImportHook(builtins.__import__)
 604 else:
 605     sys.meta_path = [hook(c) for c in sys.meta_path]