build/mach_bootstrap.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import math
   8 import os
   9 import platform
  10 import shutil
  11 import site
  12 import sys
  13
  14 if sys.version_info[0] < 3:
  15     import __builtin__ as builtins
  16 else:
  17     import builtins
  18
  19 from types import ModuleType
  20
  21
  22 STATE_DIR_FIRST_RUN = """
  23 mach and the build system store shared state in a common directory on the
  24 filesystem. The following directory will be created:
  25
  26   {userdir}
  27
  28 If you would like to use a different directory, hit CTRL+c and set the
  29 MOZBUILD_STATE_PATH environment variable to the directory you would like to
  30 use and re-run mach. For this change to take effect forever, you'll likely
  31 want to export this environment variable from your shell's init scripts.
  32
  33 Press ENTER/RETURN to continue or CTRL+c to abort.
  34 """.lstrip()
  35
  36
  37 # Individual files providing mach commands.
  38 MACH_MODULES = [
  39     "build/valgrind/mach_commands.py",
  40     "devtools/shared/css/generated/mach_commands.py",
  41     "dom/bindings/mach_commands.py",
  42     "js/src/devtools/rootAnalysis/mach_commands.py",
  43     "layout/tools/reftest/mach_commands.py",
  44     "mobile/android/mach_commands.py",
  45     "python/mach/mach/commands/commandinfo.py",
  46     "python/mach/mach/commands/settings.py",
  47     "python/mach_commands.py",
  48     "python/mozboot/mozboot/mach_commands.py",
  49     "python/mozbuild/mozbuild/artifact_commands.py",
  50     "python/mozbuild/mozbuild/backend/mach_commands.py",
  51     "python/mozbuild/mozbuild/build_commands.py",
  52     "python/mozbuild/mozbuild/code_analysis/mach_commands.py",
  53     "python/mozbuild/mozbuild/compilation/codecomplete.py",
  54     "python/mozbuild/mozbuild/frontend/mach_commands.py",
  55     "python/mozbuild/mozbuild/vendor/mach_commands.py",
  56     "python/mozbuild/mozbuild/mach_commands.py",
  57     "python/mozperftest/mozperftest/mach_commands.py",
  58     "python/mozrelease/mozrelease/mach_commands.py",
  59     "remote/mach_commands.py",
  60     "security/manager/tools/mach_commands.py",
  61     "taskcluster/mach_commands.py",
  62     "testing/awsy/mach_commands.py",
  63     "testing/condprofile/mach_commands.py",
  64     "testing/firefox-ui/mach_commands.py",
  65     "testing/geckodriver/mach_commands.py",
  66     "testing/mach_commands.py",
  67     "testing/marionette/mach_commands.py",
  68     "testing/mochitest/mach_commands.py",
  69     "testing/mozharness/mach_commands.py",
  70     "testing/raptor/mach_commands.py",
  71     "testing/talos/mach_commands.py",
  72     "testing/tps/mach_commands.py",
  73     "testing/web-platform/mach_commands.py",
  74     "testing/xpcshell/mach_commands.py",
  75     "toolkit/components/telemetry/tests/marionette/mach_commands.py",
  76     "tools/browsertime/mach_commands.py",
  77     "tools/compare-locales/mach_commands.py",
  78     "tools/lint/mach_commands.py",
  79     "tools/mach_commands.py",
  80     "tools/moztreedocs/mach_commands.py",
  81     "tools/phabricator/mach_commands.py",
  82     "tools/power/mach_commands.py",
  83     "tools/tryselect/mach_commands.py",
  84     "tools/vcs/mach_commands.py",
  85 ]
  86
  87
  88 CATEGORIES = {
  89     "build": {
  90         "short": "Build Commands",
  91         "long": "Interact with the build system",
  92         "priority": 80,
  93     },
  94     "post-build": {
  95         "short": "Post-build Commands",
  96         "long": "Common actions performed after completing a build.",
  97         "priority": 70,
  98     },
  99     "testing": {
 100         "short": "Testing",
 101         "long": "Run tests.",
 102         "priority": 60,
 103     },
 104     "ci": {
 105         "short": "CI",
 106         "long": "Taskcluster commands",
 107         "priority": 59,
 108     },
 109     "devenv": {
 110         "short": "Development Environment",
 111         "long": "Set up and configure your development environment.",
 112         "priority": 50,
 113     },
 114     "build-dev": {
 115         "short": "Low-level Build System Interaction",
 116         "long": "Interact with specific parts of the build system.",
 117         "priority": 20,
 118     },
 119     "misc": {
 120         "short": "Potpourri",
 121         "long": "Potent potables and assorted snacks.",
 122         "priority": 10,
 123     },
 124     "release": {
 125         "short": "Release automation",
 126         "long": "Commands for used in release automation.",
 127         "priority": 5,
 128     },
 129     "disabled": {
 130         "short": "Disabled",
 131         "long": "The disabled commands are hidden by default. Use -v to display them. "
 132         "These commands are unavailable for your current context, "
 133         'run "mach <command>" to see why.',
 134         "priority": 0,
 135     },
 136 }
 137
 138
 139 def search_path(mozilla_dir, packages_txt):
 140     with open(os.path.join(mozilla_dir, packages_txt)) as f:
 141         packages = [line.rstrip().split(":") for line in f]
 142
 143     def handle_package(package):
 144         if package[0] == "optional":
 145             try:
 146                 for path in handle_package(package[1:]):
 147                     yield path
 148             except Exception:
 149                 pass
 150
 151         if package[0] in ("windows", "!windows"):
 152             for_win = not package[0].startswith("!")
 153             is_win = sys.platform == "win32"
 154             if is_win == for_win:
 155                 for path in handle_package(package[1:]):
 156                     yield path
 157
 158         if package[0] in ("python2", "python3"):
 159             for_python3 = package[0].endswith("3")
 160             is_python3 = sys.version_info[0] > 2
 161             if is_python3 == for_python3:
 162                 for path in handle_package(package[1:]):
 163                     yield path
 164
 165         if package[0] == "packages.txt":
 166             assert len(package) == 2
 167             for p in search_path(mozilla_dir, package[1]):
 168                 yield os.path.join(mozilla_dir, p)
 169
 170         if package[0].endswith(".pth"):
 171             assert len(package) == 2
 172             yield os.path.join(mozilla_dir, package[1])
 173
 174     for package in packages:
 175         for path in handle_package(package):
 176             yield path
 177
 178
 179 def mach_sys_path(mozilla_dir):
 180     return [
 181         os.path.join(mozilla_dir, path)
 182         for path in search_path(mozilla_dir, "build/mach_virtualenv_packages.txt")
 183     ]
 184
 185
 186 def bootstrap(topsrcdir):
 187     # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
 188     # generate a user-friendly error message rather than a cryptic stack trace
 189     # on module import.
 190     major, minor = sys.version_info[:2]
 191     if (major == 2 and minor < 7) or (major == 3 and minor < 5):
 192         print("Python 2.7 or Python 3.5+ is required to run mach.")
 193         print("You are running Python", platform.python_version())
 194         sys.exit(1)
 195
 196     # This directory was deleted in bug 1666345, but there may be some ignored
 197     # files here. We can safely just delete it for the user so they don't have
 198     # to clean the repo themselves.
 199     deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
 200     if os.path.exists(deleted_dir):
 201         shutil.rmtree(deleted_dir, ignore_errors=True)
 202
 203     if major == 3 and sys.prefix == sys.base_prefix:
 204         # We are not in a virtualenv. Remove global site packages
 205         # from sys.path.
 206         # Note that we don't ever invoke mach from a Python 2 virtualenv,
 207         # and "sys.base_prefix" doesn't exist before Python 3.3, so we
 208         # guard with the "major == 3" check.
 209         site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
 210         sys.path = [path for path in sys.path if path not in site_paths]
 211
 212     # Global build system and mach state is stored in a central directory. By
 213     # default, this is ~/.mozbuild. However, it can be defined via an
 214     # environment variable. We detect first run (by lack of this directory
 215     # existing) and notify the user that it will be created. The logic for
 216     # creation is much simpler for the "advanced" environment variable use
 217     # case. For default behavior, we educate users and give them an opportunity
 218     # to react. We always exit after creating the directory because users don't
 219     # like surprises.
 220     sys.path[0:0] = mach_sys_path(topsrcdir)
 221     import mach.base
 222     import mach.main
 223     from mach.util import setenv
 224     from mozboot.util import get_state_dir
 225
 226     # Set a reasonable limit to the number of open files.
 227     #
 228     # Some linux systems set `ulimit -n` to a very high number, which works
 229     # well for systems that run servers, but this setting causes performance
 230     # problems when programs close file descriptors before forking, like
 231     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 232     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 233     # same thing when spawning processes.  We would prefer to lower this limit
 234     # to avoid such performance problems; processes spawned by `mach` will
 235     # inherit the limit set here.
 236     #
 237     # The Firefox build defaults the soft limit to 1024, except for builds that
 238     # do LTO, where the soft limit is 8192.  We're going to default to the
 239     # latter, since people do occasionally do LTO builds on their local
 240     # machines, and requiring them to discover another magical setting after
 241     # setting up an LTO build in the first place doesn't seem good.
 242     #
 243     # This code mimics the code in taskcluster/scripts/run-task.
 244     try:
 245         import resource
 246
 247         # Keep the hard limit the same, though, allowing processes to change
 248         # their soft limit if they need to (Firefox does, for instance).
 249         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 250         # Permit people to override our default limit if necessary via
 251         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 252         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 253         if limit:
 254             limit = int(limit)
 255         else:
 256             # If no explicit limit is given, use our default if it's less than
 257             # the current soft limit.  For instance, the default on macOS is
 258             # 256, so we'd pick that rather than our default.
 259             limit = min(soft, 8192)
 260         # Now apply the limit, if it's different from the original one.
 261         if limit != soft:
 262             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 263     except ImportError:
 264         # The resource module is UNIX only.
 265         pass
 266
 267     from mozbuild.util import patch_main
 268
 269     patch_main()
 270
 271     def resolve_repository():
 272         import mozversioncontrol
 273
 274         try:
 275             # This API doesn't respect the vcs binary choices from configure.
 276             # If we ever need to use the VCS binary here, consider something
 277             # more robust.
 278             return mozversioncontrol.get_repository_object(path=topsrcdir)
 279         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 280             return None
 281
 282     def pre_dispatch_handler(context, handler, args):
 283         # If --disable-tests flag was enabled in the mozconfig used to compile
 284         # the build, tests will be disabled. Instead of trying to run
 285         # nonexistent tests then reporting a failure, this will prevent mach
 286         # from progressing beyond this point.
 287         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 288             from mozbuild.base import BuildEnvironmentNotFoundException
 289
 290             try:
 291                 from mozbuild.base import MozbuildObject
 292
 293                 # all environments should have an instance of build object.
 294                 build = MozbuildObject.from_environment()
 295                 if build is not None and hasattr(build, "mozconfig"):
 296                     ac_options = build.mozconfig["configure_args"]
 297                     if ac_options and "--disable-tests" in ac_options:
 298                         print(
 299                             "Tests have been disabled by mozconfig with the flag "
 300                             + '"ac_add_options --disable-tests".\n'
 301                             + "Remove the flag, and re-compile to enable tests."
 302                         )
 303                         sys.exit(1)
 304             except BuildEnvironmentNotFoundException:
 305                 # likely automation environment, so do nothing.
 306                 pass
 307
 308     def post_dispatch_handler(
 309         context, handler, instance, success, start_time, end_time, depth, args
 310     ):
 311         """Perform global operations after command dispatch.
 312
 313
 314         For now,  we will use this to handle build system telemetry.
 315         """
 316
 317         # Don't finalize telemetry data if this mach command was invoked as part of
 318         # another mach command.
 319         if depth != 1:
 320             return
 321
 322         _finalize_telemetry_glean(
 323             context.telemetry, handler.name == "bootstrap", success
 324         )
 325
 326     def populate_context(key=None):
 327         if key is None:
 328             return
 329         if key == "state_dir":
 330             state_dir = get_state_dir()
 331             if state_dir == os.environ.get("MOZBUILD_STATE_PATH"):
 332                 if not os.path.exists(state_dir):
 333                     print(
 334                         "Creating global state directory from environment variable: %s"
 335                         % state_dir
 336                     )
 337                     os.makedirs(state_dir, mode=0o770)
 338             else:
 339                 if not os.path.exists(state_dir):
 340                     if not os.environ.get("MOZ_AUTOMATION"):
 341                         print(STATE_DIR_FIRST_RUN.format(userdir=state_dir))
 342                         try:
 343                             sys.stdin.readline()
 344                         except KeyboardInterrupt:
 345                             sys.exit(1)
 346
 347                     print("\nCreating default state directory: %s" % state_dir)
 348                     os.makedirs(state_dir, mode=0o770)
 349
 350             return state_dir
 351
 352         if key == "local_state_dir":
 353             return get_state_dir(srcdir=True)
 354
 355         if key == "topdir":
 356             return topsrcdir
 357
 358         if key == "pre_dispatch_handler":
 359             return pre_dispatch_handler
 360
 361         if key == "post_dispatch_handler":
 362             return post_dispatch_handler
 363
 364         if key == "repository":
 365             return resolve_repository()
 366
 367         raise AttributeError(key)
 368
 369     # Note which process is top-level so that recursive mach invocations can avoid writing
 370     # telemetry data.
 371     if "MACH_MAIN_PID" not in os.environ:
 372         setenv("MACH_MAIN_PID", str(os.getpid()))
 373
 374     driver = mach.main.Mach(os.getcwd())
 375     driver.populate_context_handler = populate_context
 376
 377     if not driver.settings_paths:
 378         # default global machrc location
 379         driver.settings_paths.append(get_state_dir())
 380     # always load local repository configuration
 381     driver.settings_paths.append(topsrcdir)
 382
 383     for category, meta in CATEGORIES.items():
 384         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 385
 386     # Sparse checkouts may not have all mach_commands.py files. Ignore
 387     # errors from missing files. Same for spidermonkey tarballs.
 388     repo = resolve_repository()
 389     missing_ok = (
 390         repo is not None and repo.sparse_checkout_present()
 391     ) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
 392
 393     for path in MACH_MODULES:
 394         try:
 395             driver.load_commands_from_file(os.path.join(topsrcdir, path))
 396         except mach.base.MissingFileError:
 397             if not missing_ok:
 398                 raise
 399
 400     return driver
 401
 402
 403 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 404     """Submit telemetry collected by Glean.
 405
 406     Finalizes some metrics (command success state and duration, system information) and
 407     requests Glean to send the collected data.
 408     """
 409
 410     from mach.telemetry import MACH_METRICS_PATH
 411     from mozbuild.telemetry import (
 412         get_cpu_brand,
 413         get_distro_and_version,
 414         get_psutil_stats,
 415     )
 416
 417     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 418     mach_metrics.mach.duration.stop()
 419     mach_metrics.mach.success.set(success)
 420     system_metrics = mach_metrics.mach.system
 421     cpu_brand = get_cpu_brand()
 422     if cpu_brand:
 423         system_metrics.cpu_brand.set(cpu_brand)
 424     distro, version = get_distro_and_version()
 425     system_metrics.distro.set(distro)
 426     system_metrics.distro_version.set(version)
 427
 428     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 429     if has_psutil:
 430         # psutil may not be available (we allow `mach create-mach-environment`
 431         # to fail to install it).
 432         system_metrics.logical_cores.add(logical_cores)
 433         system_metrics.physical_cores.add(physical_cores)
 434         if memory_total is not None:
 435             system_metrics.memory.accumulate(
 436                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 437             )
 438     telemetry.submit(is_bootstrap)
 439
 440
 441 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 442 # the source directory are essentially ignored. See further below for details
 443 # and caveats.
 444 # Objdirs outside the source directory are ignored because in most cases, if
 445 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 446 class ImportHook(object):
 447     def __init__(self, original_import):
 448         self._original_import = original_import
 449         # Assume the source directory is the parent directory of the one
 450         # containing this file.
 451         self._source_dir = (
 452             os.path.normcase(
 453                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 454             )
 455             + os.sep
 456         )
 457         self._modules = set()
 458
 459     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 460         if sys.version_info[0] >= 3 and level < 0:
 461             level = 0
 462
 463         # name might be a relative import. Instead of figuring out what that
 464         # resolves to, which is complex, just rely on the real import.
 465         # Since we don't know the full module name, we can't check sys.modules,
 466         # so we need to keep track of which modules we've already seen to avoid
 467         # to stat() them again when they are imported multiple times.
 468         module = self._original_import(name, globals, locals, fromlist, level)
 469
 470         # Some tests replace modules in sys.modules with non-module instances.
 471         if not isinstance(module, ModuleType):
 472             return module
 473
 474         resolved_name = module.__name__
 475         if resolved_name in self._modules:
 476             return module
 477         self._modules.add(resolved_name)
 478
 479         # Builtin modules don't have a __file__ attribute.
 480         if not getattr(module, "__file__", None):
 481             return module
 482
 483         # Note: module.__file__ is not always absolute.
 484         path = os.path.normcase(os.path.abspath(module.__file__))
 485         # Note: we could avoid normcase and abspath above for non pyc/pyo
 486         # files, but those are actually rare, so it doesn't really matter.
 487         if not path.endswith((".pyc", ".pyo")):
 488             return module
 489
 490         # Ignore modules outside our source directory
 491         if not path.startswith(self._source_dir):
 492             return module
 493
 494         # If there is no .py corresponding to the .pyc/.pyo module we're
 495         # loading, remove the .pyc/.pyo file, and reload the module.
 496         # Since we already loaded the .pyc/.pyo module, if it had side
 497         # effects, they will have happened already, and loading the module
 498         # with the same name, from another directory may have the same side
 499         # effects (or different ones). We assume it's not a problem for the
 500         # python modules under our source directory (either because it
 501         # doesn't happen or because it doesn't matter).
 502         if not os.path.exists(module.__file__[:-1]):
 503             if os.path.exists(module.__file__):
 504                 os.remove(module.__file__)
 505             del sys.modules[module.__name__]
 506             module = self(name, globals, locals, fromlist, level)
 507
 508         return module
 509
 510
 511 # Install our hook. This can be deleted when the Python 3 migration is complete.
 512 if sys.version_info[0] < 3:
 513     builtins.__import__ = ImportHook(builtins.__import__)