build/mach_bootstrap.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import division, print_function, unicode_literals
   6
   7 import errno
   8 import json
   9 import math
  10 import os
  11 import platform
  12 import subprocess
  13 import sys
  14 import uuid
  15
  16 if sys.version_info[0] < 3:
  17     import __builtin__ as builtins
  18 else:
  19     import builtins
  20
  21 from types import ModuleType
  22
  23
  24 STATE_DIR_FIRST_RUN = """
  25 mach and the build system store shared state in a common directory on the
  26 filesystem. The following directory will be created:
  27
  28   {userdir}
  29
  30 If you would like to use a different directory, hit CTRL+c and set the
  31 MOZBUILD_STATE_PATH environment variable to the directory you would like to
  32 use and re-run mach. For this change to take effect forever, you'll likely
  33 want to export this environment variable from your shell's init scripts.
  34
  35 Press ENTER/RETURN to continue or CTRL+c to abort.
  36 """.lstrip()
  37
  38
  39 # Individual files providing mach commands.
  40 MACH_MODULES = [
  41     "build/valgrind/mach_commands.py",
  42     "devtools/shared/css/generated/mach_commands.py",
  43     "dom/bindings/mach_commands.py",
  44     "js/src/devtools/rootAnalysis/mach_commands.py",
  45     "layout/tools/reftest/mach_commands.py",
  46     "mobile/android/mach_commands.py",
  47     "python/mach/mach/commands/commandinfo.py",
  48     "python/mach/mach/commands/settings.py",
  49     "python/mach_commands.py",
  50     "python/mozboot/mozboot/mach_commands.py",
  51     "python/mozbuild/mozbuild/artifact_commands.py",
  52     "python/mozbuild/mozbuild/backend/mach_commands.py",
  53     "python/mozbuild/mozbuild/build_commands.py",
  54     "python/mozbuild/mozbuild/code_analysis/mach_commands.py",
  55     "python/mozbuild/mozbuild/compilation/codecomplete.py",
  56     "python/mozbuild/mozbuild/frontend/mach_commands.py",
  57     "python/mozbuild/mozbuild/vendor/mach_commands.py",
  58     "python/mozbuild/mozbuild/mach_commands.py",
  59     "python/mozperftest/mozperftest/mach_commands.py",
  60     "python/mozrelease/mozrelease/mach_commands.py",
  61     "remote/mach_commands.py",
  62     "taskcluster/mach_commands.py",
  63     "testing/awsy/mach_commands.py",
  64     "testing/condprofile/mach_commands.py",
  65     "testing/firefox-ui/mach_commands.py",
  66     "testing/geckodriver/mach_commands.py",
  67     "testing/mach_commands.py",
  68     "testing/marionette/mach_commands.py",
  69     "testing/mochitest/mach_commands.py",
  70     "testing/mozharness/mach_commands.py",
  71     "testing/raptor/mach_commands.py",
  72     "testing/talos/mach_commands.py",
  73     "testing/tps/mach_commands.py",
  74     "testing/web-platform/mach_commands.py",
  75     "testing/xpcshell/mach_commands.py",
  76     "toolkit/components/telemetry/tests/marionette/mach_commands.py",
  77     "tools/browsertime/mach_commands.py",
  78     "tools/compare-locales/mach_commands.py",
  79     "tools/lint/mach_commands.py",
  80     "tools/mach_commands.py",
  81     "tools/moztreedocs/mach_commands.py",
  82     "tools/phabricator/mach_commands.py",
  83     "tools/power/mach_commands.py",
  84     "tools/tryselect/mach_commands.py",
  85     "tools/vcs/mach_commands.py",
  86 ]
  87
  88
  89 CATEGORIES = {
  90     "build": {
  91         "short": "Build Commands",
  92         "long": "Interact with the build system",
  93         "priority": 80,
  94     },
  95     "post-build": {
  96         "short": "Post-build Commands",
  97         "long": "Common actions performed after completing a build.",
  98         "priority": 70,
  99     },
 100     "testing": {
 101         "short": "Testing",
 102         "long": "Run tests.",
 103         "priority": 60,
 104     },
 105     "ci": {
 106         "short": "CI",
 107         "long": "Taskcluster commands",
 108         "priority": 59,
 109     },
 110     "devenv": {
 111         "short": "Development Environment",
 112         "long": "Set up and configure your development environment.",
 113         "priority": 50,
 114     },
 115     "build-dev": {
 116         "short": "Low-level Build System Interaction",
 117         "long": "Interact with specific parts of the build system.",
 118         "priority": 20,
 119     },
 120     "misc": {
 121         "short": "Potpourri",
 122         "long": "Potent potables and assorted snacks.",
 123         "priority": 10,
 124     },
 125     "release": {
 126         "short": "Release automation",
 127         "long": "Commands for used in release automation.",
 128         "priority": 5,
 129     },
 130     "disabled": {
 131         "short": "Disabled",
 132         "long": "The disabled commands are hidden by default. Use -v to display them. "
 133         "These commands are unavailable for your current context, "
 134         'run "mach <command>" to see why.',
 135         "priority": 0,
 136     },
 137 }
 138
 139
 140 def search_path(mozilla_dir, packages_txt):
 141     with open(os.path.join(mozilla_dir, packages_txt)) as f:
 142         packages = [line.rstrip().split(":") for line in f]
 143
 144     def handle_package(package):
 145         if package[0] == "optional":
 146             try:
 147                 for path in handle_package(package[1:]):
 148                     yield path
 149             except Exception:
 150                 pass
 151
 152         if package[0] in ("windows", "!windows"):
 153             for_win = not package[0].startswith("!")
 154             is_win = sys.platform == "win32"
 155             if is_win == for_win:
 156                 for path in handle_package(package[1:]):
 157                     yield path
 158
 159         if package[0] in ("python2", "python3"):
 160             for_python3 = package[0].endswith("3")
 161             is_python3 = sys.version_info[0] > 2
 162             if is_python3 == for_python3:
 163                 for path in handle_package(package[1:]):
 164                     yield path
 165
 166         if package[0] == "packages.txt":
 167             assert len(package) == 2
 168             for p in search_path(mozilla_dir, package[1]):
 169                 yield os.path.join(mozilla_dir, p)
 170
 171         if package[0].endswith(".pth"):
 172             assert len(package) == 2
 173             yield os.path.join(mozilla_dir, package[1])
 174
 175     for package in packages:
 176         for path in handle_package(package):
 177             yield path
 178
 179
 180 def bootstrap(topsrcdir, mozilla_dir=None):
 181     if mozilla_dir is None:
 182         mozilla_dir = topsrcdir
 183
 184     # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
 185     # generate a user-friendly error message rather than a cryptic stack trace
 186     # on module import.
 187     major, minor = sys.version_info[:2]
 188     if (major == 2 and minor < 7) or (major == 3 and minor < 5):
 189         print("Python 2.7 or Python 3.5+ is required to run mach.")
 190         print("You are running Python", platform.python_version())
 191         sys.exit(1)
 192
 193     # Global build system and mach state is stored in a central directory. By
 194     # default, this is ~/.mozbuild. However, it can be defined via an
 195     # environment variable. We detect first run (by lack of this directory
 196     # existing) and notify the user that it will be created. The logic for
 197     # creation is much simpler for the "advanced" environment variable use
 198     # case. For default behavior, we educate users and give them an opportunity
 199     # to react. We always exit after creating the directory because users don't
 200     # like surprises.
 201     sys.path[0:0] = [
 202         os.path.join(mozilla_dir, path)
 203         for path in search_path(mozilla_dir, "build/mach_virtualenv_packages.txt")
 204     ]
 205     import mach.base
 206     import mach.main
 207     from mach.util import setenv
 208     from mozboot.util import get_state_dir
 209
 210     # Set a reasonable limit to the number of open files.
 211     #
 212     # Some linux systems set `ulimit -n` to a very high number, which works
 213     # well for systems that run servers, but this setting causes performance
 214     # problems when programs close file descriptors before forking, like
 215     # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
 216     # default in Python 3), or Rust's stdlib.  In some cases, Firefox does the
 217     # same thing when spawning processes.  We would prefer to lower this limit
 218     # to avoid such performance problems; processes spawned by `mach` will
 219     # inherit the limit set here.
 220     #
 221     # The Firefox build defaults the soft limit to 1024, except for builds that
 222     # do LTO, where the soft limit is 8192.  We're going to default to the
 223     # latter, since people do occasionally do LTO builds on their local
 224     # machines, and requiring them to discover another magical setting after
 225     # setting up an LTO build in the first place doesn't seem good.
 226     #
 227     # This code mimics the code in taskcluster/scripts/run-task.
 228     try:
 229         import resource
 230
 231         # Keep the hard limit the same, though, allowing processes to change
 232         # their soft limit if they need to (Firefox does, for instance).
 233         (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
 234         # Permit people to override our default limit if necessary via
 235         # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
 236         limit = os.environ.get("MOZ_LIMIT_NOFILE")
 237         if limit:
 238             limit = int(limit)
 239         else:
 240             # If no explicit limit is given, use our default if it's less than
 241             # the current soft limit.  For instance, the default on macOS is
 242             # 256, so we'd pick that rather than our default.
 243             limit = min(soft, 8192)
 244         # Now apply the limit, if it's different from the original one.
 245         if limit != soft:
 246             resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 247     except ImportError:
 248         # The resource module is UNIX only.
 249         pass
 250
 251     from mozbuild.util import patch_main
 252
 253     patch_main()
 254
 255     def resolve_repository():
 256         import mozversioncontrol
 257
 258         try:
 259             # This API doesn't respect the vcs binary choices from configure.
 260             # If we ever need to use the VCS binary here, consider something
 261             # more robust.
 262             return mozversioncontrol.get_repository_object(path=mozilla_dir)
 263         except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
 264             return None
 265
 266     def pre_dispatch_handler(context, handler, args):
 267         # If --disable-tests flag was enabled in the mozconfig used to compile
 268         # the build, tests will be disabled. Instead of trying to run
 269         # nonexistent tests then reporting a failure, this will prevent mach
 270         # from progressing beyond this point.
 271         if handler.category == "testing" and not handler.ok_if_tests_disabled:
 272             from mozbuild.base import BuildEnvironmentNotFoundException
 273
 274             try:
 275                 from mozbuild.base import MozbuildObject
 276
 277                 # all environments should have an instance of build object.
 278                 build = MozbuildObject.from_environment()
 279                 if build is not None and hasattr(build, "mozconfig"):
 280                     ac_options = build.mozconfig["configure_args"]
 281                     if ac_options and "--disable-tests" in ac_options:
 282                         print(
 283                             "Tests have been disabled by mozconfig with the flag "
 284                             + '"ac_add_options --disable-tests".\n'
 285                             + "Remove the flag, and re-compile to enable tests."
 286                         )
 287                         sys.exit(1)
 288             except BuildEnvironmentNotFoundException:
 289                 # likely automation environment, so do nothing.
 290                 pass
 291
 292     def post_dispatch_handler(
 293         context, handler, instance, success, start_time, end_time, depth, args
 294     ):
 295         """Perform global operations after command dispatch.
 296
 297
 298         For now,  we will use this to handle build system telemetry.
 299         """
 300
 301         # Don't finalize telemetry data if this mach command was invoked as part of
 302         # another mach command.
 303         if depth != 1:
 304             return
 305
 306         _finalize_telemetry_glean(
 307             context.telemetry, handler.name == "bootstrap", success
 308         )
 309         _finalize_telemetry_legacy(
 310             context, instance, handler, success, start_time, end_time, topsrcdir
 311         )
 312
 313     def populate_context(key=None):
 314         if key is None:
 315             return
 316         if key == "state_dir":
 317             state_dir = get_state_dir()
 318             if state_dir == os.environ.get("MOZBUILD_STATE_PATH"):
 319                 if not os.path.exists(state_dir):
 320                     print(
 321                         "Creating global state directory from environment variable: %s"
 322                         % state_dir
 323                     )
 324                     os.makedirs(state_dir, mode=0o770)
 325             else:
 326                 if not os.path.exists(state_dir):
 327                     if not os.environ.get("MOZ_AUTOMATION"):
 328                         print(STATE_DIR_FIRST_RUN.format(userdir=state_dir))
 329                         try:
 330                             sys.stdin.readline()
 331                         except KeyboardInterrupt:
 332                             sys.exit(1)
 333
 334                     print("\nCreating default state directory: %s" % state_dir)
 335                     os.makedirs(state_dir, mode=0o770)
 336
 337             return state_dir
 338
 339         if key == "local_state_dir":
 340             return get_state_dir(srcdir=True)
 341
 342         if key == "topdir":
 343             return topsrcdir
 344
 345         if key == "pre_dispatch_handler":
 346             return pre_dispatch_handler
 347
 348         if key == "post_dispatch_handler":
 349             return post_dispatch_handler
 350
 351         if key == "repository":
 352             return resolve_repository()
 353
 354         raise AttributeError(key)
 355
 356     # Note which process is top-level so that recursive mach invocations can avoid writing
 357     # telemetry data.
 358     if "MACH_MAIN_PID" not in os.environ:
 359         setenv("MACH_MAIN_PID", str(os.getpid()))
 360
 361     driver = mach.main.Mach(os.getcwd())
 362     driver.populate_context_handler = populate_context
 363
 364     if not driver.settings_paths:
 365         # default global machrc location
 366         driver.settings_paths.append(get_state_dir())
 367     # always load local repository configuration
 368     driver.settings_paths.append(mozilla_dir)
 369
 370     for category, meta in CATEGORIES.items():
 371         driver.define_category(category, meta["short"], meta["long"], meta["priority"])
 372
 373     repo = resolve_repository()
 374
 375     for path in MACH_MODULES:
 376         # Sparse checkouts may not have all mach_commands.py files. Ignore
 377         # errors from missing files.
 378         try:
 379             driver.load_commands_from_file(os.path.join(mozilla_dir, path))
 380         except mach.base.MissingFileError:
 381             if not repo or not repo.sparse_checkout_present():
 382                 raise
 383
 384     return driver
 385
 386
 387 def _finalize_telemetry_legacy(
 388     context, instance, handler, success, start_time, end_time, topsrcdir
 389 ):
 390     """Record and submit legacy telemetry.
 391
 392     Parameterized by the raw gathered telemetry, this function handles persisting and
 393     submission of the data.
 394
 395     This has been designated as "legacy" telemetry because modern telemetry is being
 396     submitted with "Glean".
 397     """
 398     from mozboot.util import get_state_dir
 399     from mozbuild.base import MozbuildObject
 400     from mozbuild.telemetry import gather_telemetry
 401     from mach.telemetry import is_telemetry_enabled, is_applicable_telemetry_environment
 402
 403     if not (
 404         is_applicable_telemetry_environment() and is_telemetry_enabled(context.settings)
 405     ):
 406         return
 407
 408     if not isinstance(instance, MozbuildObject):
 409         instance = MozbuildObject.from_environment()
 410
 411     command_attrs = getattr(context, "command_attrs", {})
 412
 413     # We gather telemetry for every operation.
 414     data = gather_telemetry(
 415         command=handler.name,
 416         success=success,
 417         start_time=start_time,
 418         end_time=end_time,
 419         mach_context=context,
 420         instance=instance,
 421         command_attrs=command_attrs,
 422     )
 423     if data:
 424         telemetry_dir = os.path.join(get_state_dir(), "telemetry")
 425         try:
 426             os.mkdir(telemetry_dir)
 427         except OSError as e:
 428             if e.errno != errno.EEXIST:
 429                 raise
 430         outgoing_dir = os.path.join(telemetry_dir, "outgoing")
 431         try:
 432             os.mkdir(outgoing_dir)
 433         except OSError as e:
 434             if e.errno != errno.EEXIST:
 435                 raise
 436
 437         with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + ".json"), "w") as f:
 438             json.dump(data, f, sort_keys=True)
 439
 440     # The user is performing a maintenance command, skip the upload
 441     if handler.name in (
 442         "bootstrap",
 443         "doctor",
 444         "mach-commands",
 445         "vcs-setup",
 446         "create-mach-environment",
 447         "install-moz-phab",
 448         # We call mach environment in client.mk which would cause the
 449         # data submission to block the forward progress of make.
 450         "environment",
 451     ):
 452         return False
 453
 454     if "TEST_MACH_TELEMETRY_NO_SUBMIT" in os.environ:
 455         # In our telemetry tests, we want telemetry to be collected for analysis, but
 456         # we don't want it submitted.
 457         return False
 458
 459     state_dir = get_state_dir()
 460
 461     machpath = os.path.join(instance.topsrcdir, "mach")
 462     with open(os.devnull, "wb") as devnull:
 463         subprocess.Popen(
 464             [
 465                 sys.executable,
 466                 machpath,
 467                 "python",
 468                 "--no-virtualenv",
 469                 os.path.join(topsrcdir, "build", "submit_telemetry_data.py"),
 470                 state_dir,
 471             ],
 472             stdout=devnull,
 473             stderr=devnull,
 474         )
 475
 476
 477 def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
 478     """Submit telemetry collected by Glean.
 479
 480     Finalizes some metrics (command success state and duration, system information) and
 481     requests Glean to send the collected data.
 482     """
 483
 484     from mach.telemetry import MACH_METRICS_PATH
 485     from mozbuild.telemetry import (
 486         get_cpu_brand,
 487         get_distro_and_version,
 488         get_psutil_stats,
 489     )
 490
 491     mach_metrics = telemetry.metrics(MACH_METRICS_PATH)
 492     mach_metrics.mach.duration.stop()
 493     mach_metrics.mach.success.set(success)
 494     system_metrics = mach_metrics.mach.system
 495     system_metrics.cpu_brand.set(get_cpu_brand())
 496     distro, version = get_distro_and_version()
 497     system_metrics.distro.set(distro)
 498     system_metrics.distro_version.set(version)
 499
 500     has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
 501     if has_psutil:
 502         # psutil may not be available if a successful build hasn't occurred yet.
 503         system_metrics.logical_cores.add(logical_cores)
 504         system_metrics.physical_cores.add(physical_cores)
 505         if memory_total is not None:
 506             system_metrics.memory.accumulate(
 507                 int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
 508             )
 509     telemetry.submit(is_bootstrap)
 510
 511
 512 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 513 # the source directory are essentially ignored. See further below for details
 514 # and caveats.
 515 # Objdirs outside the source directory are ignored because in most cases, if
 516 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
 517 class ImportHook(object):
 518     def __init__(self, original_import):
 519         self._original_import = original_import
 520         # Assume the source directory is the parent directory of the one
 521         # containing this file.
 522         self._source_dir = (
 523             os.path.normcase(
 524                 os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 525             )
 526             + os.sep
 527         )
 528         self._modules = set()
 529
 530     def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
 531         if sys.version_info[0] >= 3 and level < 0:
 532             level = 0
 533
 534         # name might be a relative import. Instead of figuring out what that
 535         # resolves to, which is complex, just rely on the real import.
 536         # Since we don't know the full module name, we can't check sys.modules,
 537         # so we need to keep track of which modules we've already seen to avoid
 538         # to stat() them again when they are imported multiple times.
 539         module = self._original_import(name, globals, locals, fromlist, level)
 540
 541         # Some tests replace modules in sys.modules with non-module instances.
 542         if not isinstance(module, ModuleType):
 543             return module
 544
 545         resolved_name = module.__name__
 546         if resolved_name in self._modules:
 547             return module
 548         self._modules.add(resolved_name)
 549
 550         # Builtin modules don't have a __file__ attribute.
 551         if not getattr(module, "__file__", None):
 552             return module
 553
 554         # Note: module.__file__ is not always absolute.
 555         path = os.path.normcase(os.path.abspath(module.__file__))
 556         # Note: we could avoid normcase and abspath above for non pyc/pyo
 557         # files, but those are actually rare, so it doesn't really matter.
 558         if not path.endswith((".pyc", ".pyo")):
 559             return module
 560
 561         # Ignore modules outside our source directory
 562         if not path.startswith(self._source_dir):
 563             return module
 564
 565         # If there is no .py corresponding to the .pyc/.pyo module we're
 566         # loading, remove the .pyc/.pyo file, and reload the module.
 567         # Since we already loaded the .pyc/.pyo module, if it had side
 568         # effects, they will have happened already, and loading the module
 569         # with the same name, from another directory may have the same side
 570         # effects (or different ones). We assume it's not a problem for the
 571         # python modules under our source directory (either because it
 572         # doesn't happen or because it doesn't matter).
 573         if not os.path.exists(module.__file__[:-1]):
 574             if os.path.exists(module.__file__):
 575                 os.remove(module.__file__)
 576             del sys.modules[module.__name__]
 577             module = self(name, globals, locals, fromlist, level)
 578
 579         return module
 580
 581
 582 # Install our hook. This can be deleted when the Python 3 migration is complete.
 583 if sys.version_info[0] < 3:
 584     builtins.__import__ = ImportHook(builtins.__import__)