1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import division
, print_function
, unicode_literals
14 if sys
.version_info
[0] < 3:
15 import __builtin__
as builtins
19 from types
import ModuleType
22 STATE_DIR_FIRST_RUN
= """
23 mach and the build system store shared state in a common directory on the
24 filesystem. The following directory will be created:
28 If you would like to use a different directory, hit CTRL+c and set the
29 MOZBUILD_STATE_PATH environment variable to the directory you would like to
30 use and re-run mach. For this change to take effect forever, you'll likely
31 want to export this environment variable from your shell's init scripts.
33 Press ENTER/RETURN to continue or CTRL+c to abort.
37 # Individual files providing mach commands.
39 "build/valgrind/mach_commands.py",
40 "devtools/shared/css/generated/mach_commands.py",
41 "dom/bindings/mach_commands.py",
42 "js/src/devtools/rootAnalysis/mach_commands.py",
43 "layout/tools/reftest/mach_commands.py",
44 "mobile/android/mach_commands.py",
45 "python/mach/mach/commands/commandinfo.py",
46 "python/mach/mach/commands/settings.py",
47 "python/mach_commands.py",
48 "python/mozboot/mozboot/mach_commands.py",
49 "python/mozbuild/mozbuild/artifact_commands.py",
50 "python/mozbuild/mozbuild/backend/mach_commands.py",
51 "python/mozbuild/mozbuild/build_commands.py",
52 "python/mozbuild/mozbuild/code_analysis/mach_commands.py",
53 "python/mozbuild/mozbuild/compilation/codecomplete.py",
54 "python/mozbuild/mozbuild/frontend/mach_commands.py",
55 "python/mozbuild/mozbuild/vendor/mach_commands.py",
56 "python/mozbuild/mozbuild/mach_commands.py",
57 "python/mozperftest/mozperftest/mach_commands.py",
58 "python/mozrelease/mozrelease/mach_commands.py",
59 "remote/mach_commands.py",
60 "security/manager/tools/mach_commands.py",
61 "taskcluster/mach_commands.py",
62 "testing/awsy/mach_commands.py",
63 "testing/condprofile/mach_commands.py",
64 "testing/firefox-ui/mach_commands.py",
65 "testing/geckodriver/mach_commands.py",
66 "testing/mach_commands.py",
67 "testing/marionette/mach_commands.py",
68 "testing/mochitest/mach_commands.py",
69 "testing/mozharness/mach_commands.py",
70 "testing/raptor/mach_commands.py",
71 "testing/talos/mach_commands.py",
72 "testing/tps/mach_commands.py",
73 "testing/web-platform/mach_commands.py",
74 "testing/xpcshell/mach_commands.py",
75 "toolkit/components/telemetry/tests/marionette/mach_commands.py",
76 "tools/browsertime/mach_commands.py",
77 "tools/compare-locales/mach_commands.py",
78 "tools/lint/mach_commands.py",
79 "tools/mach_commands.py",
80 "tools/moztreedocs/mach_commands.py",
81 "tools/phabricator/mach_commands.py",
82 "tools/power/mach_commands.py",
83 "tools/tryselect/mach_commands.py",
84 "tools/vcs/mach_commands.py",
90 "short": "Build Commands",
91 "long": "Interact with the build system",
95 "short": "Post-build Commands",
96 "long": "Common actions performed after completing a build.",
101 "long": "Run tests.",
106 "long": "Taskcluster commands",
110 "short": "Development Environment",
111 "long": "Set up and configure your development environment.",
115 "short": "Low-level Build System Interaction",
116 "long": "Interact with specific parts of the build system.",
120 "short": "Potpourri",
121 "long": "Potent potables and assorted snacks.",
125 "short": "Release automation",
126 "long": "Commands for used in release automation.",
131 "long": "The disabled commands are hidden by default. Use -v to display them. "
132 "These commands are unavailable for your current context, "
133 'run "mach <command>" to see why.',
139 def search_path(mozilla_dir
, packages_txt
):
140 with
open(os
.path
.join(mozilla_dir
, packages_txt
)) as f
:
141 packages
= [line
.rstrip().split(":") for line
in f
]
143 def handle_package(package
):
144 if package
[0] == "optional":
146 for path
in handle_package(package
[1:]):
151 if package
[0] in ("windows", "!windows"):
152 for_win
= not package
[0].startswith("!")
153 is_win
= sys
.platform
== "win32"
154 if is_win
== for_win
:
155 for path
in handle_package(package
[1:]):
158 if package
[0] in ("python2", "python3"):
159 for_python3
= package
[0].endswith("3")
160 is_python3
= sys
.version_info
[0] > 2
161 if is_python3
== for_python3
:
162 for path
in handle_package(package
[1:]):
165 if package
[0] == "packages.txt":
166 assert len(package
) == 2
167 for p
in search_path(mozilla_dir
, package
[1]):
168 yield os
.path
.join(mozilla_dir
, p
)
170 if package
[0].endswith(".pth"):
171 assert len(package
) == 2
172 yield os
.path
.join(mozilla_dir
, package
[1])
174 for package
in packages
:
175 for path
in handle_package(package
):
179 def mach_sys_path(mozilla_dir
):
181 os
.path
.join(mozilla_dir
, path
)
182 for path
in search_path(mozilla_dir
, "build/mach_virtualenv_packages.txt")
186 def bootstrap(topsrcdir
):
187 # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
188 # generate a user-friendly error message rather than a cryptic stack trace
190 major
, minor
= sys
.version_info
[:2]
191 if (major
== 2 and minor
< 7) or (major
== 3 and minor
< 5):
192 print("Python 2.7 or Python 3.5+ is required to run mach.")
193 print("You are running Python", platform
.python_version())
196 # This directory was deleted in bug 1666345, but there may be some ignored
197 # files here. We can safely just delete it for the user so they don't have
198 # to clean the repo themselves.
199 deleted_dir
= os
.path
.join(topsrcdir
, "third_party", "python", "psutil")
200 if os
.path
.exists(deleted_dir
):
201 shutil
.rmtree(deleted_dir
, ignore_errors
=True)
203 if major
== 3 and sys
.prefix
== sys
.base_prefix
:
204 # We are not in a virtualenv. Remove global site packages
206 # Note that we don't ever invoke mach from a Python 2 virtualenv,
207 # and "sys.base_prefix" doesn't exist before Python 3.3, so we
208 # guard with the "major == 3" check.
209 site_paths
= set(site
.getsitepackages() + [site
.getusersitepackages()])
210 sys
.path
= [path
for path
in sys
.path
if path
not in site_paths
]
212 # Global build system and mach state is stored in a central directory. By
213 # default, this is ~/.mozbuild. However, it can be defined via an
214 # environment variable. We detect first run (by lack of this directory
215 # existing) and notify the user that it will be created. The logic for
216 # creation is much simpler for the "advanced" environment variable use
217 # case. For default behavior, we educate users and give them an opportunity
218 # to react. We always exit after creating the directory because users don't
220 sys
.path
[0:0] = mach_sys_path(topsrcdir
)
223 from mach
.util
import setenv
224 from mozboot
.util
import get_state_dir
226 # Set a reasonable limit to the number of open files.
228 # Some linux systems set `ulimit -n` to a very high number, which works
229 # well for systems that run servers, but this setting causes performance
230 # problems when programs close file descriptors before forking, like
231 # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
232 # default in Python 3), or Rust's stdlib. In some cases, Firefox does the
233 # same thing when spawning processes. We would prefer to lower this limit
234 # to avoid such performance problems; processes spawned by `mach` will
235 # inherit the limit set here.
237 # The Firefox build defaults the soft limit to 1024, except for builds that
238 # do LTO, where the soft limit is 8192. We're going to default to the
239 # latter, since people do occasionally do LTO builds on their local
240 # machines, and requiring them to discover another magical setting after
241 # setting up an LTO build in the first place doesn't seem good.
243 # This code mimics the code in taskcluster/scripts/run-task.
247 # Keep the hard limit the same, though, allowing processes to change
248 # their soft limit if they need to (Firefox does, for instance).
249 (soft
, hard
) = resource
.getrlimit(resource
.RLIMIT_NOFILE
)
250 # Permit people to override our default limit if necessary via
251 # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
252 limit
= os
.environ
.get("MOZ_LIMIT_NOFILE")
256 # If no explicit limit is given, use our default if it's less than
257 # the current soft limit. For instance, the default on macOS is
258 # 256, so we'd pick that rather than our default.
259 limit
= min(soft
, 8192)
260 # Now apply the limit, if it's different from the original one.
262 resource
.setrlimit(resource
.RLIMIT_NOFILE
, (limit
, hard
))
264 # The resource module is UNIX only.
267 from mozbuild
.util
import patch_main
271 def resolve_repository():
272 import mozversioncontrol
275 # This API doesn't respect the vcs binary choices from configure.
276 # If we ever need to use the VCS binary here, consider something
278 return mozversioncontrol
.get_repository_object(path
=topsrcdir
)
279 except (mozversioncontrol
.InvalidRepoPath
, mozversioncontrol
.MissingVCSTool
):
282 def pre_dispatch_handler(context
, handler
, args
):
283 # If --disable-tests flag was enabled in the mozconfig used to compile
284 # the build, tests will be disabled. Instead of trying to run
285 # nonexistent tests then reporting a failure, this will prevent mach
286 # from progressing beyond this point.
287 if handler
.category
== "testing" and not handler
.ok_if_tests_disabled
:
288 from mozbuild
.base
import BuildEnvironmentNotFoundException
291 from mozbuild
.base
import MozbuildObject
293 # all environments should have an instance of build object.
294 build
= MozbuildObject
.from_environment()
295 if build
is not None and hasattr(build
, "mozconfig"):
296 ac_options
= build
.mozconfig
["configure_args"]
297 if ac_options
and "--disable-tests" in ac_options
:
299 "Tests have been disabled by mozconfig with the flag "
300 + '"ac_add_options --disable-tests".\n'
301 + "Remove the flag, and re-compile to enable tests."
304 except BuildEnvironmentNotFoundException
:
305 # likely automation environment, so do nothing.
308 def post_dispatch_handler(
309 context
, handler
, instance
, success
, start_time
, end_time
, depth
, args
311 """Perform global operations after command dispatch.
314 For now, we will use this to handle build system telemetry.
317 # Don't finalize telemetry data if this mach command was invoked as part of
318 # another mach command.
322 _finalize_telemetry_glean(
323 context
.telemetry
, handler
.name
== "bootstrap", success
326 def populate_context(key
=None):
329 if key
== "state_dir":
330 state_dir
= get_state_dir()
331 if state_dir
== os
.environ
.get("MOZBUILD_STATE_PATH"):
332 if not os
.path
.exists(state_dir
):
334 "Creating global state directory from environment variable: %s"
337 os
.makedirs(state_dir
, mode
=0o770)
339 if not os
.path
.exists(state_dir
):
340 if not os
.environ
.get("MOZ_AUTOMATION"):
341 print(STATE_DIR_FIRST_RUN
.format(userdir
=state_dir
))
344 except KeyboardInterrupt:
347 print("\nCreating default state directory: %s" % state_dir
)
348 os
.makedirs(state_dir
, mode
=0o770)
352 if key
== "local_state_dir":
353 return get_state_dir(srcdir
=True)
358 if key
== "pre_dispatch_handler":
359 return pre_dispatch_handler
361 if key
== "post_dispatch_handler":
362 return post_dispatch_handler
364 if key
== "repository":
365 return resolve_repository()
367 raise AttributeError(key
)
369 # Note which process is top-level so that recursive mach invocations can avoid writing
371 if "MACH_MAIN_PID" not in os
.environ
:
372 setenv("MACH_MAIN_PID", str(os
.getpid()))
374 driver
= mach
.main
.Mach(os
.getcwd())
375 driver
.populate_context_handler
= populate_context
377 if not driver
.settings_paths
:
378 # default global machrc location
379 driver
.settings_paths
.append(get_state_dir())
380 # always load local repository configuration
381 driver
.settings_paths
.append(topsrcdir
)
383 for category
, meta
in CATEGORIES
.items():
384 driver
.define_category(category
, meta
["short"], meta
["long"], meta
["priority"])
386 # Sparse checkouts may not have all mach_commands.py files. Ignore
387 # errors from missing files. Same for spidermonkey tarballs.
388 repo
= resolve_repository()
390 repo
is not None and repo
.sparse_checkout_present()
391 ) or os
.path
.exists(os
.path
.join(topsrcdir
, "INSTALL"))
393 for path
in MACH_MODULES
:
395 driver
.load_commands_from_file(os
.path
.join(topsrcdir
, path
))
396 except mach
.base
.MissingFileError
:
403 def _finalize_telemetry_glean(telemetry
, is_bootstrap
, success
):
404 """Submit telemetry collected by Glean.
406 Finalizes some metrics (command success state and duration, system information) and
407 requests Glean to send the collected data.
410 from mach
.telemetry
import MACH_METRICS_PATH
411 from mozbuild
.telemetry
import (
413 get_distro_and_version
,
417 mach_metrics
= telemetry
.metrics(MACH_METRICS_PATH
)
418 mach_metrics
.mach
.duration
.stop()
419 mach_metrics
.mach
.success
.set(success
)
420 system_metrics
= mach_metrics
.mach
.system
421 cpu_brand
= get_cpu_brand()
423 system_metrics
.cpu_brand
.set(cpu_brand
)
424 distro
, version
= get_distro_and_version()
425 system_metrics
.distro
.set(distro
)
426 system_metrics
.distro_version
.set(version
)
428 has_psutil
, logical_cores
, physical_cores
, memory_total
= get_psutil_stats()
430 # psutil may not be available (we allow `mach create-mach-environment`
431 # to fail to install it).
432 system_metrics
.logical_cores
.add(logical_cores
)
433 system_metrics
.physical_cores
.add(physical_cores
)
434 if memory_total
is not None:
435 system_metrics
.memory
.accumulate(
436 int(math
.ceil(float(memory_total
) / (1024 * 1024 * 1024)))
438 telemetry
.submit(is_bootstrap
)
441 # Hook import such that .pyc/.pyo files without a corresponding .py file in
442 # the source directory are essentially ignored. See further below for details
444 # Objdirs outside the source directory are ignored because in most cases, if
445 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
446 class ImportHook(object):
447 def __init__(self
, original_import
):
448 self
._original
_import
= original_import
449 # Assume the source directory is the parent directory of the one
450 # containing this file.
453 os
.path
.abspath(os
.path
.dirname(os
.path
.dirname(__file__
)))
457 self
._modules
= set()
459 def __call__(self
, name
, globals=None, locals=None, fromlist
=None, level
=-1):
460 if sys
.version_info
[0] >= 3 and level
< 0:
463 # name might be a relative import. Instead of figuring out what that
464 # resolves to, which is complex, just rely on the real import.
465 # Since we don't know the full module name, we can't check sys.modules,
466 # so we need to keep track of which modules we've already seen to avoid
467 # to stat() them again when they are imported multiple times.
468 module
= self
._original
_import
(name
, globals, locals, fromlist
, level
)
470 # Some tests replace modules in sys.modules with non-module instances.
471 if not isinstance(module
, ModuleType
):
474 resolved_name
= module
.__name
__
475 if resolved_name
in self
._modules
:
477 self
._modules
.add(resolved_name
)
479 # Builtin modules don't have a __file__ attribute.
480 if not getattr(module
, "__file__", None):
483 # Note: module.__file__ is not always absolute.
484 path
= os
.path
.normcase(os
.path
.abspath(module
.__file
__))
485 # Note: we could avoid normcase and abspath above for non pyc/pyo
486 # files, but those are actually rare, so it doesn't really matter.
487 if not path
.endswith((".pyc", ".pyo")):
490 # Ignore modules outside our source directory
491 if not path
.startswith(self
._source
_dir
):
494 # If there is no .py corresponding to the .pyc/.pyo module we're
495 # loading, remove the .pyc/.pyo file, and reload the module.
496 # Since we already loaded the .pyc/.pyo module, if it had side
497 # effects, they will have happened already, and loading the module
498 # with the same name, from another directory may have the same side
499 # effects (or different ones). We assume it's not a problem for the
500 # python modules under our source directory (either because it
501 # doesn't happen or because it doesn't matter).
502 if not os
.path
.exists(module
.__file
__[:-1]):
503 if os
.path
.exists(module
.__file
__):
504 os
.remove(module
.__file
__)
505 del sys
.modules
[module
.__name
__]
506 module
= self(name
, globals, locals, fromlist
, level
)
511 # Install our hook. This can be deleted when the Python 3 migration is complete.
512 if sys
.version_info
[0] < 3:
513 builtins
.__import
__ = ImportHook(builtins
.__import
__)