1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import print_function
, unicode_literals
14 if sys
.version_info
[0] < 3:
15 import __builtin__
as builtins
19 from types
import ModuleType
22 STATE_DIR_FIRST_RUN
= '''
23 mach and the build system store shared state in a common directory on the
24 filesystem. The following directory will be created:
28 If you would like to use a different directory, hit CTRL+c and set the
29 MOZBUILD_STATE_PATH environment variable to the directory you would like to
30 use and re-run mach. For this change to take effect forever, you'll likely
31 want to export this environment variable from your shell's init scripts.
33 Press ENTER/RETURN to continue or CTRL+c to abort.
37 # Individual files providing mach commands.
39 'build/valgrind/mach_commands.py',
40 'devtools/shared/css/generated/mach_commands.py',
41 'dom/bindings/mach_commands.py',
42 'js/src/devtools/rootAnalysis/mach_commands.py',
43 'layout/tools/reftest/mach_commands.py',
44 'mobile/android/mach_commands.py',
45 'python/mach/mach/commands/commandinfo.py',
46 'python/mach/mach/commands/settings.py',
47 'python/mach_commands.py',
48 'python/mozboot/mozboot/mach_commands.py',
49 'python/mozbuild/mozbuild/artifact_commands.py',
50 'python/mozbuild/mozbuild/backend/mach_commands.py',
51 'python/mozbuild/mozbuild/build_commands.py',
52 'python/mozbuild/mozbuild/code_analysis/mach_commands.py',
53 'python/mozbuild/mozbuild/compilation/codecomplete.py',
54 'python/mozbuild/mozbuild/frontend/mach_commands.py',
55 'python/mozbuild/mozbuild/vendor/mach_commands.py',
56 'python/mozbuild/mozbuild/mach_commands.py',
57 'python/mozperftest/mozperftest/mach_commands.py',
58 'python/mozrelease/mozrelease/mach_commands.py',
59 'remote/mach_commands.py',
60 'taskcluster/mach_commands.py',
61 'testing/awsy/mach_commands.py',
62 'testing/condprofile/mach_commands.py',
63 'testing/firefox-ui/mach_commands.py',
64 'testing/geckodriver/mach_commands.py',
65 'testing/mach_commands.py',
66 'testing/marionette/mach_commands.py',
67 'testing/mochitest/mach_commands.py',
68 'testing/mozharness/mach_commands.py',
69 'testing/raptor/mach_commands.py',
70 'testing/talos/mach_commands.py',
71 'testing/tps/mach_commands.py',
72 'testing/web-platform/mach_commands.py',
73 'testing/xpcshell/mach_commands.py',
74 'toolkit/components/telemetry/tests/marionette/mach_commands.py',
75 'tools/browsertime/mach_commands.py',
76 'tools/compare-locales/mach_commands.py',
77 'tools/lint/mach_commands.py',
78 'tools/mach_commands.py',
79 'tools/moztreedocs/mach_commands.py',
80 'tools/phabricator/mach_commands.py',
81 'tools/power/mach_commands.py',
82 'tools/tryselect/mach_commands.py',
83 'tools/vcs/mach_commands.py',
89 'short': 'Build Commands',
90 'long': 'Interact with the build system',
94 'short': 'Post-build Commands',
95 'long': 'Common actions performed after completing a build.',
100 'long': 'Run tests.',
105 'long': 'Taskcluster commands',
109 'short': 'Development Environment',
110 'long': 'Set up and configure your development environment.',
114 'short': 'Low-level Build System Interaction',
115 'long': 'Interact with specific parts of the build system.',
119 'short': 'Potpourri',
120 'long': 'Potent potables and assorted snacks.',
124 'short': 'Release automation',
125 'long': 'Commands for used in release automation.',
130 'long': 'The disabled commands are hidden by default. Use -v to display them. '
131 'These commands are unavailable for your current context, '
132 'run "mach <command>" to see why.',
138 def search_path(mozilla_dir
, packages_txt
):
139 with
open(os
.path
.join(mozilla_dir
, packages_txt
)) as f
:
140 packages
= [line
.rstrip().split(':') for line
in f
]
142 def handle_package(package
):
143 if package
[0] == 'optional':
145 for path
in handle_package(package
[1:]):
150 if package
[0] in ('windows', '!windows'):
151 for_win
= not package
[0].startswith('!')
152 is_win
= sys
.platform
== 'win32'
153 if is_win
== for_win
:
154 for path
in handle_package(package
[1:]):
157 if package
[0] in ('python2', 'python3'):
158 for_python3
= package
[0].endswith('3')
159 is_python3
= sys
.version_info
[0] > 2
160 if is_python3
== for_python3
:
161 for path
in handle_package(package
[1:]):
164 if package
[0] == 'packages.txt':
165 assert len(package
) == 2
166 for p
in search_path(mozilla_dir
, package
[1]):
167 yield os
.path
.join(mozilla_dir
, p
)
169 if package
[0].endswith('.pth'):
170 assert len(package
) == 2
171 yield os
.path
.join(mozilla_dir
, package
[1])
173 for package
in packages
:
174 for path
in handle_package(package
):
178 def bootstrap(topsrcdir
, mozilla_dir
=None):
179 if mozilla_dir
is None:
180 mozilla_dir
= topsrcdir
182 # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
183 # generate a user-friendly error message rather than a cryptic stack trace
185 major
, minor
= sys
.version_info
[:2]
186 if (major
== 2 and minor
< 7) or (major
== 3 and minor
< 5):
187 print('Python 2.7 or Python 3.5+ is required to run mach.')
188 print('You are running Python', platform
.python_version())
191 # Global build system and mach state is stored in a central directory. By
192 # default, this is ~/.mozbuild. However, it can be defined via an
193 # environment variable. We detect first run (by lack of this directory
194 # existing) and notify the user that it will be created. The logic for
195 # creation is much simpler for the "advanced" environment variable use
196 # case. For default behavior, we educate users and give them an opportunity
197 # to react. We always exit after creating the directory because users don't
200 os
.path
.join(mozilla_dir
, path
)
201 for path
in search_path(mozilla_dir
,
202 'build/mach_virtualenv_packages.txt')]
205 from mach
.util
import setenv
206 from mozboot
.util
import get_state_dir
208 # Set a reasonable limit to the number of open files.
210 # Some linux systems set `ulimit -n` to a very high number, which works
211 # well for systems that run servers, but this setting causes performance
212 # problems when programs close file descriptors before forking, like
213 # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
214 # default in Python 3), or Rust's stdlib. In some cases, Firefox does the
215 # same thing when spawning processes. We would prefer to lower this limit
216 # to avoid such performance problems; processes spawned by `mach` will
217 # inherit the limit set here.
219 # The Firefox build defaults the soft limit to 1024, except for builds that
220 # do LTO, where the soft limit is 8192. We're going to default to the
221 # latter, since people do occasionally do LTO builds on their local
222 # machines, and requiring them to discover another magical setting after
223 # setting up an LTO build in the first place doesn't seem good.
225 # This code mimics the code in taskcluster/scripts/run-task.
228 # Keep the hard limit the same, though, allowing processes to change
229 # their soft limit if they need to (Firefox does, for instance).
230 (soft
, hard
) = resource
.getrlimit(resource
.RLIMIT_NOFILE
)
231 # Permit people to override our default limit if necessary via
232 # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
233 limit
= os
.environ
.get('MOZ_LIMIT_NOFILE')
237 # If no explicit limit is given, use our default if it's less than
238 # the current soft limit. For instance, the default on macOS is
239 # 256, so we'd pick that rather than our default.
240 limit
= min(soft
, 8192)
241 # Now apply the limit, if it's different from the original one.
243 resource
.setrlimit(resource
.RLIMIT_NOFILE
, (limit
, hard
))
245 # The resource module is UNIX only.
248 from mozbuild
.util
import patch_main
251 def resolve_repository():
252 import mozversioncontrol
255 # This API doesn't respect the vcs binary choices from configure.
256 # If we ever need to use the VCS binary here, consider something
258 return mozversioncontrol
.get_repository_object(path
=mozilla_dir
)
259 except (mozversioncontrol
.InvalidRepoPath
,
260 mozversioncontrol
.MissingVCSTool
):
263 def pre_dispatch_handler(context
, handler
, args
):
264 # If --disable-tests flag was enabled in the mozconfig used to compile
265 # the build, tests will be disabled. Instead of trying to run
266 # nonexistent tests then reporting a failure, this will prevent mach
267 # from progressing beyond this point.
268 if handler
.category
== 'testing':
269 from mozbuild
.base
import BuildEnvironmentNotFoundException
271 from mozbuild
.base
import MozbuildObject
272 # all environments should have an instance of build object.
273 build
= MozbuildObject
.from_environment()
274 if build
is not None and hasattr(build
, 'mozconfig'):
275 ac_options
= build
.mozconfig
['configure_args']
276 if ac_options
and '--disable-tests' in ac_options
:
277 print('Tests have been disabled by mozconfig with the flag ' +
278 '"ac_add_options --disable-tests".\n' +
279 'Remove the flag, and re-compile to enable tests.')
281 except BuildEnvironmentNotFoundException
:
282 # likely automation environment, so do nothing.
285 def should_skip_telemetry_submission(handler
):
286 # The user is performing a maintenance command.
288 'bootstrap', 'doctor', 'mach-commands', 'vcs-setup',
289 'create-mach-environment',
290 # We call mach environment in client.mk which would cause the
291 # data submission to block the forward progress of make.
295 # Never submit data when running in automation or when running tests.
296 if any(e
in os
.environ
for e
in ('MOZ_AUTOMATION', 'TASK_ID', 'MACH_TELEMETRY_NO_SUBMIT')):
301 def post_dispatch_handler(context
, handler
, instance
, result
,
302 start_time
, end_time
, depth
, args
):
303 """Perform global operations after command dispatch.
306 For now, we will use this to handle build system telemetry.
308 # Don't write telemetry data if this mach command was invoked as part of another
310 if depth
!= 1 or os
.environ
.get('MACH_MAIN_PID') != str(os
.getpid()):
313 from mozbuild
.telemetry
import is_telemetry_enabled
314 if not is_telemetry_enabled(context
.settings
):
317 from mozbuild
.telemetry
import gather_telemetry
318 from mozbuild
.base
import MozbuildObject
319 import mozpack
.path
as mozpath
321 if not isinstance(instance
, MozbuildObject
):
322 instance
= MozbuildObject
.from_environment()
325 substs
= instance
.substs
329 command_attrs
= getattr(context
, 'command_attrs', {})
331 # We gather telemetry for every operation.
333 instance
.topsrcdir
: '$topsrcdir/',
334 instance
.topobjdir
: '$topobjdir/',
335 mozpath
.normpath(os
.path
.expanduser('~')): '$HOME/',
337 # This might override one of the existing entries, that's OK.
338 # We don't use a sigil here because we treat all arguments as potentially relative
339 # paths, so we'd like to get them back as they were specified.
340 paths
[mozpath
.normpath(os
.getcwd())] = ''
341 data
= gather_telemetry(command
=handler
.name
, success
=(result
== 0),
342 start_time
=start_time
, end_time
=end_time
,
343 mach_context
=context
, substs
=substs
,
344 command_attrs
=command_attrs
, paths
=paths
)
346 telemetry_dir
= os
.path
.join(get_state_dir(), 'telemetry')
348 os
.mkdir(telemetry_dir
)
350 if e
.errno
!= errno
.EEXIST
:
352 outgoing_dir
= os
.path
.join(telemetry_dir
, 'outgoing')
354 os
.mkdir(outgoing_dir
)
356 if e
.errno
!= errno
.EEXIST
:
359 with
open(os
.path
.join(outgoing_dir
, str(uuid
.uuid4()) + '.json'),
361 json
.dump(data
, f
, sort_keys
=True)
363 if should_skip_telemetry_submission(handler
):
366 state_dir
= get_state_dir()
368 machpath
= os
.path
.join(instance
.topsrcdir
, 'mach')
369 with
open(os
.devnull
, 'wb') as devnull
:
370 subprocess
.Popen([sys
.executable
, machpath
, 'python',
372 os
.path
.join(topsrcdir
, 'build',
373 'submit_telemetry_data.py'),
375 stdout
=devnull
, stderr
=devnull
)
377 def populate_context(key
=None):
380 if key
== 'state_dir':
381 state_dir
= get_state_dir()
382 if state_dir
== os
.environ
.get('MOZBUILD_STATE_PATH'):
383 if not os
.path
.exists(state_dir
):
384 print('Creating global state directory from environment variable: %s'
386 os
.makedirs(state_dir
, mode
=0o770)
388 if not os
.path
.exists(state_dir
):
389 if not os
.environ
.get('MOZ_AUTOMATION'):
390 print(STATE_DIR_FIRST_RUN
.format(userdir
=state_dir
))
393 except KeyboardInterrupt:
396 print('\nCreating default state directory: %s' % state_dir
)
397 os
.makedirs(state_dir
, mode
=0o770)
401 if key
== 'local_state_dir':
402 return get_state_dir(srcdir
=True)
407 if key
== 'pre_dispatch_handler':
408 return pre_dispatch_handler
410 if key
== 'post_dispatch_handler':
411 return post_dispatch_handler
413 if key
== 'repository':
414 return resolve_repository()
416 raise AttributeError(key
)
418 # Note which process is top-level so that recursive mach invocations can avoid writing
420 if 'MACH_MAIN_PID' not in os
.environ
:
421 setenv('MACH_MAIN_PID', str(os
.getpid()))
423 driver
= mach
.main
.Mach(os
.getcwd())
424 driver
.populate_context_handler
= populate_context
426 if not driver
.settings_paths
:
427 # default global machrc location
428 driver
.settings_paths
.append(get_state_dir())
429 # always load local repository configuration
430 driver
.settings_paths
.append(mozilla_dir
)
432 for category
, meta
in CATEGORIES
.items():
433 driver
.define_category(category
, meta
['short'], meta
['long'],
436 repo
= resolve_repository()
438 for path
in MACH_MODULES
:
439 # Sparse checkouts may not have all mach_commands.py files. Ignore
440 # errors from missing files.
442 driver
.load_commands_from_file(os
.path
.join(mozilla_dir
, path
))
443 except mach
.base
.MissingFileError
:
444 if not repo
or not repo
.sparse_checkout_present():
450 # Hook import such that .pyc/.pyo files without a corresponding .py file in
451 # the source directory are essentially ignored. See further below for details
453 # Objdirs outside the source directory are ignored because in most cases, if
454 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
455 class ImportHook(object):
456 def __init__(self
, original_import
):
457 self
._original
_import
= original_import
458 # Assume the source directory is the parent directory of the one
459 # containing this file.
460 self
._source
_dir
= os
.path
.normcase(os
.path
.abspath(
461 os
.path
.dirname(os
.path
.dirname(__file__
)))) + os
.sep
462 self
._modules
= set()
464 def __call__(self
, name
, globals=None, locals=None, fromlist
=None,
466 if sys
.version_info
[0] >= 3 and level
< 0:
469 # name might be a relative import. Instead of figuring out what that
470 # resolves to, which is complex, just rely on the real import.
471 # Since we don't know the full module name, we can't check sys.modules,
472 # so we need to keep track of which modules we've already seen to avoid
473 # to stat() them again when they are imported multiple times.
474 module
= self
._original
_import
(name
, globals, locals, fromlist
, level
)
476 # Some tests replace modules in sys.modules with non-module instances.
477 if not isinstance(module
, ModuleType
):
480 resolved_name
= module
.__name
__
481 if resolved_name
in self
._modules
:
483 self
._modules
.add(resolved_name
)
485 # Builtin modules don't have a __file__ attribute.
486 if not getattr(module
, '__file__', None):
489 # Note: module.__file__ is not always absolute.
490 path
= os
.path
.normcase(os
.path
.abspath(module
.__file
__))
491 # Note: we could avoid normcase and abspath above for non pyc/pyo
492 # files, but those are actually rare, so it doesn't really matter.
493 if not path
.endswith(('.pyc', '.pyo')):
496 # Ignore modules outside our source directory
497 if not path
.startswith(self
._source
_dir
):
500 # If there is no .py corresponding to the .pyc/.pyo module we're
501 # loading, remove the .pyc/.pyo file, and reload the module.
502 # Since we already loaded the .pyc/.pyo module, if it had side
503 # effects, they will have happened already, and loading the module
504 # with the same name, from another directory may have the same side
505 # effects (or different ones). We assume it's not a problem for the
506 # python modules under our source directory (either because it
507 # doesn't happen or because it doesn't matter).
508 if not os
.path
.exists(module
.__file
__[:-1]):
509 if os
.path
.exists(module
.__file
__):
510 os
.remove(module
.__file
__)
511 del sys
.modules
[module
.__name
__]
512 module
= self(name
, globals, locals, fromlist
, level
)
518 builtins
.__import
__ = ImportHook(builtins
.__import
__)