1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import print_function
, unicode_literals
14 if sys
.version_info
[0] < 3:
15 import __builtin__
as builtins
19 from types
import ModuleType
22 STATE_DIR_FIRST_RUN
= '''
23 mach and the build system store shared state in a common directory on the
24 filesystem. The following directory will be created:
28 If you would like to use a different directory, hit CTRL+c and set the
29 MOZBUILD_STATE_PATH environment variable to the directory you would like to
30 use and re-run mach. For this change to take effect forever, you'll likely
31 want to export this environment variable from your shell's init scripts.
33 Press ENTER/RETURN to continue or CTRL+c to abort.
37 # Individual files providing mach commands.
39 'build/valgrind/mach_commands.py',
40 'devtools/shared/css/generated/mach_commands.py',
41 'dom/bindings/mach_commands.py',
42 'layout/tools/reftest/mach_commands.py',
43 'mobile/android/mach_commands.py',
44 'python/mach/mach/commands/commandinfo.py',
45 'python/mach/mach/commands/settings.py',
46 'python/mach_commands.py',
47 'python/mozboot/mozboot/mach_commands.py',
48 'python/mozbuild/mozbuild/artifact_commands.py',
49 'python/mozbuild/mozbuild/backend/mach_commands.py',
50 'python/mozbuild/mozbuild/build_commands.py',
51 'python/mozbuild/mozbuild/code-analysis/mach_commands.py',
52 'python/mozbuild/mozbuild/compilation/codecomplete.py',
53 'python/mozbuild/mozbuild/frontend/mach_commands.py',
54 'python/mozbuild/mozbuild/mach_commands.py',
55 'python/mozrelease/mozrelease/mach_commands.py',
56 'python/safety/mach_commands.py',
57 'remote/mach_commands.py',
58 'taskcluster/mach_commands.py',
59 'testing/awsy/mach_commands.py',
60 'testing/firefox-ui/mach_commands.py',
61 'testing/geckodriver/mach_commands.py',
62 'testing/mach_commands.py',
63 'testing/marionette/mach_commands.py',
64 'testing/mochitest/mach_commands.py',
65 'testing/mozharness/mach_commands.py',
66 'testing/raptor/mach_commands.py',
67 'testing/talos/mach_commands.py',
68 'testing/tps/mach_commands.py',
69 'testing/web-platform/mach_commands.py',
70 'testing/xpcshell/mach_commands.py',
71 'toolkit/components/telemetry/tests/marionette/mach_commands.py',
72 'tools/browsertime/mach_commands.py',
73 'tools/compare-locales/mach_commands.py',
74 'tools/docs/mach_commands.py',
75 'tools/lint/mach_commands.py',
76 'tools/mach_commands.py',
77 'tools/power/mach_commands.py',
78 'tools/tryselect/mach_commands.py',
79 'tools/vcs/mach_commands.py',
85 'short': 'Build Commands',
86 'long': 'Interact with the build system',
90 'short': 'Post-build Commands',
91 'long': 'Common actions performed after completing a build.',
101 'long': 'Taskcluster commands',
105 'short': 'Development Environment',
106 'long': 'Set up and configure your development environment.',
110 'short': 'Low-level Build System Interaction',
111 'long': 'Interact with specific parts of the build system.',
115 'short': 'Potpourri',
116 'long': 'Potent potables and assorted snacks.',
120 'short': 'Release automation',
121 'long': 'Commands for used in release automation.',
126 'long': 'The disabled commands are hidden by default. Use -v to display them. '
127 'These commands are unavailable for your current context, '
128 'run "mach <command>" to see why.',
134 def search_path(mozilla_dir
, packages_txt
):
135 with
open(os
.path
.join(mozilla_dir
, packages_txt
)) as f
:
136 packages
= [line
.rstrip().split(':') for line
in f
]
138 def handle_package(package
):
139 if package
[0] == 'optional':
141 for path
in handle_package(package
[1:]):
146 if package
[0] in ('windows', '!windows'):
147 for_win
= not package
[0].startswith('!')
148 is_win
= sys
.platform
== 'win32'
149 if is_win
== for_win
:
150 for path
in handle_package(package
[1:]):
153 if package
[0] in ('python2', 'python3'):
154 for_python3
= package
[0].endswith('3')
155 is_python3
= sys
.version_info
[0] > 2
156 if is_python3
== for_python3
:
157 for path
in handle_package(package
[1:]):
160 if package
[0] == 'packages.txt':
161 assert len(package
) == 2
162 for p
in search_path(mozilla_dir
, package
[1]):
163 yield os
.path
.join(mozilla_dir
, p
)
165 if package
[0].endswith('.pth'):
166 assert len(package
) == 2
167 yield os
.path
.join(mozilla_dir
, package
[1])
169 for package
in packages
:
170 for path
in handle_package(package
):
174 def bootstrap(topsrcdir
, mozilla_dir
=None):
175 if mozilla_dir
is None:
176 mozilla_dir
= topsrcdir
178 # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
179 # generate a user-friendly error message rather than a cryptic stack trace
181 major
, minor
= sys
.version_info
[:2]
182 if (major
== 2 and minor
< 7) or (major
== 3 and minor
< 5):
183 print('Python 2.7 or Python 3.5+ is required to run mach.')
184 print('You are running Python', platform
.python_version())
187 # Global build system and mach state is stored in a central directory. By
188 # default, this is ~/.mozbuild. However, it can be defined via an
189 # environment variable. We detect first run (by lack of this directory
190 # existing) and notify the user that it will be created. The logic for
191 # creation is much simpler for the "advanced" environment variable use
192 # case. For default behavior, we educate users and give them an opportunity
193 # to react. We always exit after creating the directory because users don't
195 sys
.path
[0:0] = [os
.path
.join(mozilla_dir
, path
)
196 for path
in search_path(mozilla_dir
,
197 'build/virtualenv_packages.txt')]
200 from mach
.util
import setenv
201 from mozboot
.util
import get_state_dir
203 from mozbuild
.util
import patch_main
206 def resolve_repository():
207 import mozversioncontrol
210 # This API doesn't respect the vcs binary choices from configure.
211 # If we ever need to use the VCS binary here, consider something
213 return mozversioncontrol
.get_repository_object(path
=mozilla_dir
)
214 except (mozversioncontrol
.InvalidRepoPath
,
215 mozversioncontrol
.MissingVCSTool
):
218 def pre_dispatch_handler(context
, handler
, args
):
219 # If --disable-tests flag was enabled in the mozconfig used to compile
220 # the build, tests will be disabled. Instead of trying to run
221 # nonexistent tests then reporting a failure, this will prevent mach
222 # from progressing beyond this point.
223 if handler
.category
== 'testing':
224 from mozbuild
.base
import BuildEnvironmentNotFoundException
226 from mozbuild
.base
import MozbuildObject
227 # all environments should have an instance of build object.
228 build
= MozbuildObject
.from_environment()
229 if build
is not None and hasattr(build
, 'mozconfig'):
230 ac_options
= build
.mozconfig
['configure_args']
231 if ac_options
and '--disable-tests' in ac_options
:
232 print('Tests have been disabled by mozconfig with the flag ' +
233 '"ac_add_options --disable-tests".\n' +
234 'Remove the flag, and re-compile to enable tests.')
236 except BuildEnvironmentNotFoundException
:
237 # likely automation environment, so do nothing.
240 def should_skip_telemetry_submission(handler
):
241 # The user is performing a maintenance command.
242 if handler
.name
in ('bootstrap', 'doctor', 'mach-commands', 'vcs-setup',
243 # We call mach environment in client.mk which would cause the
244 # data submission to block the forward progress of make.
248 # Never submit data when running in automation or when running tests.
249 if any(e
in os
.environ
for e
in ('MOZ_AUTOMATION', 'TASK_ID', 'MACH_TELEMETRY_NO_SUBMIT')):
254 def post_dispatch_handler(context
, handler
, instance
, result
,
255 start_time
, end_time
, depth
, args
):
256 """Perform global operations after command dispatch.
259 For now, we will use this to handle build system telemetry.
261 # Don't write telemetry data if this mach command was invoked as part of another
263 if depth
!= 1 or os
.environ
.get('MACH_MAIN_PID') != str(os
.getpid()):
266 # Don't write telemetry data for 'mach' when 'DISABLE_TELEMETRY' is set.
267 if os
.environ
.get('DISABLE_TELEMETRY') == '1':
270 # We have not opted-in to telemetry
271 if not context
.settings
.build
.telemetry
:
274 from mozbuild
.telemetry
import gather_telemetry
275 from mozbuild
.base
import MozbuildObject
276 import mozpack
.path
as mozpath
278 if not isinstance(instance
, MozbuildObject
):
279 instance
= MozbuildObject
.from_environment()
282 substs
= instance
.substs
286 command_attrs
= getattr(context
, 'command_attrs', {})
288 # We gather telemetry for every operation.
290 instance
.topsrcdir
: '$topsrcdir/',
291 instance
.topobjdir
: '$topobjdir/',
292 mozpath
.normpath(os
.path
.expanduser('~')): '$HOME/',
294 # This might override one of the existing entries, that's OK.
295 # We don't use a sigil here because we treat all arguments as potentially relative
296 # paths, so we'd like to get them back as they were specified.
297 paths
[mozpath
.normpath(os
.getcwd())] = ''
298 data
= gather_telemetry(command
=handler
.name
, success
=(result
== 0),
299 start_time
=start_time
, end_time
=end_time
,
300 mach_context
=context
, substs
=substs
,
301 command_attrs
=command_attrs
, paths
=paths
)
303 telemetry_dir
= os
.path
.join(get_state_dir(), 'telemetry')
305 os
.mkdir(telemetry_dir
)
307 if e
.errno
!= errno
.EEXIST
:
309 outgoing_dir
= os
.path
.join(telemetry_dir
, 'outgoing')
311 os
.mkdir(outgoing_dir
)
313 if e
.errno
!= errno
.EEXIST
:
316 with
open(os
.path
.join(outgoing_dir
, str(uuid
.uuid4()) + '.json'),
318 json
.dump(data
, f
, sort_keys
=True)
320 if should_skip_telemetry_submission(handler
):
323 state_dir
= get_state_dir()
325 machpath
= os
.path
.join(instance
.topsrcdir
, 'mach')
326 with
open(os
.devnull
, 'wb') as devnull
:
327 subprocess
.Popen([sys
.executable
, machpath
, 'python',
329 os
.path
.join(topsrcdir
, 'build',
330 'submit_telemetry_data.py'),
332 stdout
=devnull
, stderr
=devnull
)
334 def populate_context(context
, key
=None):
337 if key
== 'state_dir':
338 state_dir
= get_state_dir()
339 if state_dir
== os
.environ
.get('MOZBUILD_STATE_PATH'):
340 if not os
.path
.exists(state_dir
):
341 print('Creating global state directory from environment variable: %s'
343 os
.makedirs(state_dir
, mode
=0o770)
345 if not os
.path
.exists(state_dir
):
346 if not os
.environ
.get('MOZ_AUTOMATION'):
347 print(STATE_DIR_FIRST_RUN
.format(userdir
=state_dir
))
350 except KeyboardInterrupt:
353 print('\nCreating default state directory: %s' % state_dir
)
354 os
.makedirs(state_dir
, mode
=0o770)
358 if key
== 'local_state_dir':
359 return get_state_dir(srcdir
=True)
364 if key
== 'pre_dispatch_handler':
365 return pre_dispatch_handler
367 if key
== 'post_dispatch_handler':
368 return post_dispatch_handler
370 if key
== 'repository':
371 return resolve_repository()
373 raise AttributeError(key
)
375 # Note which process is top-level so that recursive mach invocations can avoid writing
377 if 'MACH_MAIN_PID' not in os
.environ
:
378 setenv('MACH_MAIN_PID', str(os
.getpid()))
380 driver
= mach
.main
.Mach(os
.getcwd())
381 driver
.populate_context_handler
= populate_context
383 if not driver
.settings_paths
:
384 # default global machrc location
385 driver
.settings_paths
.append(get_state_dir())
386 # always load local repository configuration
387 driver
.settings_paths
.append(mozilla_dir
)
389 for category
, meta
in CATEGORIES
.items():
390 driver
.define_category(category
, meta
['short'], meta
['long'],
393 repo
= resolve_repository()
395 for path
in MACH_MODULES
:
396 # Sparse checkouts may not have all mach_commands.py files. Ignore
397 # errors from missing files.
399 driver
.load_commands_from_file(os
.path
.join(mozilla_dir
, path
))
400 except mach
.base
.MissingFileError
:
401 if not repo
or not repo
.sparse_checkout_present():
407 # Hook import such that .pyc/.pyo files without a corresponding .py file in
408 # the source directory are essentially ignored. See further below for details
410 # Objdirs outside the source directory are ignored because in most cases, if
411 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
412 class ImportHook(object):
413 def __init__(self
, original_import
):
414 self
._original
_import
= original_import
415 # Assume the source directory is the parent directory of the one
416 # containing this file.
417 self
._source
_dir
= os
.path
.normcase(os
.path
.abspath(
418 os
.path
.dirname(os
.path
.dirname(__file__
)))) + os
.sep
419 self
._modules
= set()
421 def __call__(self
, name
, globals=None, locals=None, fromlist
=None,
423 if sys
.version_info
[0] >= 3 and level
< 0:
426 # name might be a relative import. Instead of figuring out what that
427 # resolves to, which is complex, just rely on the real import.
428 # Since we don't know the full module name, we can't check sys.modules,
429 # so we need to keep track of which modules we've already seen to avoid
430 # to stat() them again when they are imported multiple times.
431 module
= self
._original
_import
(name
, globals, locals, fromlist
, level
)
433 # Some tests replace modules in sys.modules with non-module instances.
434 if not isinstance(module
, ModuleType
):
437 resolved_name
= module
.__name
__
438 if resolved_name
in self
._modules
:
440 self
._modules
.add(resolved_name
)
442 # Builtin modules don't have a __file__ attribute.
443 if not getattr(module
, '__file__', None):
446 # Note: module.__file__ is not always absolute.
447 path
= os
.path
.normcase(os
.path
.abspath(module
.__file
__))
448 # Note: we could avoid normcase and abspath above for non pyc/pyo
449 # files, but those are actually rare, so it doesn't really matter.
450 if not path
.endswith(('.pyc', '.pyo')):
453 # Ignore modules outside our source directory
454 if not path
.startswith(self
._source
_dir
):
457 # If there is no .py corresponding to the .pyc/.pyo module we're
458 # loading, remove the .pyc/.pyo file, and reload the module.
459 # Since we already loaded the .pyc/.pyo module, if it had side
460 # effects, they will have happened already, and loading the module
461 # with the same name, from another directory may have the same side
462 # effects (or different ones). We assume it's not a problem for the
463 # python modules under our source directory (either because it
464 # doesn't happen or because it doesn't matter).
465 if not os
.path
.exists(module
.__file
__[:-1]):
466 if os
.path
.exists(module
.__file
__):
467 os
.remove(module
.__file
__)
468 del sys
.modules
[module
.__name
__]
469 module
= self(name
, globals, locals, fromlist
, level
)
475 builtins
.__import
__ = ImportHook(builtins
.__import
__)