1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import print_function
, unicode_literals
14 if sys
.version_info
[0] < 3:
15 import __builtin__
as builtins
19 from types
import ModuleType
22 STATE_DIR_FIRST_RUN
= '''
23 mach and the build system store shared state in a common directory on the
24 filesystem. The following directory will be created:
28 If you would like to use a different directory, hit CTRL+c and set the
29 MOZBUILD_STATE_PATH environment variable to the directory you would like to
30 use and re-run mach. For this change to take effect forever, you'll likely
31 want to export this environment variable from your shell's init scripts.
33 Press ENTER/RETURN to continue or CTRL+c to abort.
37 # Individual files providing mach commands.
39 'build/valgrind/mach_commands.py',
40 'devtools/shared/css/generated/mach_commands.py',
41 'dom/bindings/mach_commands.py',
42 'layout/tools/reftest/mach_commands.py',
43 'python/mach_commands.py',
44 'python/safety/mach_commands.py',
45 'python/mach/mach/commands/commandinfo.py',
46 'python/mach/mach/commands/settings.py',
47 'python/mozboot/mozboot/mach_commands.py',
48 'python/mozbuild/mozbuild/mach_commands.py',
49 'python/mozbuild/mozbuild/backend/mach_commands.py',
50 'python/mozbuild/mozbuild/compilation/codecomplete.py',
51 'python/mozbuild/mozbuild/frontend/mach_commands.py',
52 'python/mozrelease/mozrelease/mach_commands.py',
53 'taskcluster/mach_commands.py',
54 'testing/awsy/mach_commands.py',
55 'testing/firefox-ui/mach_commands.py',
56 'testing/geckodriver/mach_commands.py',
57 'testing/mach_commands.py',
58 'testing/marionette/mach_commands.py',
59 'testing/mochitest/mach_commands.py',
60 'testing/mozharness/mach_commands.py',
61 'testing/raptor/mach_commands.py',
62 'testing/tps/mach_commands.py',
63 'testing/talos/mach_commands.py',
64 'testing/web-platform/mach_commands.py',
65 'testing/xpcshell/mach_commands.py',
66 'toolkit/components/telemetry/tests/marionette/mach_commands.py',
67 'tools/browsertime/mach_commands.py',
68 'tools/compare-locales/mach_commands.py',
69 'tools/docs/mach_commands.py',
70 'tools/lint/mach_commands.py',
71 'tools/mach_commands.py',
72 'tools/power/mach_commands.py',
73 'tools/tryselect/mach_commands.py',
74 'mobile/android/mach_commands.py',
80 'short': 'Build Commands',
81 'long': 'Interact with the build system',
85 'short': 'Post-build Commands',
86 'long': 'Common actions performed after completing a build.',
96 'long': 'Taskcluster commands',
100 'short': 'Development Environment',
101 'long': 'Set up and configure your development environment.',
105 'short': 'Low-level Build System Interaction',
106 'long': 'Interact with specific parts of the build system.',
110 'short': 'Potpourri',
111 'long': 'Potent potables and assorted snacks.',
115 'short': 'Release automation',
116 'long': 'Commands for used in release automation.',
121 'long': 'The disabled commands are hidden by default. Use -v to display them. '
122 'These commands are unavailable for your current context, '
123 'run "mach <command>" to see why.',
129 def search_path(mozilla_dir
, packages_txt
):
130 with
open(os
.path
.join(mozilla_dir
, packages_txt
)) as f
:
131 packages
= [line
.rstrip().split(':') for line
in f
]
133 def handle_package(package
):
134 if package
[0] == 'optional':
136 for path
in handle_package(package
[1:]):
141 if package
[0] in ('windows', '!windows'):
142 for_win
= not package
[0].startswith('!')
143 is_win
= sys
.platform
== 'win32'
144 if is_win
== for_win
:
145 for path
in handle_package(package
[1:]):
148 if package
[0] in ('python2', 'python3'):
149 for_python3
= package
[0].endswith('3')
150 is_python3
= sys
.version_info
[0] > 2
151 if is_python3
== for_python3
:
152 for path
in handle_package(package
[1:]):
155 if package
[0] == 'packages.txt':
156 assert len(package
) == 2
157 for p
in search_path(mozilla_dir
, package
[1]):
158 yield os
.path
.join(mozilla_dir
, p
)
160 if package
[0].endswith('.pth'):
161 assert len(package
) == 2
162 yield os
.path
.join(mozilla_dir
, package
[1])
164 for package
in packages
:
165 for path
in handle_package(package
):
169 def bootstrap(topsrcdir
, mozilla_dir
=None):
170 if mozilla_dir
is None:
171 mozilla_dir
= topsrcdir
173 # Ensure we are running Python 2.7+. We put this check here so we generate a
174 # user-friendly error message rather than a cryptic stack trace on module
176 if sys
.version_info
[0] != 2 or sys
.version_info
[1] < 7:
177 print('Python 2.7 or above (but not Python 3) is required to run mach.')
178 print('You are running Python', platform
.python_version())
181 # Global build system and mach state is stored in a central directory. By
182 # default, this is ~/.mozbuild. However, it can be defined via an
183 # environment variable. We detect first run (by lack of this directory
184 # existing) and notify the user that it will be created. The logic for
185 # creation is much simpler for the "advanced" environment variable use
186 # case. For default behavior, we educate users and give them an opportunity
187 # to react. We always exit after creating the directory because users don't
189 sys
.path
[0:0] = [os
.path
.join(mozilla_dir
, path
)
190 for path
in search_path(mozilla_dir
,
191 'build/virtualenv_packages.txt')]
194 from mozboot
.util
import get_state_dir
196 from mozbuild
.util
import patch_main
199 def resolve_repository():
200 import mozversioncontrol
203 # This API doesn't respect the vcs binary choices from configure.
204 # If we ever need to use the VCS binary here, consider something
206 return mozversioncontrol
.get_repository_object(path
=mozilla_dir
)
207 except (mozversioncontrol
.InvalidRepoPath
,
208 mozversioncontrol
.MissingVCSTool
):
211 def should_skip_telemetry_submission(handler
):
212 # The user is performing a maintenance command.
213 if handler
.name
in ('bootstrap', 'doctor', 'mach-commands', 'vcs-setup',
214 # We call mach environment in client.mk which would cause the
215 # data submission to block the forward progress of make.
219 # Never submit data when running in automation or when running tests.
220 if any(e
in os
.environ
for e
in ('MOZ_AUTOMATION', 'TASK_ID', 'MACH_TELEMETRY_NO_SUBMIT')):
225 def post_dispatch_handler(context
, handler
, instance
, result
,
226 start_time
, end_time
, depth
, args
):
227 """Perform global operations after command dispatch.
230 For now, we will use this to handle build system telemetry.
232 # Don't write telemetry data if this mach command was invoked as part of another
234 if depth
!= 1 or os
.environ
.get('MACH_MAIN_PID') != str(os
.getpid()):
237 # Don't write telemetry data for 'mach' when 'DISABLE_TELEMETRY' is set.
238 if os
.environ
.get('DISABLE_TELEMETRY') == '1':
241 # We have not opted-in to telemetry
242 if not context
.settings
.build
.telemetry
:
245 from mozbuild
.telemetry
import gather_telemetry
246 from mozbuild
.base
import MozbuildObject
247 import mozpack
.path
as mozpath
249 if not isinstance(instance
, MozbuildObject
):
250 instance
= MozbuildObject
.from_environment()
253 substs
= instance
.substs
257 command_attrs
= getattr(context
, 'command_attrs', {})
259 # We gather telemetry for every operation.
261 instance
.topsrcdir
: '$topsrcdir/',
262 instance
.topobjdir
: '$topobjdir/',
263 mozpath
.normpath(os
.path
.expanduser('~')): '$HOME/',
265 # This might override one of the existing entries, that's OK.
266 # We don't use a sigil here because we treat all arguments as potentially relative
267 # paths, so we'd like to get them back as they were specified.
268 paths
[mozpath
.normpath(os
.getcwd())] = ''
269 data
= gather_telemetry(command
=handler
.name
, success
=(result
== 0),
270 start_time
=start_time
, end_time
=end_time
,
271 mach_context
=context
, substs
=substs
,
272 command_attrs
=command_attrs
, paths
=paths
)
274 telemetry_dir
= os
.path
.join(get_state_dir(), 'telemetry')
276 os
.mkdir(telemetry_dir
)
278 if e
.errno
!= errno
.EEXIST
:
280 outgoing_dir
= os
.path
.join(telemetry_dir
, 'outgoing')
282 os
.mkdir(outgoing_dir
)
284 if e
.errno
!= errno
.EEXIST
:
287 with
open(os
.path
.join(outgoing_dir
, str(uuid
.uuid4()) + '.json'),
289 json
.dump(data
, f
, sort_keys
=True)
291 if should_skip_telemetry_submission(handler
):
294 state_dir
= get_state_dir()
296 machpath
= os
.path
.join(instance
.topsrcdir
, 'mach')
297 with
open(os
.devnull
, 'wb') as devnull
:
298 subprocess
.Popen([sys
.executable
, machpath
, 'python',
300 os
.path
.join(topsrcdir
, 'build',
301 'submit_telemetry_data.py'),
303 stdout
=devnull
, stderr
=devnull
)
305 def populate_context(context
, key
=None):
308 if key
== 'state_dir':
309 state_dir
= get_state_dir()
310 if state_dir
== os
.environ
.get('MOZBUILD_STATE_PATH'):
311 if not os
.path
.exists(state_dir
):
312 print('Creating global state directory from environment variable: %s'
314 os
.makedirs(state_dir
, mode
=0o770)
316 if not os
.path
.exists(state_dir
):
317 if not os
.environ
.get('MOZ_AUTOMATION'):
318 print(STATE_DIR_FIRST_RUN
.format(userdir
=state_dir
))
321 except KeyboardInterrupt:
324 print('\nCreating default state directory: %s' % state_dir
)
325 os
.makedirs(state_dir
, mode
=0o770)
329 if key
== 'local_state_dir':
330 return get_state_dir(srcdir
=True)
335 if key
== 'post_dispatch_handler':
336 return post_dispatch_handler
338 if key
== 'repository':
339 return resolve_repository()
341 raise AttributeError(key
)
343 # Note which process is top-level so that recursive mach invocations can avoid writing
345 if 'MACH_MAIN_PID' not in os
.environ
:
346 os
.environ
[b
'MACH_MAIN_PID'] = str(os
.getpid()).encode('ascii')
348 driver
= mach
.main
.Mach(os
.getcwd())
349 driver
.populate_context_handler
= populate_context
351 if not driver
.settings_paths
:
352 # default global machrc location
353 driver
.settings_paths
.append(get_state_dir())
354 # always load local repository configuration
355 driver
.settings_paths
.append(mozilla_dir
)
357 for category
, meta
in CATEGORIES
.items():
358 driver
.define_category(category
, meta
['short'], meta
['long'],
361 repo
= resolve_repository()
363 for path
in MACH_MODULES
:
364 # Sparse checkouts may not have all mach_commands.py files. Ignore
365 # errors from missing files.
367 driver
.load_commands_from_file(os
.path
.join(mozilla_dir
, path
))
368 except mach
.base
.MissingFileError
:
369 if not repo
or not repo
.sparse_checkout_present():
375 # Hook import such that .pyc/.pyo files without a corresponding .py file in
376 # the source directory are essentially ignored. See further below for details
378 # Objdirs outside the source directory are ignored because in most cases, if
379 # a .pyc/.pyo file exists there, a .py file will be next to it anyways.
380 class ImportHook(object):
381 def __init__(self
, original_import
):
382 self
._original
_import
= original_import
383 # Assume the source directory is the parent directory of the one
384 # containing this file.
385 self
._source
_dir
= os
.path
.normcase(os
.path
.abspath(
386 os
.path
.dirname(os
.path
.dirname(__file__
)))) + os
.sep
387 self
._modules
= set()
389 def __call__(self
, name
, globals=None, locals=None, fromlist
=None,
391 # name might be a relative import. Instead of figuring out what that
392 # resolves to, which is complex, just rely on the real import.
393 # Since we don't know the full module name, we can't check sys.modules,
394 # so we need to keep track of which modules we've already seen to avoid
395 # to stat() them again when they are imported multiple times.
396 module
= self
._original
_import
(name
, globals, locals, fromlist
, level
)
398 # Some tests replace modules in sys.modules with non-module instances.
399 if not isinstance(module
, ModuleType
):
402 resolved_name
= module
.__name
__
403 if resolved_name
in self
._modules
:
405 self
._modules
.add(resolved_name
)
407 # Builtin modules don't have a __file__ attribute.
408 if not hasattr(module
, '__file__'):
411 # Note: module.__file__ is not always absolute.
412 path
= os
.path
.normcase(os
.path
.abspath(module
.__file
__))
413 # Note: we could avoid normcase and abspath above for non pyc/pyo
414 # files, but those are actually rare, so it doesn't really matter.
415 if not path
.endswith(('.pyc', '.pyo')):
418 # Ignore modules outside our source directory
419 if not path
.startswith(self
._source
_dir
):
422 # If there is no .py corresponding to the .pyc/.pyo module we're
423 # loading, remove the .pyc/.pyo file, and reload the module.
424 # Since we already loaded the .pyc/.pyo module, if it had side
425 # effects, they will have happened already, and loading the module
426 # with the same name, from another directory may have the same side
427 # effects (or different ones). We assume it's not a problem for the
428 # python modules under our source directory (either because it
429 # doesn't happen or because it doesn't matter).
430 if not os
.path
.exists(module
.__file
__[:-1]):
431 if os
.path
.exists(module
.__file
__):
432 os
.remove(module
.__file
__)
433 del sys
.modules
[module
.__name
__]
434 module
= self(name
, globals, locals, fromlist
, level
)
440 builtins
.__import
__ = ImportHook(builtins
.__import
__)