testing/tools/mach_test_package_bootstrap.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import print_function, unicode_literals
   6
   7 import json
   8 import os
   9 import platform
  10 import sys
  11 import types
  12
  13
  14 SEARCH_PATHS = [
  15     'gtest',
  16     'marionette/client',
  17     'marionette/harness',
  18     'mochitest',
  19     'mozbase/manifestparser',
  20     'mozbase/mozcrash',
  21     'mozbase/mozdebug',
  22     'mozbase/mozdevice',
  23     'mozbase/mozfile',
  24     'mozbase/mozgeckoprofile',
  25     'mozbase/mozhttpd',
  26     'mozbase/mozinfo',
  27     'mozbase/mozinstall',
  28     'mozbase/mozleak',
  29     'mozbase/mozlog',
  30     'mozbase/moznetwork',
  31     'mozbase/mozpower',
  32     'mozbase/mozprocess',
  33     'mozbase/mozprofile',
  34     'mozbase/mozrunner',
  35     'mozbase/mozscreenshot',
  36     'mozbase/mozsystemmonitor',
  37     'mozbase/moztest',
  38     'mozbase/mozversion',
  39     'reftest',
  40     'tools/mach',
  41     'tools/mozterm',
  42     'tools/six',
  43     'tools/wptserve',
  44     'web-platform',
  45     'web-platform/tests/tools/wptrunner',
  46     'xpcshell',
  47 ]
  48
  49 # Individual files providing mach commands.
  50 MACH_MODULES = [
  51     'gtest/mach_test_package_commands.py',
  52     'marionette/mach_test_package_commands.py',
  53     'mochitest/mach_test_package_commands.py',
  54     'reftest/mach_test_package_commands.py',
  55     'tools/mach/mach/commands/commandinfo.py',
  56     'web-platform/mach_test_package_commands.py',
  57     'xpcshell/mach_test_package_commands.py',
  58 ]
  59
  60
  61 CATEGORIES = {
  62     'testing': {
  63         'short': 'Testing',
  64         'long': 'Run tests.',
  65         'priority': 30,
  66     },
  67     'devenv': {
  68         'short': 'Development Environment',
  69         'long': 'Set up and configure your development environment.',
  70         'priority': 20,
  71     },
  72     'misc': {
  73         'short': 'Potpourri',
  74         'long': 'Potent potables and assorted snacks.',
  75         'priority': 10,
  76     },
  77     'disabled': {
  78         'short': 'Disabled',
  79         'long': 'The disabled commands are hidden by default. Use -v to display them. '
  80                 'These commands are unavailable for your current context, '
  81                 'run "mach <command>" to see why.',
  82         'priority': 0,
  83     }
  84 }
  85
  86
  87 IS_WIN = sys.platform in ('win32', 'cygwin')
  88 PY3 = sys.version_info[0] == 3
  89
  90 if PY3:
  91     text_type = str
  92 else:
  93     text_type = unicode  # noqa
  94
  95
  96 def ancestors(path, depth=0):
  97     """Emit the parent directories of a path."""
  98     count = 1
  99     while path and count != depth:
 100         yield path
 101         newpath = os.path.dirname(path)
 102         if newpath == path:
 103             break
 104         path = newpath
 105         count += 1
 106
 107
 108 def activate_mozharness_venv(context):
 109     """Activate the mozharness virtualenv in-process."""
 110     venv = os.path.join(context.mozharness_workdir,
 111                         context.mozharness_config.get('virtualenv_path', 'venv'))
 112
 113     if not os.path.isdir(venv):
 114         print("No mozharness virtualenv detected at '{}'.".format(venv))
 115         return 1
 116
 117     venv_bin = os.path.join(venv, 'Scripts' if IS_WIN else 'bin')
 118     activate_path = os.path.join(venv_bin, 'activate_this.py')
 119
 120     exec(open(activate_path).read(), dict(__file__=activate_path))
 121
 122     if isinstance(os.environ['PATH'], text_type):
 123         os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
 124
 125     # sys.executable is used by mochitest-media to start the websocketprocessbridge,
 126     # for some reason it doesn't get set when calling `activate_this.py` so set it
 127     # here instead.
 128     binary = 'python'
 129     if IS_WIN:
 130         binary += '.exe'
 131     sys.executable = os.path.join(venv_bin, binary)
 132
 133
 134 def find_firefox(context):
 135     """Try to automagically find the firefox binary."""
 136     import mozinstall
 137     search_paths = []
 138
 139     # Check for a mozharness setup
 140     config = context.mozharness_config
 141     if config and 'binary_path' in config:
 142         return config['binary_path']
 143     elif config:
 144         search_paths.append(os.path.join(context.mozharness_workdir, 'application'))
 145
 146     # Check for test-stage setup
 147     dist_bin = os.path.join(os.path.dirname(context.package_root), 'bin')
 148     if os.path.isdir(dist_bin):
 149         search_paths.append(dist_bin)
 150
 151     for path in search_paths:
 152         try:
 153             return mozinstall.get_binary(path, 'firefox')
 154         except mozinstall.InvalidBinary:
 155             continue
 156
 157
 158 def find_hostutils(context):
 159     workdir = context.mozharness_workdir
 160     hostutils = os.path.join(workdir, 'hostutils')
 161     for fname in os.listdir(hostutils):
 162         fpath = os.path.join(hostutils, fname)
 163         if os.path.isdir(fpath) and fname.startswith('host-utils'):
 164             return fpath
 165
 166
 167 def normalize_test_path(test_root, path):
 168     if os.path.isabs(path) or os.path.exists(path):
 169         return os.path.normpath(os.path.abspath(path))
 170
 171     for parent in ancestors(test_root):
 172         test_path = os.path.join(parent, path)
 173         if os.path.exists(test_path):
 174             return os.path.normpath(os.path.abspath(test_path))
 175     # Not a valid path? Return as is and let test harness deal with it
 176     return path
 177
 178
 179 def bootstrap(test_package_root):
 180     test_package_root = os.path.abspath(test_package_root)
 181
 182     # Ensure we are running Python 2.7+. We put this check here so we generate a
 183     # user-friendly error message rather than a cryptic stack trace on module
 184     # import.
 185     if sys.version_info[0] != 2 or sys.version_info[1] < 7:
 186         print('Python 2.7 or above (but not Python 3) is required to run mach.')
 187         print('You are running Python', platform.python_version())
 188         sys.exit(1)
 189
 190     sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
 191     import mach.main
 192
 193     def populate_context(context, key=None):
 194         if key is None:
 195             context.package_root = test_package_root
 196             context.bin_dir = os.path.join(test_package_root, 'bin')
 197             context.certs_dir = os.path.join(test_package_root, 'certs')
 198             context.module_dir = os.path.join(test_package_root, 'modules')
 199             context.ancestors = ancestors
 200             context.normalize_test_path = normalize_test_path
 201             return
 202
 203         # The values for the following 'key's will be set lazily, and cached
 204         # after first being invoked.
 205         if key == 'firefox_bin':
 206             return find_firefox(context)
 207
 208         if key == 'hostutils':
 209             return find_hostutils(context)
 210
 211         if key == 'mozharness_config':
 212             for dir_path in ancestors(context.package_root):
 213                 mozharness_config = os.path.join(dir_path, 'logs', 'localconfig.json')
 214                 if os.path.isfile(mozharness_config):
 215                     with open(mozharness_config, 'rb') as f:
 216                         return json.load(f)
 217             return {}
 218
 219         if key == 'mozharness_workdir':
 220             config = context.mozharness_config
 221             if config:
 222                 return os.path.join(config['base_work_dir'], config['work_dir'])
 223
 224         if key == 'activate_mozharness_venv':
 225             return types.MethodType(activate_mozharness_venv, context)
 226
 227     mach = mach.main.Mach(os.getcwd())
 228     mach.populate_context_handler = populate_context
 229
 230     for category, meta in CATEGORIES.items():
 231         mach.define_category(category, meta['short'], meta['long'],
 232                              meta['priority'])
 233
 234     for path in MACH_MODULES:
 235         cmdfile = os.path.join(test_package_root, path)
 236
 237         # Depending on which test zips were extracted,
 238         # the command module might not exist
 239         if os.path.isfile(cmdfile):
 240             mach.load_commands_from_file(cmdfile)
 241
 242     return mach