testing/tools/mach_test_package_bootstrap.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import print_function, unicode_literals
   6
   7 import json
   8 import os
   9 import platform
  10 import sys
  11 import types
  12
  13
  14 SEARCH_PATHS = [
  15     'marionette/harness',
  16     'marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py',
  17     'marionette/client',
  18     'mochitest',
  19     'mozbase/manifestparser',
  20     'mozbase/mozcrash',
  21     'mozbase/mozdebug',
  22     'mozbase/mozdevice',
  23     'mozbase/mozfile',
  24     'mozbase/mozhttpd',
  25     'mozbase/mozinfo',
  26     'mozbase/mozinstall',
  27     'mozbase/mozleak',
  28     'mozbase/mozlog',
  29     'mozbase/moznetwork',
  30     'mozbase/mozprocess',
  31     'mozbase/mozprofile',
  32     'mozbase/mozrunner',
  33     'mozbase/mozscreenshot',
  34     'mozbase/mozsystemmonitor',
  35     'mozbase/moztest',
  36     'mozbase/mozversion',
  37     'reftest',
  38     'tools/mach',
  39     'tools/mozterm',
  40     'tools/six',
  41     'tools/wptserve',
  42     'web-platform',
  43     'web-platform/tests/tools/wptrunner',
  44     'xpcshell',
  45 ]
  46
  47 # Individual files providing mach commands.
  48 MACH_MODULES = [
  49     'marionette/mach_test_package_commands.py',
  50     'mochitest/mach_test_package_commands.py',
  51     'reftest/mach_test_package_commands.py',
  52     'tools/mach/mach/commands/commandinfo.py',
  53     'web-platform/mach_test_package_commands.py',
  54     'xpcshell/mach_test_package_commands.py',
  55 ]
  56
  57
  58 CATEGORIES = {
  59     'testing': {
  60         'short': 'Testing',
  61         'long': 'Run tests.',
  62         'priority': 30,
  63     },
  64     'devenv': {
  65         'short': 'Development Environment',
  66         'long': 'Set up and configure your development environment.',
  67         'priority': 20,
  68     },
  69     'misc': {
  70         'short': 'Potpourri',
  71         'long': 'Potent potables and assorted snacks.',
  72         'priority': 10,
  73     },
  74     'disabled': {
  75         'short': 'Disabled',
  76         'long': 'The disabled commands are hidden by default. Use -v to display them. '
  77                 'These commands are unavailable for your current context, '
  78                 'run "mach <command>" to see why.',
  79         'priority': 0,
  80     }
  81 }
  82
  83
  84 IS_WIN = sys.platform in ('win32', 'cygwin')
  85
  86
  87 def ancestors(path, depth=0):
  88     """Emit the parent directories of a path."""
  89     count = 1
  90     while path and count != depth:
  91         yield path
  92         newpath = os.path.dirname(path)
  93         if newpath == path:
  94             break
  95         path = newpath
  96         count += 1
  97
  98
  99 def activate_mozharness_venv(context):
 100     """Activate the mozharness virtualenv in-process."""
 101     venv = os.path.join(context.mozharness_workdir,
 102                         context.mozharness_config.get('virtualenv_path', 'venv'))
 103
 104     if not os.path.isdir(venv):
 105         print("No mozharness virtualenv detected at '{}'.".format(venv))
 106         return 1
 107
 108     venv_bin = os.path.join(venv, 'Scripts' if IS_WIN else 'bin')
 109     activate_path = os.path.join(venv_bin, 'activate_this.py')
 110
 111     execfile(activate_path, dict(__file__=activate_path))
 112
 113     if isinstance(os.environ['PATH'], unicode):
 114         os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
 115
 116     # sys.executable is used by mochitest-media to start the websocketprocessbridge,
 117     # for some reason it doesn't get set when calling `activate_this.py` so set it
 118     # here instead.
 119     binary = 'python'
 120     if IS_WIN:
 121         binary += '.exe'
 122     sys.executable = os.path.join(venv_bin, binary)
 123
 124
 125 def find_firefox(context):
 126     """Try to automagically find the firefox binary."""
 127     import mozinstall
 128     search_paths = []
 129
 130     # Check for a mozharness setup
 131     config = context.mozharness_config
 132     if config and 'binary_path' in config:
 133         return config['binary_path']
 134     elif config:
 135         search_paths.append(os.path.join(context.mozharness_workdir, 'application'))
 136
 137     # Check for test-stage setup
 138     dist_bin = os.path.join(os.path.dirname(context.package_root), 'bin')
 139     if os.path.isdir(dist_bin):
 140         search_paths.append(dist_bin)
 141
 142     for path in search_paths:
 143         try:
 144             return mozinstall.get_binary(path, 'firefox')
 145         except mozinstall.InvalidBinary:
 146             continue
 147
 148
 149 def find_hostutils(context):
 150     workdir = context.mozharness_workdir
 151     hostutils = os.path.join(workdir, 'hostutils')
 152     for fname in os.listdir(hostutils):
 153         fpath = os.path.join(hostutils, fname)
 154         if os.path.isdir(fpath) and fname.startswith('host-utils'):
 155             return fpath
 156
 157
 158 def normalize_test_path(test_root, path):
 159     if os.path.isabs(path) or os.path.exists(path):
 160         return os.path.normpath(os.path.abspath(path))
 161
 162     for parent in ancestors(test_root):
 163         test_path = os.path.join(parent, path)
 164         if os.path.exists(test_path):
 165             return os.path.normpath(os.path.abspath(test_path))
 166     # Not a valid path? Return as is and let test harness deal with it
 167     return path
 168
 169
 170 def bootstrap(test_package_root):
 171     test_package_root = os.path.abspath(test_package_root)
 172
 173     # Ensure we are running Python 2.7+. We put this check here so we generate a
 174     # user-friendly error message rather than a cryptic stack trace on module
 175     # import.
 176     if sys.version_info[0] != 2 or sys.version_info[1] < 7:
 177         print('Python 2.7 or above (but not Python 3) is required to run mach.')
 178         print('You are running Python', platform.python_version())
 179         sys.exit(1)
 180
 181     sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
 182     import mach.main
 183
 184     def populate_context(context, key=None):
 185         if key is None:
 186             context.package_root = test_package_root
 187             context.bin_dir = os.path.join(test_package_root, 'bin')
 188             context.certs_dir = os.path.join(test_package_root, 'certs')
 189             context.module_dir = os.path.join(test_package_root, 'modules')
 190             context.ancestors = ancestors
 191             context.normalize_test_path = normalize_test_path
 192             return
 193
 194         # The values for the following 'key's will be set lazily, and cached
 195         # after first being invoked.
 196         if key == 'firefox_bin':
 197             return find_firefox(context)
 198
 199         if key == 'hostutils':
 200             return find_hostutils(context)
 201
 202         if key == 'mozharness_config':
 203             for dir_path in ancestors(context.package_root):
 204                 mozharness_config = os.path.join(dir_path, 'logs', 'localconfig.json')
 205                 if os.path.isfile(mozharness_config):
 206                     with open(mozharness_config, 'rb') as f:
 207                         return json.load(f)
 208             return {}
 209
 210         if key == 'mozharness_workdir':
 211             config = context.mozharness_config
 212             if config:
 213                 return os.path.join(config['base_work_dir'], config['work_dir'])
 214
 215         if key == 'activate_mozharness_venv':
 216             return types.MethodType(activate_mozharness_venv, context)
 217
 218     mach = mach.main.Mach(os.getcwd())
 219     mach.populate_context_handler = populate_context
 220
 221     for category, meta in CATEGORIES.items():
 222         mach.define_category(category, meta['short'], meta['long'],
 223                              meta['priority'])
 224
 225     for path in MACH_MODULES:
 226         cmdfile = os.path.join(test_package_root, path)
 227
 228         # Depending on which test zips were extracted,
 229         # the command module might not exist
 230         if os.path.isfile(cmdfile):
 231             mach.load_commands_from_file(cmdfile)
 232
 233     return mach