testing/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import print_function, unicode_literals
   6
   7 import os
   8 import sys
   9
  10 from mach.decorators import (
  11     CommandArgument,
  12     CommandProvider,
  13     Command,
  14 )
  15
  16 from mozbuild.base import MachCommandBase
  17
  18
  19 UNKNOWN_TEST = '''
  20 I was unable to find tests in the argument(s) given.
  21
  22 You need to specify a test directory, filename, test suite name, or
  23 abbreviation.
  24
  25 It's possible my little brain doesn't know about the type of test you are
  26 trying to execute. If you suspect this, please request support by filing
  27 a bug at
  28 https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=General.
  29 '''.strip()
  30
  31 UNKNOWN_FLAVOR = '''
  32 I know you are trying to run a %s test. Unfortunately, I can't run those
  33 tests yet. Sorry!
  34 '''.strip()
  35
  36 MOCHITEST_CHUNK_BY_DIR = 4
  37 MOCHITEST_TOTAL_CHUNKS = 5
  38
  39 TEST_SUITES = {
  40     'cppunittest': {
  41         'aliases': ('Cpp', 'cpp'),
  42         'mach_command': 'cppunittest',
  43         'kwargs': {'test_file': None},
  44     },
  45     'crashtest': {
  46         'aliases': ('C', 'Rc', 'RC', 'rc'),
  47         'mach_command': 'crashtest',
  48         'kwargs': {'test_file': None},
  49     },
  50     'crashtest-ipc': {
  51         'aliases': ('Cipc', 'cipc'),
  52         'mach_command': 'crashtest-ipc',
  53         'kwargs': {'test_file': None},
  54     },
  55     'jetpack': {
  56         'aliases': ('J',),
  57         'mach_command': 'jetpack-test',
  58         'kwargs': {},
  59     },
  60     'check-spidermonkey': {
  61         'aliases': ('Sm', 'sm'),
  62         'mach_command': 'check-spidermonkey',
  63         'kwargs': {'valgrind': False},
  64     },
  65     'mochitest-a11y': {
  66         'mach_command': 'mochitest',
  67         'kwargs': {'flavor': 'a11y', 'test_paths': None},
  68     },
  69     'mochitest-browser': {
  70         'aliases': ('bc', 'BC', 'Bc'),
  71         'mach_command': 'mochitest-browser',
  72         'kwargs': {'flavor': 'browser-chrome', 'test_paths': None},
  73     },
  74     'mochitest-chrome': {
  75         'mach_command': 'mochitest',
  76         'kwargs': {'flavor': 'chrome', 'test_paths': None},
  77     },
  78     'mochitest-devtools': {
  79         'aliases': ('dt', 'DT', 'Dt'),
  80         'mach_command': 'mochitest-browser',
  81         'kwargs': {'subsuite': 'devtools', 'test_paths': None},
  82     },
  83     'mochitest-ipcplugins': {
  84         'make_target': 'mochitest-ipcplugins',
  85     },
  86     'mochitest-plain': {
  87         'mach_command': 'mochitest',
  88         'kwargs': {'flavor': 'mochitest', 'test_paths': None},
  89     },
  90     'reftest': {
  91         'aliases': ('RR', 'rr', 'Rr'),
  92         'mach_command': 'reftest',
  93         'kwargs': {'test_file': None},
  94     },
  95     'reftest-ipc': {
  96         'aliases': ('Ripc',),
  97         'mach_command': 'reftest-ipc',
  98         'kwargs': {'test_file': None},
  99     },
 100     'valgrind': {
 101         'aliases': ('V', 'v'),
 102         'mach_command': 'valgrind-test',
 103         'kwargs': {},
 104     },
 105     'xpcshell': {
 106         'aliases': ('X', 'x'),
 107         'mach_command': 'xpcshell-test',
 108         'kwargs': {'test_file': 'all'},
 109     },
 110 }
 111
 112 # Maps test flavors to metadata on how to run that test.
 113 TEST_FLAVORS = {
 114     'a11y': {
 115         'mach_command': 'mochitest',
 116         'kwargs': {'flavor': 'a11y', 'test_paths': []},
 117     },
 118     'browser-chrome': {
 119         'mach_command': 'mochitest',
 120         'kwargs': {'flavor': 'browser-chrome', 'test_paths': []},
 121     },
 122     'chrashtest': { },
 123     'chrome': {
 124         'mach_command': 'mochitest',
 125         'kwargs': {'flavor': 'chrome', 'test_paths': []},
 126     },
 127     'mochitest': {
 128         'mach_command': 'mochitest',
 129         'kwargs': {'flavor': 'mochitest', 'test_paths': []},
 130     },
 131     'reftest': { },
 132     'steeplechase': { },
 133     'webapprt-chrome': {
 134         'mach_command': 'mochitest',
 135         'kwargs': {'flavor': 'webapprt-chrome', 'test_paths': []},
 136     },
 137     'xpcshell': {
 138         'mach_command': 'xpcshell-test',
 139         'kwargs': {'test_paths': []},
 140     },
 141 }
 142
 143
 144 for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
 145     TEST_SUITES['mochitest-%d' %i] = {
 146         'aliases': ('M%d' % i, 'm%d' % i),
 147         'mach_command': 'mochitest',
 148         'kwargs': {
 149             'flavor': 'mochitest',
 150             'chunk_by_dir': MOCHITEST_CHUNK_BY_DIR,
 151             'total_chunks': MOCHITEST_TOTAL_CHUNKS,
 152             'this_chunk': i,
 153             'test_paths': None,
 154         },
 155     }
 156
 157 TEST_HELP = '''
 158 Test or tests to run. Tests can be specified by filename, directory, suite
 159 name or suite alias.
 160
 161 The following test suites and aliases are supported: %s
 162 ''' % ', '.join(sorted(TEST_SUITES))
 163 TEST_HELP = TEST_HELP.strip()
 164
 165
 166 @CommandProvider
 167 class Test(MachCommandBase):
 168     @Command('test', category='testing', description='Run tests (detects the kind of test and runs it).')
 169     @CommandArgument('what', default=None, nargs='*', help=TEST_HELP)
 170     def test(self, what):
 171         from mozbuild.testing import TestResolver
 172
 173         # Parse arguments and assemble a test "plan."
 174         run_suites = set()
 175         run_tests = []
 176         resolver = self._spawn(TestResolver)
 177
 178         for entry in what:
 179             # If the path matches the name or alias of an entire suite, run
 180             # the entire suite.
 181             if entry in TEST_SUITES:
 182                 run_suites.add(entry)
 183                 continue
 184             suitefound = False
 185             for suite, v in TEST_SUITES.items():
 186                 if entry in v.get('aliases', []):
 187                     run_suites.add(suite)
 188                     suitefound = True
 189             if suitefound:
 190                 continue
 191
 192             # Now look for file/directory matches in the TestResolver.
 193             relpath = self._wrap_path_argument(entry).relpath()
 194             tests = list(resolver.resolve_tests(paths=[relpath]))
 195             run_tests.extend(tests)
 196
 197             if not tests:
 198                 print('UNKNOWN TEST: %s' % entry, file=sys.stderr)
 199
 200         if not run_suites and not run_tests:
 201             print(UNKNOWN_TEST)
 202             return 1
 203
 204         status = None
 205         for suite_name in run_suites:
 206             suite = TEST_SUITES[suite_name]
 207
 208             if 'mach_command' in suite:
 209                 res = self._mach_context.commands.dispatch(
 210                     suite['mach_command'], self._mach_context,
 211                     **suite['kwargs'])
 212                 if res:
 213                     status = res
 214
 215             elif 'make_target' in suite:
 216                 res = self._run_make(target=suite['make_target'],
 217                     pass_thru=True)
 218                 if res:
 219                     status = res
 220
 221         flavors = {}
 222         for test in run_tests:
 223             flavors.setdefault(test['flavor'], []).append(test)
 224
 225         for flavor, tests in sorted(flavors.items()):
 226             if flavor not in TEST_FLAVORS:
 227                 print(UNKNOWN_FLAVOR % flavor)
 228                 status = 1
 229                 continue
 230
 231             m = TEST_FLAVORS[flavor]
 232             if 'mach_command' not in m:
 233                 print(UNKNOWN_FLAVOR % flavor)
 234                 status = 1
 235                 continue
 236
 237             res = self._mach_context.commands.dispatch(
 238                     m['mach_command'], self._mach_context,
 239                     test_objects=tests, **m['kwargs'])
 240             if res:
 241                 status = res
 242
 243         return status
 244
 245
 246 @CommandProvider
 247 class MachCommands(MachCommandBase):
 248     @Command('cppunittest', category='testing',
 249         description='Run cpp unit tests (C++ tests).')
 250     @CommandArgument('test_files', nargs='*', metavar='N',
 251         help='Test to run. Can be specified as one or more files or ' \
 252             'directories, or omitted. If omitted, the entire test suite is ' \
 253             'executed.')
 254
 255     def run_cppunit_test(self, **params):
 256         import mozinfo
 257         from mozlog.structured import commandline
 258         import runcppunittests as cppunittests
 259
 260         log = commandline.setup_logging("cppunittest",
 261                                         {},
 262                                         {"tbpl": sys.stdout})
 263
 264         if len(params['test_files']) == 0:
 265             testdir = os.path.join(self.distdir, 'cppunittests')
 266             tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
 267         else:
 268             tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
 269
 270         # See if we have crash symbols
 271         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
 272         if not os.path.isdir(symbols_path):
 273             symbols_path = None
 274
 275         tester = cppunittests.CPPUnitTests()
 276         try:
 277             result = tester.run_tests(tests, self.bindir, symbols_path, interactive=True)
 278         except Exception as e:
 279             log.error("Caught exception running cpp unit tests: %s" % str(e))
 280             result = False
 281
 282         return 0 if result else 1
 283
 284 @CommandProvider
 285 class CheckSpiderMonkeyCommand(MachCommandBase):
 286     @Command('check-spidermonkey', category='testing', description='Run SpiderMonkey tests (JavaScript engine).')
 287     @CommandArgument('--valgrind', action='store_true', help='Run jit-test suite with valgrind flag')
 288
 289     def run_checkspidermonkey(self, **params):
 290         import subprocess
 291         import sys
 292
 293         bin_suffix = ''
 294         if sys.platform.startswith('win'):
 295             bin_suffix = '.exe'
 296
 297         js = os.path.join(self.bindir, 'js%s' % bin_suffix)
 298
 299         print('Running jit-tests')
 300         jittest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'jit-test', 'jit_test.py'),
 301               js, '--no-slow', '--tbpl']
 302         if params['valgrind']:
 303             jittest_cmd.append('--valgrind')
 304
 305         jittest_result = subprocess.call(jittest_cmd)
 306
 307         print('running jstests')
 308         jstest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'tests', 'jstests.py'),
 309               js, '--tbpl']
 310         jstest_result = subprocess.call(jstest_cmd)
 311
 312         print('running jsapi-tests')
 313         jsapi_tests_cmd = [os.path.join(self.bindir, 'jsapi-tests%s' % bin_suffix)]
 314         jsapi_tests_result = subprocess.call(jsapi_tests_cmd)
 315
 316         print('running check-style')
 317         check_style_cmd = [sys.executable, os.path.join(self.topsrcdir, 'config', 'check_spidermonkey_style.py')]
 318         check_style_result = subprocess.call(check_style_cmd, cwd=os.path.join(self.topsrcdir, 'js', 'src'))
 319
 320         all_passed = jittest_result and jstest_result and jsapi_tests_result and check_style_result
 321
 322         return all_passed