testing/xpcshell/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the xpcshell test runner with mach.
   6
   7 from __future__ import unicode_literals, print_function
   8
   9 import argparse
  10 import os
  11 import shutil
  12 import sys
  13 import urllib2
  14
  15 from mozlog import structured
  16
  17 from mozbuild.base import (
  18     MachCommandBase,
  19     MozbuildObject,
  20     MachCommandConditions as conditions,
  21 )
  22
  23 from mach.decorators import (
  24     CommandArgument,
  25     CommandProvider,
  26     Command,
  27 )
  28
  29 _parser = argparse.ArgumentParser()
  30 structured.commandline.add_logging_group(_parser)
  31
  32 ADB_NOT_FOUND = '''
  33 The %s command requires the adb binary to be on your path.
  34
  35 If you have a B2G build, this can be found in
  36 '%s/out/host/<platform>/bin'.
  37 '''.lstrip()
  38
  39 BUSYBOX_URLS = {
  40     'arm': 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l',
  41     'x86': 'http://www.busybox.net/downloads/binaries/latest/busybox-i686'
  42 }
  43
  44
  45 if sys.version_info[0] < 3:
  46     unicode_type = unicode
  47 else:
  48     unicode_type = str
  49
  50 # This should probably be consolidated with similar classes in other test
  51 # runners.
  52 class InvalidTestPathError(Exception):
  53     """Exception raised when the test path is not valid."""
  54
  55
  56 class XPCShellRunner(MozbuildObject):
  57     """Run xpcshell tests."""
  58     def run_suite(self, **kwargs):
  59         from manifestparser import TestManifest
  60         manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
  61             '_tests', 'xpcshell', 'xpcshell.ini')])
  62
  63         return self._run_xpcshell_harness(manifest=manifest, **kwargs)
  64
  65     def run_test(self, test_paths, interactive=False,
  66                  keep_going=False, sequential=False, shuffle=False,
  67                  debugger=None, debuggerArgs=None, debuggerInteractive=None,
  68                  jsDebugger=False, jsDebuggerPort=None,
  69                  rerun_failures=False, test_objects=None, verbose=False,
  70                  log=None,
  71                  # ignore parameters from other platforms' options
  72                  **kwargs):
  73         """Runs an individual xpcshell test."""
  74         from mozbuild.testing import TestResolver
  75         from manifestparser import TestManifest
  76
  77         # TODO Bug 794506 remove once mach integrates with virtualenv.
  78         build_path = os.path.join(self.topobjdir, 'build')
  79         if build_path not in sys.path:
  80             sys.path.append(build_path)
  81
  82         if test_paths == ['all']:
  83             self.run_suite(interactive=interactive,
  84                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,
  85                            debugger=debugger, debuggerArgs=debuggerArgs,
  86                            debuggerInteractive=debuggerInteractive,
  87                            jsDebugger=jsDebugger, jsDebuggerPort=jsDebuggerPort,
  88                            rerun_failures=rerun_failures,
  89                            verbose=verbose, log=log)
  90             return
  91         elif test_paths:
  92             test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
  93
  94         if test_objects:
  95             tests = test_objects
  96         else:
  97             resolver = self._spawn(TestResolver)
  98             tests = list(resolver.resolve_tests(paths=test_paths,
  99                 flavor='xpcshell'))
 100
 101         if not tests:
 102             raise InvalidTestPathError('We could not find an xpcshell test '
 103                 'for the passed test path. Please select a path that is '
 104                 'a test file or is a directory containing xpcshell tests.')
 105
 106         # Dynamically write out a manifest holding all the discovered tests.
 107         manifest = TestManifest()
 108         manifest.tests.extend(tests)
 109
 110         args = {
 111             'interactive': interactive,
 112             'keep_going': keep_going,
 113             'shuffle': shuffle,
 114             'sequential': sequential,
 115             'debugger': debugger,
 116             'debuggerArgs': debuggerArgs,
 117             'debuggerInteractive': debuggerInteractive,
 118             'jsDebugger': jsDebugger,
 119             'jsDebuggerPort': jsDebuggerPort,
 120             'rerun_failures': rerun_failures,
 121             'manifest': manifest,
 122             'verbose': verbose,
 123             'log': log,
 124         }
 125
 126         return self._run_xpcshell_harness(**args)
 127
 128     def _run_xpcshell_harness(self, manifest,
 129                               test_path=None, shuffle=False, interactive=False,
 130                               keep_going=False, sequential=False,
 131                               debugger=None, debuggerArgs=None, debuggerInteractive=None,
 132                               jsDebugger=False, jsDebuggerPort=None,
 133                               rerun_failures=False, verbose=False, log=None):
 134
 135         # Obtain a reference to the xpcshell test runner.
 136         import runxpcshelltests
 137
 138         xpcshell = runxpcshelltests.XPCShellTests(log=log)
 139         self.log_manager.enable_unstructured()
 140
 141         tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
 142         modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
 143         # We want output from the test to be written immediately if we are only
 144         # running a single test.
 145         single_test = (test_path is not None or
 146                        (manifest and len(manifest.test_paths())==1))
 147         sequential = sequential or single_test
 148
 149         args = {
 150             'manifest': manifest,
 151             'xpcshell': self.get_binary_path('xpcshell'),
 152             'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
 153             'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
 154             'interactive': interactive,
 155             'keepGoing': keep_going,
 156             'logfiles': False,
 157             'sequential': sequential,
 158             'shuffle': shuffle,
 159             'testsRootDir': tests_dir,
 160             'testingModulesDir': modules_dir,
 161             'profileName': 'firefox',
 162             'verbose': verbose or single_test,
 163             'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
 164             'xunitName': 'xpcshell',
 165             'pluginsPath': os.path.join(self.distdir, 'plugins'),
 166             'debugger': debugger,
 167             'debuggerArgs': debuggerArgs,
 168             'debuggerInteractive': debuggerInteractive,
 169             'jsDebugger': jsDebugger,
 170             'jsDebuggerPort': jsDebuggerPort,
 171         }
 172
 173         if test_path is not None:
 174             args['testPath'] = test_path
 175
 176         # A failure manifest is written by default. If --rerun-failures is
 177         # specified and a prior failure manifest is found, the prior manifest
 178         # will be run. A new failure manifest is always written over any
 179         # prior failure manifest.
 180         failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
 181         rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
 182         if os.path.exists(failure_manifest_path) and rerun_failures:
 183             shutil.move(failure_manifest_path, rerun_manifest_path)
 184             args['manifest'] = rerun_manifest_path
 185         elif os.path.exists(failure_manifest_path):
 186             os.remove(failure_manifest_path)
 187         elif rerun_failures:
 188             print("No failures were found to re-run.")
 189             return 0
 190         args['failureManifest'] = failure_manifest_path
 191
 192         # Python through 2.7.2 has issues with unicode in some of the
 193         # arguments. Work around that.
 194         filtered_args = {}
 195         for k, v in args.items():
 196             if isinstance(v, unicode_type):
 197                 v = v.encode('utf-8')
 198
 199             if isinstance(k, unicode_type):
 200                 k = k.encode('utf-8')
 201
 202             filtered_args[k] = v
 203
 204         result = xpcshell.runTests(**filtered_args)
 205
 206         self.log_manager.disable_unstructured()
 207
 208         if not result and not xpcshell.sequential:
 209             print("Tests were run in parallel. Try running with --sequential "
 210                   "to make sure the failures were not caused by this.")
 211         return int(not result)
 212
 213 class AndroidXPCShellRunner(MozbuildObject):
 214     """Get specified DeviceManager"""
 215     def get_devicemanager(self, devicemanager, ip, port, remote_test_root):
 216         from mozdevice import devicemanagerADB, devicemanagerSUT
 217         dm = None
 218         if devicemanager == "adb":
 219             if ip:
 220                 dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root)
 221             else:
 222                 dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root)
 223         else:
 224             if ip:
 225                 dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root)
 226             else:
 227                 raise Exception("You must provide a device IP to connect to via the --ip option")
 228         return dm
 229
 230     """Run Android xpcshell tests."""
 231     def run_test(self,
 232                  test_paths, keep_going,
 233                  devicemanager, ip, port, remote_test_root, no_setup, local_apk,
 234                  test_objects=None, log=None,
 235                  # ignore parameters from other platforms' options
 236                  **kwargs):
 237         # TODO Bug 794506 remove once mach integrates with virtualenv.
 238         build_path = os.path.join(self.topobjdir, 'build')
 239         if build_path not in sys.path:
 240             sys.path.append(build_path)
 241
 242         import remotexpcshelltests
 243
 244         dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
 245
 246         options = remotexpcshelltests.RemoteXPCShellOptions()
 247         options.shuffle = False
 248         options.sequential = True
 249         options.interactive = False
 250         options.debugger = None
 251         options.debuggerArgs = None
 252         options.setup = not no_setup
 253         options.keepGoing = keep_going
 254         options.objdir = self.topobjdir
 255         options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
 256         options.localBin = os.path.join(self.topobjdir, 'dist/bin')
 257         options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
 258         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
 259         options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
 260         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
 261         if local_apk:
 262             options.localAPK = local_apk
 263         else:
 264             for file in os.listdir(os.path.join(options.objdir, "dist")):
 265                 if file.endswith(".apk") and file.startswith("fennec"):
 266                     options.localAPK = os.path.join(options.objdir, "dist")
 267                     options.localAPK = os.path.join(options.localAPK, file)
 268                     print ("using APK: " + options.localAPK)
 269                     break
 270             else:
 271                 raise Exception("You must specify an APK")
 272
 273         if test_paths == ['all']:
 274             testdirs = []
 275             options.testPath = None
 276             options.verbose = False
 277         elif test_objects:
 278             if len(test_objects) > 1:
 279                 print('Warning: only the first test will be used.')
 280             testdirs = test_objects[0]['dir_relpath']
 281             options.testPath = test_objects[0]['path']
 282             options.verbose = True
 283         else:
 284             if len(test_paths) > 1:
 285                 print('Warning: only the first test path argument will be used.')
 286             testdirs = test_paths[0]
 287             options.testPath = test_paths[0]
 288             options.verbose = True
 289
 290         xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, testdirs, log)
 291
 292         result = xpcshell.runTests(xpcshell='xpcshell',
 293                       testClass=remotexpcshelltests.RemoteXPCShellTestThread,
 294                       testdirs=testdirs,
 295                       mobileArgs=xpcshell.mobileArgs,
 296                       **options.__dict__)
 297
 298
 299         return int(not result)
 300
 301 class B2GXPCShellRunner(MozbuildObject):
 302     def __init__(self, *args, **kwargs):
 303         MozbuildObject.__init__(self, *args, **kwargs)
 304
 305         # TODO Bug 794506 remove once mach integrates with virtualenv.
 306         build_path = os.path.join(self.topobjdir, 'build')
 307         if build_path not in sys.path:
 308             sys.path.append(build_path)
 309
 310         build_path = os.path.join(self.topsrcdir, 'build')
 311         if build_path not in sys.path:
 312             sys.path.append(build_path)
 313
 314         self.tests_dir = os.path.join(self.topobjdir, '_tests')
 315         self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell')
 316         self.bin_dir = os.path.join(self.distdir, 'bin')
 317
 318     def _download_busybox(self, b2g_home, emulator):
 319         target_device = 'generic'
 320         if emulator == 'x86':
 321             target_device = 'generic_x86'
 322         system_bin = os.path.join(b2g_home, 'out', 'target', 'product', target_device, 'system', 'bin')
 323         busybox_path = os.path.join(system_bin, 'busybox')
 324
 325         if os.path.isfile(busybox_path):
 326             return busybox_path
 327
 328         if not os.path.isdir(system_bin):
 329             os.makedirs(system_bin)
 330
 331         try:
 332             data = urllib2.urlopen(BUSYBOX_URLS[emulator])
 333         except urllib2.URLError:
 334             print('There was a problem downloading busybox. Proceeding without it,' \
 335                   'initial setup will be slow.')
 336             return
 337
 338         with open(busybox_path, 'wb') as f:
 339             f.write(data.read())
 340         return busybox_path
 341
 342     def run_test(self, test_paths, b2g_home=None, busybox=None, device_name=None,
 343                  test_objects=None, log=None,
 344                  # ignore parameters from other platforms' options
 345                  **kwargs):
 346         try:
 347             import which
 348             which.which('adb')
 349         except which.WhichError:
 350             # TODO Find adb automatically if it isn't on the path
 351             print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
 352             sys.exit(1)
 353
 354         test_path = None
 355         if test_objects:
 356             if len(test_objects) > 1:
 357                 print('Warning: Only the first test will be used.')
 358
 359             test_path = self._wrap_path_argument(test_objects[0]['path'])
 360         elif test_paths:
 361             if len(test_paths) > 1:
 362                 print('Warning: Only the first test path will be used.')
 363
 364             test_path = self._wrap_path_argument(test_paths[0]).relpath()
 365
 366         import runtestsb2g
 367         parser = runtestsb2g.B2GOptions()
 368         options, args = parser.parse_args([])
 369
 370         options.b2g_path = b2g_home
 371         options.busybox = busybox or os.environ.get('BUSYBOX')
 372         options.localLib = self.bin_dir
 373         options.localBin = self.bin_dir
 374         options.logdir = self.xpcshell_dir
 375         options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
 376         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
 377         options.objdir = self.topobjdir
 378         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
 379         options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
 380         options.testsRootDir = self.xpcshell_dir
 381         options.testPath = test_path
 382         options.use_device_libs = True
 383
 384         options.emulator = 'arm'
 385         if device_name.startswith('emulator'):
 386             if 'x86' in device_name:
 387                 options.emulator = 'x86'
 388
 389         if not options.busybox:
 390             options.busybox = self._download_busybox(b2g_home, options.emulator)
 391
 392         return runtestsb2g.run_remote_xpcshell(parser, options, args, log)
 393
 394 def is_platform_supported(cls):
 395     """Must have a Firefox, Android or B2G build."""
 396     return conditions.is_android(cls) or \
 397            conditions.is_b2g(cls) or \
 398            conditions.is_firefox(cls)
 399
 400 @CommandProvider
 401 class MachCommands(MachCommandBase):
 402     def __init__(self, context):
 403         MachCommandBase.__init__(self, context)
 404
 405         for attr in ('b2g_home', 'device_name'):
 406             setattr(self, attr, getattr(context, attr, None))
 407
 408     @Command('xpcshell-test', category='testing',
 409         conditions=[is_platform_supported],
 410         description='Run XPCOM Shell tests (API direct unit testing)',
 411         parser=_parser)
 412     @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
 413         help='Test to run. Can be specified as a single JS file, a directory, '
 414              'or omitted. If omitted, the entire test suite is executed.')
 415     @CommandArgument('--verbose', '-v', action='store_true',
 416         help='Provide full output from each test process.')
 417     @CommandArgument("--debugger", default=None, metavar='DEBUGGER',
 418                      help = "Run xpcshell under the given debugger.")
 419     @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
 420                      dest = "debuggerArgs",
 421                      help = "pass the given args to the debugger _before_ "
 422                             "the application on the command line")
 423     @CommandArgument("--debugger-interactive", action = "store_true",
 424                      dest = "debuggerInteractive",
 425                      help = "prevents the test harness from redirecting "
 426                             "stdout and stderr for interactive debuggers")
 427     @CommandArgument("--jsdebugger", dest="jsDebugger", action="store_true",
 428                      help="Waits for a devtools JS debugger to connect before "
 429                           "starting the test.")
 430     @CommandArgument("--jsdebugger-port", dest="jsDebuggerPort",
 431                      type=int, default=6000,
 432                      help="The port to listen on for a debugger connection if "
 433                           "--jsdebugger is specified (default=6000).")
 434     @CommandArgument('--interactive', '-i', action='store_true',
 435         help='Open an xpcshell prompt before running tests.')
 436     @CommandArgument('--keep-going', '-k', action='store_true',
 437         help='Continue running tests after a SIGINT is received.')
 438     @CommandArgument('--sequential', action='store_true',
 439         help='Run the tests sequentially.')
 440     @CommandArgument('--shuffle', '-s', action='store_true',
 441         help='Randomize the execution order of tests.')
 442     @CommandArgument('--rerun-failures', action='store_true',
 443         help='Reruns failures from last time.')
 444     @CommandArgument('--devicemanager', default='adb', type=str,
 445         help='(Android) Type of devicemanager to use for communication: adb or sut')
 446     @CommandArgument('--ip', type=str, default=None,
 447         help='(Android) IP address of device')
 448     @CommandArgument('--port', type=int, default=20701,
 449         help='(Android) Port of device')
 450     @CommandArgument('--remote_test_root', type=str, default=None,
 451         help='(Android) Remote test root such as /mnt/sdcard or /data/local')
 452     @CommandArgument('--no-setup', action='store_true',
 453         help='(Android) Do not copy files to device')
 454     @CommandArgument('--local-apk', type=str, default=None,
 455         help='(Android) Use specified Fennec APK')
 456     @CommandArgument('--busybox', type=str, default=None,
 457         help='(B2G) Path to busybox binary (speeds up installation of tests).')
 458     def run_xpcshell_test(self, **params):
 459         from mozbuild.controller.building import BuildDriver
 460
 461         # We should probably have a utility function to ensure the tree is
 462         # ready to run tests. Until then, we just create the state dir (in
 463         # case the tree wasn't built with mach).
 464         self._ensure_state_subdir_exists('.')
 465
 466         driver = self._spawn(BuildDriver)
 467         driver.install_tests(remove=False)
 468
 469         structured.commandline.formatter_option_defaults['verbose'] = True
 470         params['log'] = structured.commandline.setup_logging("XPCShellTests",
 471                                                              params,
 472                                                              {"mach": sys.stdout})
 473
 474         if conditions.is_android(self):
 475             xpcshell = self._spawn(AndroidXPCShellRunner)
 476         elif conditions.is_b2g(self):
 477             xpcshell = self._spawn(B2GXPCShellRunner)
 478             params['b2g_home'] = self.b2g_home
 479             params['device_name'] = self.device_name
 480         else:
 481             xpcshell = self._spawn(XPCShellRunner)
 482         xpcshell.cwd = self._mach_context.cwd
 483
 484         try:
 485             return xpcshell.run_test(**params)
 486         except InvalidTestPathError as e:
 487             print(e.message)
 488             return 1