testing/xpcshell/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the xpcshell test runner with mach.
   6
   7 from __future__ import unicode_literals, print_function
   8
   9 import mozpack.path
  10 import logging
  11 import os
  12 import shutil
  13 import sys
  14 import urllib2
  15
  16 from StringIO import StringIO
  17
  18 from mozbuild.base import (
  19     MachCommandBase,
  20     MozbuildObject,
  21     MachCommandConditions as conditions,
  22 )
  23
  24 from mach.decorators import (
  25     CommandArgument,
  26     CommandProvider,
  27     Command,
  28 )
  29
  30 ADB_NOT_FOUND = '''
  31 The %s command requires the adb binary to be on your path.
  32
  33 If you have a B2G build, this can be found in
  34 '%s/out/host/<platform>/bin'.
  35 '''.lstrip()
  36
  37 BUSYBOX_URLS = {
  38     'arm': 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l',
  39     'x86': 'http://www.busybox.net/downloads/binaries/latest/busybox-i686'
  40 }
  41
  42
  43 if sys.version_info[0] < 3:
  44     unicode_type = unicode
  45 else:
  46     unicode_type = str
  47
  48 # Simple filter to omit the message emitted as a test file begins.
  49 class TestStartFilter(logging.Filter):
  50     def filter(self, record):
  51         return not record.params['msg'].endswith("running test ...")
  52
  53 # This should probably be consolidated with similar classes in other test
  54 # runners.
  55 class InvalidTestPathError(Exception):
  56     """Exception raised when the test path is not valid."""
  57
  58
  59 class XPCShellRunner(MozbuildObject):
  60     """Run xpcshell tests."""
  61     def run_suite(self, **kwargs):
  62         from manifestparser import TestManifest
  63         manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
  64             '_tests', 'xpcshell', 'xpcshell.ini')])
  65
  66         return self._run_xpcshell_harness(manifest=manifest, **kwargs)
  67
  68     def run_test(self, test_paths, interactive=False,
  69                  keep_going=False, sequential=False, shuffle=False,
  70                  debugger=None, debuggerArgs=None, debuggerInteractive=None,
  71                  rerun_failures=False, test_objects=None, verbose=False,
  72                  # ignore parameters from other platforms' options
  73                  **kwargs):
  74         """Runs an individual xpcshell test."""
  75         from mozbuild.testing import TestResolver
  76         from manifestparser import TestManifest
  77
  78         # TODO Bug 794506 remove once mach integrates with virtualenv.
  79         build_path = os.path.join(self.topobjdir, 'build')
  80         if build_path not in sys.path:
  81             sys.path.append(build_path)
  82
  83         if test_paths == ['all']:
  84             self.run_suite(interactive=interactive,
  85                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,
  86                            debugger=debugger, debuggerArgs=debuggerArgs,
  87                            debuggerInteractive=debuggerInteractive,
  88                            rerun_failures=rerun_failures,
  89                            verbose=verbose)
  90             return
  91         elif test_paths:
  92             test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
  93
  94         if test_objects:
  95             tests = test_objects
  96         else:
  97             resolver = self._spawn(TestResolver)
  98             tests = list(resolver.resolve_tests(paths=test_paths,
  99                 flavor='xpcshell'))
 100
 101         if not tests:
 102             raise InvalidTestPathError('We could not find an xpcshell test '
 103                 'for the passed test path. Please select a path that is '
 104                 'a test file or is a directory containing xpcshell tests.')
 105
 106         # Dynamically write out a manifest holding all the discovered tests.
 107         manifest = TestManifest()
 108         manifest.tests.extend(tests)
 109
 110         args = {
 111             'interactive': interactive,
 112             'keep_going': keep_going,
 113             'shuffle': shuffle,
 114             'sequential': sequential,
 115             'debugger': debugger,
 116             'debuggerArgs': debuggerArgs,
 117             'debuggerInteractive': debuggerInteractive,
 118             'rerun_failures': rerun_failures,
 119             'manifest': manifest,
 120             'verbose': verbose,
 121         }
 122
 123         return self._run_xpcshell_harness(**args)
 124
 125     def _run_xpcshell_harness(self, manifest,
 126                               test_path=None, shuffle=False, interactive=False,
 127                               keep_going=False, sequential=False,
 128                               debugger=None, debuggerArgs=None, debuggerInteractive=None,
 129                               rerun_failures=False, verbose=False):
 130
 131         # Obtain a reference to the xpcshell test runner.
 132         import runxpcshelltests
 133
 134         dummy_log = StringIO()
 135         xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
 136         self.log_manager.enable_unstructured()
 137
 138         xpcshell_filter = TestStartFilter()
 139         self.log_manager.terminal_handler.addFilter(xpcshell_filter)
 140
 141         tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
 142         modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
 143         # We want output from the test to be written immediately if we are only
 144         # running a single test.
 145         verbose_output = (test_path is not None or
 146                           (manifest and len(manifest.test_paths())==1) or
 147                           verbose)
 148
 149         args = {
 150             'manifest': manifest,
 151             'xpcshell': self.get_binary_path('xpcshell'),
 152             'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
 153             'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
 154             'interactive': interactive,
 155             'keepGoing': keep_going,
 156             'logfiles': False,
 157             'sequential': sequential,
 158             'shuffle': shuffle,
 159             'testsRootDir': tests_dir,
 160             'testingModulesDir': modules_dir,
 161             'profileName': 'firefox',
 162             'verbose': test_path is not None,
 163             'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
 164             'xunitName': 'xpcshell',
 165             'pluginsPath': os.path.join(self.distdir, 'plugins'),
 166             'debugger': debugger,
 167             'debuggerArgs': debuggerArgs,
 168             'debuggerInteractive': debuggerInteractive,
 169             'on_message': (lambda obj, msg: xpcshell.log.info(msg.decode('utf-8', 'replace'))) \
 170                             if verbose_output else None,
 171         }
 172
 173         if test_path is not None:
 174             args['testPath'] = test_path
 175
 176         # A failure manifest is written by default. If --rerun-failures is
 177         # specified and a prior failure manifest is found, the prior manifest
 178         # will be run. A new failure manifest is always written over any
 179         # prior failure manifest.
 180         failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
 181         rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
 182         if os.path.exists(failure_manifest_path) and rerun_failures:
 183             shutil.move(failure_manifest_path, rerun_manifest_path)
 184             args['manifest'] = rerun_manifest_path
 185         elif os.path.exists(failure_manifest_path):
 186             os.remove(failure_manifest_path)
 187         elif rerun_failures:
 188             print("No failures were found to re-run.")
 189             return 0
 190         args['failureManifest'] = failure_manifest_path
 191
 192         # Python through 2.7.2 has issues with unicode in some of the
 193         # arguments. Work around that.
 194         filtered_args = {}
 195         for k, v in args.items():
 196             if isinstance(v, unicode_type):
 197                 v = v.encode('utf-8')
 198
 199             if isinstance(k, unicode_type):
 200                 k = k.encode('utf-8')
 201
 202             filtered_args[k] = v
 203
 204         result = xpcshell.runTests(**filtered_args)
 205
 206         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
 207         self.log_manager.disable_unstructured()
 208
 209         if not result and not xpcshell.sequential:
 210             print("Tests were run in parallel. Try running with --sequential "
 211                   "to make sure the failures were not caused by this.")
 212         return int(not result)
 213
 214 class AndroidXPCShellRunner(MozbuildObject):
 215     """Get specified DeviceManager"""
 216     def get_devicemanager(self, devicemanager, ip, port, remote_test_root):
 217         from mozdevice import devicemanagerADB, devicemanagerSUT
 218         dm = None
 219         if devicemanager == "adb":
 220             if ip:
 221                 dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root)
 222             else:
 223                 dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root)
 224         else:
 225             if ip:
 226                 dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root)
 227             else:
 228                 raise Exception("You must provide a device IP to connect to via the --ip option")
 229         return dm
 230
 231     """Run Android xpcshell tests."""
 232     def run_test(self,
 233                  test_paths, keep_going,
 234                  devicemanager, ip, port, remote_test_root, no_setup, local_apk,
 235                  test_objects=None,
 236                  # ignore parameters from other platforms' options
 237                  **kwargs):
 238         # TODO Bug 794506 remove once mach integrates with virtualenv.
 239         build_path = os.path.join(self.topobjdir, 'build')
 240         if build_path not in sys.path:
 241             sys.path.append(build_path)
 242
 243         import remotexpcshelltests
 244
 245         dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
 246
 247         options = remotexpcshelltests.RemoteXPCShellOptions()
 248         options.shuffle = False
 249         options.sequential = True
 250         options.interactive = False
 251         options.debugger = None
 252         options.debuggerArgs = None
 253         options.setup = not no_setup
 254         options.keepGoing = keep_going
 255         options.objdir = self.topobjdir
 256         options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
 257         options.localBin = os.path.join(self.topobjdir, 'dist/bin')
 258         options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
 259         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
 260         options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
 261         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
 262         if local_apk:
 263             options.localAPK = local_apk
 264         else:
 265             for file in os.listdir(os.path.join(options.objdir, "dist")):
 266                 if file.endswith(".apk") and file.startswith("fennec"):
 267                     options.localAPK = os.path.join(options.objdir, "dist")
 268                     options.localAPK = os.path.join(options.localAPK, file)
 269                     print ("using APK: " + options.localAPK)
 270                     break
 271             else:
 272                 raise Exception("You must specify an APK")
 273
 274         if test_paths == ['all']:
 275             testdirs = []
 276             options.testPath = None
 277             options.verbose = False
 278         elif test_objects:
 279             if len(test_objects) > 1:
 280                 print('Warning: only the first test will be used.')
 281             testdirs = test_objects[0]['dir_relpath']
 282             options.testPath = test_objects[0]['path']
 283             options.verbose = True
 284         else:
 285             if len(test_paths) > 1:
 286                 print('Warning: only the first test path argument will be used.')
 287             testdirs = test_paths[0]
 288             options.testPath = test_paths[0]
 289             options.verbose = True
 290         dummy_log = StringIO()
 291         xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log)
 292         self.log_manager.enable_unstructured()
 293
 294         xpcshell_filter = TestStartFilter()
 295         self.log_manager.terminal_handler.addFilter(xpcshell_filter)
 296
 297         result = xpcshell.runTests(xpcshell='xpcshell',
 298                       testClass=remotexpcshelltests.RemoteXPCShellTestThread,
 299                       testdirs=testdirs,
 300                       mobileArgs=xpcshell.mobileArgs,
 301                       **options.__dict__)
 302
 303         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
 304         self.log_manager.disable_unstructured()
 305
 306         return int(not result)
 307
 308 class B2GXPCShellRunner(MozbuildObject):
 309     def __init__(self, *args, **kwargs):
 310         MozbuildObject.__init__(self, *args, **kwargs)
 311
 312         # TODO Bug 794506 remove once mach integrates with virtualenv.
 313         build_path = os.path.join(self.topobjdir, 'build')
 314         if build_path not in sys.path:
 315             sys.path.append(build_path)
 316
 317         build_path = os.path.join(self.topsrcdir, 'build')
 318         if build_path not in sys.path:
 319             sys.path.append(build_path)
 320
 321         self.tests_dir = os.path.join(self.topobjdir, '_tests')
 322         self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell')
 323         self.bin_dir = os.path.join(self.distdir, 'bin')
 324
 325     def _download_busybox(self, b2g_home, emulator):
 326         target_device = 'generic'
 327         if emulator == 'x86':
 328             target_device = 'generic_x86'
 329         system_bin = os.path.join(b2g_home, 'out', 'target', 'product', target_device, 'system', 'bin')
 330         busybox_path = os.path.join(system_bin, 'busybox')
 331
 332         if os.path.isfile(busybox_path):
 333             return busybox_path
 334
 335         if not os.path.isdir(system_bin):
 336             os.makedirs(system_bin)
 337
 338         try:
 339             data = urllib2.urlopen(BUSYBOX_URLS[emulator])
 340         except urllib2.URLError:
 341             print('There was a problem downloading busybox. Proceeding without it,' \
 342                   'initial setup will be slow.')
 343             return
 344
 345         with open(busybox_path, 'wb') as f:
 346             f.write(data.read())
 347         return busybox_path
 348
 349     def run_test(self, test_paths, b2g_home=None, busybox=None, device_name=None,
 350                  test_objects=None,
 351                  # ignore parameters from other platforms' options
 352                  **kwargs):
 353         try:
 354             import which
 355             which.which('adb')
 356         except which.WhichError:
 357             # TODO Find adb automatically if it isn't on the path
 358             print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
 359             sys.exit(1)
 360
 361         test_path = None
 362         if test_objects:
 363             if len(test_objects) > 1:
 364                 print('Warning: Only the first test will be used.')
 365
 366             test_path = self._wrap_path_argument(test_objects[0]['path'])
 367         elif test_paths:
 368             if len(test_paths) > 1:
 369                 print('Warning: Only the first test path will be used.')
 370
 371             test_path = self._wrap_path_argument(test_paths[0]).relpath()
 372
 373         import runtestsb2g
 374         parser = runtestsb2g.B2GOptions()
 375         options, args = parser.parse_args([])
 376
 377         options.b2g_path = b2g_home
 378         options.busybox = busybox or os.environ.get('BUSYBOX')
 379         options.localLib = self.bin_dir
 380         options.localBin = self.bin_dir
 381         options.logdir = self.xpcshell_dir
 382         options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
 383         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
 384         options.objdir = self.topobjdir
 385         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
 386         options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
 387         options.testsRootDir = self.xpcshell_dir
 388         options.testPath = test_path
 389         options.use_device_libs = True
 390
 391         options.emulator = 'arm'
 392         if device_name.startswith('emulator'):
 393             if 'x86' in device_name:
 394                 options.emulator = 'x86'
 395
 396         if not options.busybox:
 397             options.busybox = self._download_busybox(b2g_home, options.emulator)
 398
 399         return runtestsb2g.run_remote_xpcshell(parser, options, args)
 400
 401 def is_platform_supported(cls):
 402     """Must have a Firefox, Android or B2G build."""
 403     return conditions.is_android(cls) or \
 404            conditions.is_b2g(cls) or \
 405            conditions.is_firefox(cls)
 406
 407 @CommandProvider
 408 class MachCommands(MachCommandBase):
 409     def __init__(self, context):
 410         MachCommandBase.__init__(self, context)
 411
 412         for attr in ('b2g_home', 'device_name'):
 413             setattr(self, attr, getattr(context, attr, None))
 414
 415     @Command('xpcshell-test', category='testing',
 416         conditions=[is_platform_supported],
 417         description='Run XPCOM Shell tests (API direct unit testing)')
 418     @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
 419         help='Test to run. Can be specified as a single JS file, a directory, '
 420              'or omitted. If omitted, the entire test suite is executed.')
 421     @CommandArgument('--verbose', '-v', action='store_true',
 422         help='Provide full output from each test process.')
 423     @CommandArgument("--debugger", default=None, metavar='DEBUGGER',
 424                      help = "Run xpcshell under the given debugger.")
 425     @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
 426                      dest = "debuggerArgs",
 427                      help = "pass the given args to the debugger _before_ "
 428                             "the application on the command line")
 429     @CommandArgument("--debugger-interactive", action = "store_true",
 430                      dest = "debuggerInteractive",
 431                      help = "prevents the test harness from redirecting "
 432                             "stdout and stderr for interactive debuggers")
 433     @CommandArgument('--interactive', '-i', action='store_true',
 434         help='Open an xpcshell prompt before running tests.')
 435     @CommandArgument('--keep-going', '-k', action='store_true',
 436         help='Continue running tests after a SIGINT is received.')
 437     @CommandArgument('--sequential', action='store_true',
 438         help='Run the tests sequentially.')
 439     @CommandArgument('--shuffle', '-s', action='store_true',
 440         help='Randomize the execution order of tests.')
 441     @CommandArgument('--rerun-failures', action='store_true',
 442         help='Reruns failures from last time.')
 443     @CommandArgument('--devicemanager', default='adb', type=str,
 444         help='(Android) Type of devicemanager to use for communication: adb or sut')
 445     @CommandArgument('--ip', type=str, default=None,
 446         help='(Android) IP address of device')
 447     @CommandArgument('--port', type=int, default=20701,
 448         help='(Android) Port of device')
 449     @CommandArgument('--remote_test_root', type=str, default=None,
 450         help='(Android) Remote test root such as /mnt/sdcard or /data/local')
 451     @CommandArgument('--no-setup', action='store_true',
 452         help='(Android) Do not copy files to device')
 453     @CommandArgument('--local-apk', type=str, default=None,
 454         help='(Android) Use specified Fennec APK')
 455     @CommandArgument('--busybox', type=str, default=None,
 456         help='(B2G) Path to busybox binary (speeds up installation of tests).')
 457     def run_xpcshell_test(self, **params):
 458         from mozbuild.controller.building import BuildDriver
 459
 460         # We should probably have a utility function to ensure the tree is
 461         # ready to run tests. Until then, we just create the state dir (in
 462         # case the tree wasn't built with mach).
 463         self._ensure_state_subdir_exists('.')
 464
 465         driver = self._spawn(BuildDriver)
 466         driver.install_tests(remove=False)
 467
 468         if conditions.is_android(self):
 469             xpcshell = self._spawn(AndroidXPCShellRunner)
 470         elif conditions.is_b2g(self):
 471             xpcshell = self._spawn(B2GXPCShellRunner)
 472             params['b2g_home'] = self.b2g_home
 473             params['device_name'] = self.device_name
 474         else:
 475             xpcshell = self._spawn(XPCShellRunner)
 476         xpcshell.cwd = self._mach_context.cwd
 477
 478         try:
 479             return xpcshell.run_test(**params)
 480         except InvalidTestPathError as e:
 481             print(e.message)
 482             return 1