testing/mochitest/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, unicode_literals
   6
   7 from argparse import Namespace
   8 from collections import defaultdict
   9 import logging
  10 import os
  11 import sys
  12 import warnings
  13
  14 from mozbuild.base import (
  15     MachCommandBase,
  16     MachCommandConditions as conditions,
  17     MozbuildObject,
  18 )
  19
  20 from mach.decorators import (
  21     CommandArgument,
  22     CommandProvider,
  23     Command,
  24 )
  25
  26 here = os.path.abspath(os.path.dirname(__file__))
  27
  28
  29 ENG_BUILD_REQUIRED = '''
  30 The mochitest command requires an engineering build. It may be the case that
  31 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
  32
  33     $ VARIANT=eng ./build.sh
  34
  35 There should be an app called 'test-container.gaiamobile.org' located in
  36 {}.
  37 '''.lstrip()
  38
  39 SUPPORTED_TESTS_NOT_FOUND = '''
  40 The mochitest command could not find any supported tests to run! The
  41 following flavors and subsuites were found, but are either not supported on
  42 {} builds, or were excluded on the command line:
  43
  44 {}
  45
  46 Double check the command line you used, and make sure you are running in
  47 context of the proper build. To switch build contexts, either run |mach|
  48 from the appropriate objdir, or export the correct mozconfig:
  49
  50     $ export MOZCONFIG=path/to/mozconfig
  51 '''.lstrip()
  52
  53 TESTS_NOT_FOUND = '''
  54 The mochitest command could not find any mochitests under the following
  55 test path(s):
  56
  57 {}
  58
  59 Please check spelling and make sure there are mochitests living there.
  60 '''.lstrip()
  61
  62 ROBOCOP_TESTS_NOT_FOUND = '''
  63 The robocop command could not find any tests under the following
  64 test path(s):
  65
  66 {}
  67
  68 Please check spelling and make sure the named tests exist.
  69 '''.lstrip()
  70
  71 SUPPORTED_APPS = ['firefox', 'android']
  72
  73 parser = None
  74
  75
  76 class MochitestRunner(MozbuildObject):
  77
  78     """Easily run mochitests.
  79
  80     This currently contains just the basics for running mochitests. We may want
  81     to hook up result parsing, etc.
  82     """
  83
  84     def __init__(self, *args, **kwargs):
  85         MozbuildObject.__init__(self, *args, **kwargs)
  86
  87         # TODO Bug 794506 remove once mach integrates with virtualenv.
  88         build_path = os.path.join(self.topobjdir, 'build')
  89         if build_path not in sys.path:
  90             sys.path.append(build_path)
  91
  92         self.tests_dir = os.path.join(self.topobjdir, '_tests')
  93         self.mochitest_dir = os.path.join(
  94             self.tests_dir,
  95             'testing',
  96             'mochitest')
  97         self.bin_dir = os.path.join(self.topobjdir, 'dist', 'bin')
  98
  99     def resolve_tests(self, test_paths, test_objects=None, cwd=None):
 100         if test_objects:
 101             return test_objects
 102
 103         from moztest.resolve import TestResolver
 104         resolver = self._spawn(TestResolver)
 105         tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
 106         return tests
 107
 108     def run_desktop_test(self, context, tests=None, suite=None, **kwargs):
 109         """Runs a mochitest.
 110
 111         suite is the type of mochitest to run. It can be one of ('plain',
 112         'chrome', 'browser', 'a11y').
 113         """
 114         # runtests.py is ambiguous, so we load the file/module manually.
 115         if 'mochitest' not in sys.modules:
 116             import imp
 117             path = os.path.join(self.mochitest_dir, 'runtests.py')
 118             with open(path, 'r') as fh:
 119                 imp.load_module('mochitest', fh, path,
 120                                 ('.py', 'r', imp.PY_SOURCE))
 121
 122         import mochitest
 123
 124         # This is required to make other components happy. Sad, isn't it?
 125         os.chdir(self.topobjdir)
 126
 127         # Automation installs its own stream handler to stdout. Since we want
 128         # all logging to go through us, we just remove their handler.
 129         remove_handlers = [l for l in logging.getLogger().handlers
 130                            if isinstance(l, logging.StreamHandler)]
 131         for handler in remove_handlers:
 132             logging.getLogger().removeHandler(handler)
 133
 134         options = Namespace(**kwargs)
 135         options.topsrcdir = self.topsrcdir
 136
 137         from manifestparser import TestManifest
 138         if tests and not options.manifestFile:
 139             manifest = TestManifest()
 140             manifest.tests.extend(tests)
 141             options.manifestFile = manifest
 142
 143             # When developing mochitest-plain tests, it's often useful to be able to
 144             # refresh the page to pick up modifications. Therefore leave the browser
 145             # open if only running a single mochitest-plain test. This behaviour can
 146             # be overridden by passing in --keep-open=false.
 147             if len(tests) == 1 and options.keep_open is None and suite == 'plain':
 148                 options.keep_open = True
 149
 150         # We need this to enable colorization of output.
 151         self.log_manager.enable_unstructured()
 152         result = mochitest.run_test_harness(parser, options)
 153         self.log_manager.disable_unstructured()
 154         return result
 155
 156     def run_android_test(self, context, tests, suite=None, **kwargs):
 157         host_ret = verify_host_bin()
 158         if host_ret != 0:
 159             return host_ret
 160
 161         import imp
 162         path = os.path.join(self.mochitest_dir, 'runtestsremote.py')
 163         with open(path, 'r') as fh:
 164             imp.load_module('runtestsremote', fh, path,
 165                             ('.py', 'r', imp.PY_SOURCE))
 166         import runtestsremote
 167
 168         from mozrunner.devices.android_device import get_adb_path
 169         if not kwargs['adbPath']:
 170             kwargs['adbPath'] = get_adb_path(self)
 171
 172         options = Namespace(**kwargs)
 173
 174         from manifestparser import TestManifest
 175         if tests and not options.manifestFile:
 176             manifest = TestManifest()
 177             manifest.tests.extend(tests)
 178             options.manifestFile = manifest
 179
 180         # Firefox for Android doesn't use e10s
 181         if options.app is None or 'geckoview' not in options.app:
 182             options.e10s = False
 183             print("using e10s=False for non-geckoview app")
 184
 185         return runtestsremote.run_test_harness(parser, options)
 186
 187     def run_geckoview_junit_test(self, context, **kwargs):
 188         host_ret = verify_host_bin()
 189         if host_ret != 0:
 190             return host_ret
 191
 192         import runjunit
 193         options = Namespace(**kwargs)
 194         return runjunit.run_test_harness(parser, options)
 195
 196     def run_robocop_test(self, context, tests, suite=None, **kwargs):
 197         host_ret = verify_host_bin()
 198         if host_ret != 0:
 199             return host_ret
 200
 201         import imp
 202         path = os.path.join(self.mochitest_dir, 'runrobocop.py')
 203         with open(path, 'r') as fh:
 204             imp.load_module('runrobocop', fh, path,
 205                             ('.py', 'r', imp.PY_SOURCE))
 206         import runrobocop
 207
 208         options = Namespace(**kwargs)
 209
 210         from manifestparser import TestManifest
 211         if tests and not options.manifestFile:
 212             manifest = TestManifest()
 213             manifest.tests.extend(tests)
 214             options.manifestFile = manifest
 215
 216         # robocop only used for Firefox for Android - non-e10s
 217         options.e10s = False
 218         print("using e10s=False for robocop")
 219
 220         return runrobocop.run_test_harness(parser, options)
 221
 222 # parser
 223
 224
 225 def setup_argument_parser():
 226     build_obj = MozbuildObject.from_environment(cwd=here)
 227
 228     build_path = os.path.join(build_obj.topobjdir, 'build')
 229     if build_path not in sys.path:
 230         sys.path.append(build_path)
 231
 232     mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest')
 233
 234     with warnings.catch_warnings():
 235         warnings.simplefilter('ignore')
 236
 237         import imp
 238         path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py')
 239         if not os.path.exists(path):
 240             path = os.path.join(here, "runtests.py")
 241
 242         with open(path, 'r') as fh:
 243             imp.load_module('mochitest', fh, path,
 244                             ('.py', 'r', imp.PY_SOURCE))
 245
 246         from mochitest_options import MochitestArgumentParser
 247
 248     if conditions.is_android(build_obj):
 249         # On Android, check for a connected device (and offer to start an
 250         # emulator if appropriate) before running tests. This check must
 251         # be done in this admittedly awkward place because
 252         # MochitestArgumentParser initialization fails if no device is found.
 253         from mozrunner.devices.android_device import verify_android_device
 254         # verify device and xre
 255         verify_android_device(build_obj, install=False, xre=True)
 256
 257     global parser
 258     parser = MochitestArgumentParser()
 259     return parser
 260
 261
 262 def setup_junit_argument_parser():
 263     build_obj = MozbuildObject.from_environment(cwd=here)
 264
 265     build_path = os.path.join(build_obj.topobjdir, 'build')
 266     if build_path not in sys.path:
 267         sys.path.append(build_path)
 268
 269     mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest')
 270
 271     with warnings.catch_warnings():
 272         warnings.simplefilter('ignore')
 273
 274         # runtests.py contains MochitestDesktop, required by runjunit
 275         import imp
 276         path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py')
 277         if not os.path.exists(path):
 278             path = os.path.join(here, "runtests.py")
 279
 280         with open(path, 'r') as fh:
 281             imp.load_module('mochitest', fh, path,
 282                             ('.py', 'r', imp.PY_SOURCE))
 283
 284         import runjunit
 285
 286         from mozrunner.devices.android_device import verify_android_device
 287         verify_android_device(build_obj, install=False, xre=True)
 288
 289     global parser
 290     parser = runjunit.JunitArgumentParser()
 291     return parser
 292
 293
 294 # condition filters
 295
 296 def is_buildapp_in(*apps):
 297     def is_buildapp_supported(cls):
 298         for a in apps:
 299             c = getattr(conditions, 'is_{}'.format(a), None)
 300             if c and c(cls):
 301                 return True
 302         return False
 303
 304     is_buildapp_supported.__doc__ = 'Must have a {} build.'.format(
 305         ' or '.join(apps))
 306     return is_buildapp_supported
 307
 308
 309 def verify_host_bin():
 310     # validate MOZ_HOST_BIN environment variables for Android tests
 311     MOZ_HOST_BIN = os.environ.get('MOZ_HOST_BIN')
 312     if not MOZ_HOST_BIN:
 313         print('environment variable MOZ_HOST_BIN must be set to a directory containing host '
 314               'xpcshell')
 315         return 1
 316     elif not os.path.isdir(MOZ_HOST_BIN):
 317         print('$MOZ_HOST_BIN does not specify a directory')
 318         return 1
 319     elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, 'xpcshell')):
 320         print('$MOZ_HOST_BIN/xpcshell does not exist')
 321         return 1
 322     return 0
 323
 324
 325 @CommandProvider
 326 class MachCommands(MachCommandBase):
 327     @Command('mochitest', category='testing',
 328              conditions=[is_buildapp_in(*SUPPORTED_APPS)],
 329              description='Run any flavor of mochitest (integration test).',
 330              parser=setup_argument_parser)
 331     def run_mochitest_general(self, flavor=None, test_objects=None, resolve_tests=True, **kwargs):
 332         from mochitest_options import ALL_FLAVORS
 333         from mozlog.commandline import setup_logging
 334         from mozlog.handlers import StreamHandler
 335
 336         buildapp = None
 337         for app in SUPPORTED_APPS:
 338             if is_buildapp_in(app)(self):
 339                 buildapp = app
 340                 break
 341
 342         flavors = None
 343         if flavor:
 344             for fname, fobj in ALL_FLAVORS.iteritems():
 345                 if flavor in fobj['aliases']:
 346                     if buildapp not in fobj['enabled_apps']:
 347                         continue
 348                     flavors = [fname]
 349                     break
 350         else:
 351             flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]
 352
 353         from mozbuild.controller.building import BuildDriver
 354         self._ensure_state_subdir_exists('.')
 355
 356         test_paths = kwargs['test_paths']
 357         kwargs['test_paths'] = []
 358
 359         mochitest = self._spawn(MochitestRunner)
 360         tests = []
 361         if resolve_tests:
 362             tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)
 363
 364         if not kwargs.get('log'):
 365             # Create shared logger
 366             format_args = {'level': self._mach_context.settings['test']['level']}
 367             if len(tests) == 1:
 368                 format_args['verbose'] = True
 369                 format_args['compact'] = False
 370
 371             default_format = self._mach_context.settings['test']['format']
 372             kwargs['log'] = setup_logging('mach-mochitest', kwargs, {default_format: sys.stdout},
 373                                           format_args)
 374             for handler in kwargs['log'].handlers:
 375                 if isinstance(handler, StreamHandler):
 376                     handler.formatter.inner.summary_on_shutdown = True
 377
 378         driver = self._spawn(BuildDriver)
 379         driver.install_tests(tests)
 380
 381         subsuite = kwargs.get('subsuite')
 382         if subsuite == 'default':
 383             kwargs['subsuite'] = None
 384
 385         suites = defaultdict(list)
 386         unsupported = set()
 387         for test in tests:
 388             # Filter out non-mochitests and unsupported flavors.
 389             if test['flavor'] not in ALL_FLAVORS:
 390                 continue
 391
 392             key = (test['flavor'], test.get('subsuite', ''))
 393             if test['flavor'] not in flavors:
 394                 unsupported.add(key)
 395                 continue
 396
 397             if subsuite == 'default':
 398                 # "--subsuite default" means only run tests that don't have a subsuite
 399                 if test.get('subsuite'):
 400                     unsupported.add(key)
 401                     continue
 402             elif subsuite and test.get('subsuite', '') != subsuite:
 403                 unsupported.add(key)
 404                 continue
 405
 406             suites[key].append(test)
 407
 408         if ('mochitest', 'media') in suites:
 409             req = os.path.join('testing', 'tools', 'websocketprocessbridge',
 410                                'websocketprocessbridge_requirements.txt')
 411             self.virtualenv_manager.activate()
 412             self.virtualenv_manager.install_pip_requirements(req, require_hashes=False)
 413
 414             # sys.executable is used to start the websocketprocessbridge, though for some
 415             # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
 416             sys.executable = self.virtualenv_manager.python_path
 417
 418         # This is a hack to introduce an option in mach to not send
 419         # filtered tests to the mochitest harness. Mochitest harness will read
 420         # the master manifest in that case.
 421         if not resolve_tests:
 422             for flavor in flavors:
 423                 key = (flavor, kwargs.get('subsuite'))
 424                 suites[key] = []
 425
 426         if not suites:
 427             # Make it very clear why no tests were found
 428             if not unsupported:
 429                 print(TESTS_NOT_FOUND.format('\n'.join(
 430                     sorted(list(test_paths or test_objects)))))
 431                 return 1
 432
 433             msg = []
 434             for f, s in unsupported:
 435                 fobj = ALL_FLAVORS[f]
 436                 apps = fobj['enabled_apps']
 437                 name = fobj['aliases'][0]
 438                 if s:
 439                     name = '{} --subsuite {}'.format(name, s)
 440
 441                 if buildapp not in apps:
 442                     reason = 'requires {}'.format(' or '.join(apps))
 443                 else:
 444                     reason = 'excluded by the command line'
 445                 msg.append('    mochitest -f {} ({})'.format(name, reason))
 446             print(SUPPORTED_TESTS_NOT_FOUND.format(
 447                 buildapp, '\n'.join(sorted(msg))))
 448             return 1
 449
 450         if buildapp == 'android':
 451             from mozrunner.devices.android_device import grant_runtime_permissions
 452             from mozrunner.devices.android_device import verify_android_device
 453             app = kwargs.get('app')
 454             if not app:
 455                 app = self.substs["ANDROID_PACKAGE_NAME"]
 456             device_serial = kwargs.get('deviceSerial')
 457
 458             # verify installation
 459             verify_android_device(self, install=True, xre=False, app=app,
 460                                   device_serial=device_serial)
 461             grant_runtime_permissions(self, app, device_serial=device_serial)
 462             run_mochitest = mochitest.run_android_test
 463         else:
 464             run_mochitest = mochitest.run_desktop_test
 465
 466         overall = None
 467         for (flavor, subsuite), tests in sorted(suites.items()):
 468             fobj = ALL_FLAVORS[flavor]
 469
 470             harness_args = kwargs.copy()
 471             harness_args['subsuite'] = subsuite
 472             harness_args.update(fobj.get('extra_args', {}))
 473
 474             result = run_mochitest(
 475                 self._mach_context,
 476                 tests=tests,
 477                 suite=fobj['suite'],
 478                 **harness_args)
 479
 480             if result:
 481                 overall = result
 482
 483             # Halt tests on keyboard interrupt
 484             if result == -1:
 485                 break
 486
 487         # Only shutdown the logger if we created it
 488         if kwargs['log'].name == 'mach-mochitest':
 489             kwargs['log'].shutdown()
 490
 491         return overall
 492
 493
 494 @CommandProvider
 495 class GeckoviewJunitCommands(MachCommandBase):
 496
 497     @Command('geckoview-junit', category='testing',
 498              conditions=[conditions.is_android],
 499              description='Run remote geckoview junit tests.',
 500              parser=setup_junit_argument_parser)
 501     def run_junit(self, **kwargs):
 502         self._ensure_state_subdir_exists('.')
 503
 504         from mozrunner.devices.android_device import (grant_runtime_permissions,
 505                                                       get_adb_path,
 506                                                       verify_android_device)
 507         # verify installation
 508         app = kwargs.get('app')
 509         device_serial = kwargs.get('deviceSerial')
 510         verify_android_device(self, install=True, xre=False, app=app,
 511                               device_serial=device_serial)
 512         grant_runtime_permissions(self, app, device_serial=device_serial)
 513
 514         if not kwargs.get('adbPath'):
 515             kwargs['adbPath'] = get_adb_path(self)
 516
 517         if not kwargs.get('log'):
 518             from mozlog.commandline import setup_logging
 519             format_args = {'level': self._mach_context.settings['test']['level']}
 520             default_format = self._mach_context.settings['test']['format']
 521             kwargs['log'] = setup_logging('mach-mochitest', kwargs,
 522                                           {default_format: sys.stdout}, format_args)
 523
 524         mochitest = self._spawn(MochitestRunner)
 525         return mochitest.run_geckoview_junit_test(self._mach_context, **kwargs)
 526
 527
 528 @CommandProvider
 529 class RobocopCommands(MachCommandBase):
 530
 531     @Command('robocop', category='testing',
 532              conditions=[conditions.is_android],
 533              description='Run a Robocop test.',
 534              parser=setup_argument_parser)
 535     @CommandArgument('--serve', default=False, action='store_true',
 536                      help='Run no tests but start the mochi.test web server '
 537                      'and launch Fennec with a test profile.')
 538     def run_robocop(self, serve=False, **kwargs):
 539         if serve:
 540             kwargs['autorun'] = False
 541
 542         if not kwargs.get('robocopIni'):
 543             kwargs['robocopIni'] = os.path.join(self.topobjdir, '_tests', 'testing',
 544                                                 'mochitest', 'robocop.ini')
 545
 546         from mozbuild.controller.building import BuildDriver
 547         self._ensure_state_subdir_exists('.')
 548
 549         test_paths = kwargs['test_paths']
 550         kwargs['test_paths'] = []
 551
 552         from moztest.resolve import TestResolver
 553         resolver = self._spawn(TestResolver)
 554         tests = list(resolver.resolve_tests(paths=test_paths, cwd=self._mach_context.cwd,
 555                                             flavor='instrumentation', subsuite='robocop'))
 556         driver = self._spawn(BuildDriver)
 557         driver.install_tests(tests)
 558
 559         if len(tests) < 1:
 560             print(ROBOCOP_TESTS_NOT_FOUND.format('\n'.join(
 561                 sorted(list(test_paths)))))
 562             return 1
 563
 564         from mozrunner.devices.android_device import grant_runtime_permissions, get_adb_path
 565         from mozrunner.devices.android_device import verify_android_device
 566         # verify installation
 567         app = kwargs.get('app')
 568         if not app:
 569             app = self.substs["ANDROID_PACKAGE_NAME"]
 570         device_serial = kwargs.get('deviceSerial')
 571         verify_android_device(self, install=True, xre=False, app=app,
 572                               device_serial=device_serial)
 573         grant_runtime_permissions(self, app, device_serial=device_serial)
 574
 575         if not kwargs['adbPath']:
 576             kwargs['adbPath'] = get_adb_path(self)
 577
 578         mochitest = self._spawn(MochitestRunner)
 579         return mochitest.run_robocop_test(self._mach_context, tests, 'robocop', **kwargs)
 580
 581
 582 # NOTE python/mach/mach/commands/commandinfo.py references this function
 583 #      by name. If this function is renamed or removed, that file should
 584 #      be updated accordingly as well.
 585 def REMOVED(cls):
 586     """Command no longer exists! Use |mach mochitest| instead.
 587
 588     The |mach mochitest| command will automatically detect which flavors and
 589     subsuites exist in a given directory. If desired, flavors and subsuites
 590     can be restricted using `--flavor` and `--subsuite` respectively. E.g:
 591
 592         $ ./mach mochitest dom/indexedDB
 593
 594     will run all of the plain, chrome and browser-chrome mochitests in that
 595     directory. To only run the plain mochitests:
 596
 597         $ ./mach mochitest -f plain dom/indexedDB
 598     """
 599     return False
 600
 601
 602 @CommandProvider
 603 class DeprecatedCommands(MachCommandBase):
 604     @Command('mochitest-plain', category='testing', conditions=[REMOVED])
 605     def mochitest_plain(self):
 606         pass
 607
 608     @Command('mochitest-chrome', category='testing', conditions=[REMOVED])
 609     def mochitest_chrome(self):
 610         pass
 611
 612     @Command('mochitest-browser', category='testing', conditions=[REMOVED])
 613     def mochitest_browser(self):
 614         pass
 615
 616     @Command('mochitest-devtools', category='testing', conditions=[REMOVED])
 617     def mochitest_devtools(self):
 618         pass
 619
 620     @Command('mochitest-a11y', category='testing', conditions=[REMOVED])
 621     def mochitest_a11y(self):
 622         pass