testing/mochitest/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import functools
   6 import logging
   7 import os
   8 import sys
   9 import warnings
  10 from argparse import Namespace
  11 from collections import defaultdict
  12
  13 import six
  14 from mach.decorators import Command, CommandArgument
  15 from mozbuild.base import MachCommandConditions as conditions
  16 from mozbuild.base import MozbuildObject
  17
  18 here = os.path.abspath(os.path.dirname(__file__))
  19
  20
  21 ENG_BUILD_REQUIRED = """
  22 The mochitest command requires an engineering build. It may be the case that
  23 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
  24
  25     $ VARIANT=eng ./build.sh
  26
  27 There should be an app called 'test-container.gaiamobile.org' located in
  28 {}.
  29 """.lstrip()
  30
  31 SUPPORTED_TESTS_NOT_FOUND = """
  32 The mochitest command could not find any supported tests to run! The
  33 following flavors and subsuites were found, but are either not supported on
  34 {} builds, or were excluded on the command line:
  35
  36 {}
  37
  38 Double check the command line you used, and make sure you are running in
  39 context of the proper build. To switch build contexts, either run |mach|
  40 from the appropriate objdir, or export the correct mozconfig:
  41
  42     $ export MOZCONFIG=path/to/mozconfig
  43 """.lstrip()
  44
  45 TESTS_NOT_FOUND = """
  46 The mochitest command could not find any mochitests under the following
  47 test path(s):
  48
  49 {}
  50
  51 Please check spelling and make sure there are mochitests living there.
  52 """.lstrip()
  53
  54 SUPPORTED_APPS = ["firefox", "android", "thunderbird"]
  55
  56 parser = None
  57
  58
  59 class MochitestRunner(MozbuildObject):
  60
  61     """Easily run mochitests.
  62
  63     This currently contains just the basics for running mochitests. We may want
  64     to hook up result parsing, etc.
  65     """
  66
  67     def __init__(self, *args, **kwargs):
  68         MozbuildObject.__init__(self, *args, **kwargs)
  69
  70         # TODO Bug 794506 remove once mach integrates with virtualenv.
  71         build_path = os.path.join(self.topobjdir, "build")
  72         if build_path not in sys.path:
  73             sys.path.append(build_path)
  74
  75         self.tests_dir = os.path.join(self.topobjdir, "_tests")
  76         self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest")
  77         self.bin_dir = os.path.join(self.topobjdir, "dist", "bin")
  78
  79     def resolve_tests(self, test_paths, test_objects=None, cwd=None):
  80         if test_objects:
  81             return test_objects
  82
  83         from moztest.resolve import TestResolver
  84
  85         resolver = self._spawn(TestResolver)
  86         tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
  87         return tests
  88
  89     def run_desktop_test(self, command_context, tests=None, **kwargs):
  90         """Runs a mochitest."""
  91         # runtests.py is ambiguous, so we load the file/module manually.
  92         if "mochitest" not in sys.modules:
  93             import imp
  94
  95             path = os.path.join(self.mochitest_dir, "runtests.py")
  96             with open(path, "r") as fh:
  97                 imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
  98
  99         import mochitest
 100
 101         # This is required to make other components happy. Sad, isn't it?
 102         os.chdir(self.topobjdir)
 103
 104         # Automation installs its own stream handler to stdout. Since we want
 105         # all logging to go through us, we just remove their handler.
 106         remove_handlers = [
 107             l
 108             for l in logging.getLogger().handlers
 109             if isinstance(l, logging.StreamHandler)
 110         ]
 111         for handler in remove_handlers:
 112             logging.getLogger().removeHandler(handler)
 113
 114         options = Namespace(**kwargs)
 115         options.topsrcdir = self.topsrcdir
 116         options.topobjdir = self.topobjdir
 117
 118         from manifestparser import TestManifest
 119
 120         if tests and not options.manifestFile:
 121             manifest = TestManifest()
 122             manifest.tests.extend(tests)
 123             options.manifestFile = manifest
 124
 125             # When developing mochitest-plain tests, it's often useful to be able to
 126             # refresh the page to pick up modifications. Therefore leave the browser
 127             # open if only running a single mochitest-plain test. This behaviour can
 128             # be overridden by passing in --keep-open=false.
 129             if (
 130                 len(tests) == 1
 131                 and options.keep_open is None
 132                 and not options.headless
 133                 and getattr(options, "flavor", "plain") == "plain"
 134             ):
 135                 options.keep_open = True
 136
 137         # We need this to enable colorization of output.
 138         self.log_manager.enable_unstructured()
 139         result = mochitest.run_test_harness(parser, options)
 140         self.log_manager.disable_unstructured()
 141         return result
 142
 143     def run_android_test(self, command_context, tests, **kwargs):
 144         host_ret = verify_host_bin()
 145         if host_ret != 0:
 146             return host_ret
 147
 148         import imp
 149
 150         path = os.path.join(self.mochitest_dir, "runtestsremote.py")
 151         with open(path, "r") as fh:
 152             imp.load_module("runtestsremote", fh, path, (".py", "r", imp.PY_SOURCE))
 153         import runtestsremote
 154
 155         options = Namespace(**kwargs)
 156
 157         from manifestparser import TestManifest
 158
 159         if tests and not options.manifestFile:
 160             manifest = TestManifest()
 161             manifest.tests.extend(tests)
 162             options.manifestFile = manifest
 163
 164         # Firefox for Android doesn't use e10s
 165         if options.app is not None and "geckoview" not in options.app:
 166             options.e10s = False
 167             print("using e10s=False for non-geckoview app")
 168
 169         return runtestsremote.run_test_harness(parser, options)
 170
 171     def run_geckoview_junit_test(self, context, **kwargs):
 172         host_ret = verify_host_bin()
 173         if host_ret != 0:
 174             return host_ret
 175
 176         import runjunit
 177
 178         options = Namespace(**kwargs)
 179
 180         return runjunit.run_test_harness(parser, options)
 181
 182
 183 # parser
 184
 185
 186 def setup_argument_parser():
 187     build_obj = MozbuildObject.from_environment(cwd=here)
 188
 189     build_path = os.path.join(build_obj.topobjdir, "build")
 190     if build_path not in sys.path:
 191         sys.path.append(build_path)
 192
 193     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 194
 195     with warnings.catch_warnings():
 196         warnings.simplefilter("ignore")
 197
 198         import imp
 199
 200         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 201         if not os.path.exists(path):
 202             path = os.path.join(here, "runtests.py")
 203
 204         with open(path, "r") as fh:
 205             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 206
 207         from mochitest_options import MochitestArgumentParser
 208
 209     if conditions.is_android(build_obj):
 210         # On Android, check for a connected device (and offer to start an
 211         # emulator if appropriate) before running tests. This check must
 212         # be done in this admittedly awkward place because
 213         # MochitestArgumentParser initialization fails if no device is found.
 214         from mozrunner.devices.android_device import (
 215             InstallIntent,
 216             verify_android_device,
 217         )
 218
 219         # verify device and xre
 220         verify_android_device(build_obj, install=InstallIntent.NO, xre=True)
 221
 222     global parser
 223     parser = MochitestArgumentParser()
 224     return parser
 225
 226
 227 def setup_junit_argument_parser():
 228     build_obj = MozbuildObject.from_environment(cwd=here)
 229
 230     build_path = os.path.join(build_obj.topobjdir, "build")
 231     if build_path not in sys.path:
 232         sys.path.append(build_path)
 233
 234     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 235
 236     with warnings.catch_warnings():
 237         warnings.simplefilter("ignore")
 238
 239         # runtests.py contains MochitestDesktop, required by runjunit
 240         import imp
 241
 242         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 243         if not os.path.exists(path):
 244             path = os.path.join(here, "runtests.py")
 245
 246         with open(path, "r") as fh:
 247             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 248
 249         import runjunit
 250         from mozrunner.devices.android_device import (
 251             InstallIntent,
 252             verify_android_device,
 253         )
 254
 255         verify_android_device(
 256             build_obj, install=InstallIntent.NO, xre=True, network=True
 257         )
 258
 259     global parser
 260     parser = runjunit.JunitArgumentParser()
 261     return parser
 262
 263
 264 def verify_host_bin():
 265     # validate MOZ_HOST_BIN environment variables for Android tests
 266     xpcshell_binary = "xpcshell"
 267     if os.name == "nt":
 268         xpcshell_binary = "xpcshell.exe"
 269     MOZ_HOST_BIN = os.environ.get("MOZ_HOST_BIN")
 270     if not MOZ_HOST_BIN:
 271         print(
 272             "environment variable MOZ_HOST_BIN must be set to a directory containing host "
 273             "%s" % xpcshell_binary
 274         )
 275         return 1
 276     elif not os.path.isdir(MOZ_HOST_BIN):
 277         print("$MOZ_HOST_BIN does not specify a directory")
 278         return 1
 279     elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, xpcshell_binary)):
 280         print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary)
 281         return 1
 282     return 0
 283
 284
 285 @Command(
 286     "mochitest",
 287     category="testing",
 288     conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
 289     description="Run any flavor of mochitest (integration test).",
 290     parser=setup_argument_parser,
 291 )
 292 def run_mochitest_general(
 293     command_context, flavor=None, test_objects=None, resolve_tests=True, **kwargs
 294 ):
 295     from mochitest_options import ALL_FLAVORS
 296     from mozlog.commandline import setup_logging
 297     from mozlog.handlers import StreamHandler
 298     from moztest.resolve import get_suite_definition
 299
 300     # TODO: This is only strictly necessary while mochitest is using Python
 301     # 2 and can be removed once the command is migrated to Python 3.
 302     command_context.activate_virtualenv()
 303
 304     buildapp = None
 305     for app in SUPPORTED_APPS:
 306         if conditions.is_buildapp_in(command_context, apps=[app]):
 307             buildapp = app
 308             break
 309
 310     flavors = None
 311     if flavor:
 312         for fname, fobj in six.iteritems(ALL_FLAVORS):
 313             if flavor in fobj["aliases"]:
 314                 if buildapp not in fobj["enabled_apps"]:
 315                     continue
 316                 flavors = [fname]
 317                 break
 318     else:
 319         flavors = [
 320             f for f, v in six.iteritems(ALL_FLAVORS) if buildapp in v["enabled_apps"]
 321         ]
 322
 323     from mozbuild.controller.building import BuildDriver
 324
 325     command_context._ensure_state_subdir_exists(".")
 326
 327     test_paths = kwargs["test_paths"]
 328     kwargs["test_paths"] = []
 329
 330     if kwargs.get("debugger", None):
 331         import mozdebug
 332
 333         if not mozdebug.get_debugger_info(kwargs.get("debugger")):
 334             sys.exit(1)
 335
 336     mochitest = command_context._spawn(MochitestRunner)
 337     tests = []
 338     if resolve_tests:
 339         tests = mochitest.resolve_tests(
 340             test_paths, test_objects, cwd=command_context._mach_context.cwd
 341         )
 342
 343     if not kwargs.get("log"):
 344         # Create shared logger
 345         format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 346         if len(tests) == 1:
 347             format_args["verbose"] = True
 348             format_args["compact"] = False
 349
 350         default_format = command_context._mach_context.settings["test"]["format"]
 351         kwargs["log"] = setup_logging(
 352             "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 353         )
 354         for handler in kwargs["log"].handlers:
 355             if isinstance(handler, StreamHandler):
 356                 handler.formatter.inner.summary_on_shutdown = True
 357
 358     driver = command_context._spawn(BuildDriver)
 359     driver.install_tests()
 360
 361     subsuite = kwargs.get("subsuite")
 362     if subsuite == "default":
 363         kwargs["subsuite"] = None
 364
 365     suites = defaultdict(list)
 366     is_webrtc_tag_present = False
 367     unsupported = set()
 368     for test in tests:
 369         # Check if we're running a webrtc test so we can enable webrtc
 370         # specific test logic later if needed.
 371         if "webrtc" in test.get("tags", ""):
 372             is_webrtc_tag_present = True
 373
 374         # Filter out non-mochitests and unsupported flavors.
 375         if test["flavor"] not in ALL_FLAVORS:
 376             continue
 377
 378         key = (test["flavor"], test.get("subsuite", ""))
 379         if test["flavor"] not in flavors:
 380             unsupported.add(key)
 381             continue
 382
 383         if subsuite == "default":
 384             # "--subsuite default" means only run tests that don't have a subsuite
 385             if test.get("subsuite"):
 386                 unsupported.add(key)
 387                 continue
 388         elif subsuite and test.get("subsuite", "") != subsuite:
 389             unsupported.add(key)
 390             continue
 391
 392         suites[key].append(test)
 393
 394     # Only webrtc mochitests in the media suite need the websocketprocessbridge.
 395     if ("mochitest", "media") in suites and is_webrtc_tag_present:
 396         req = os.path.join(
 397             "testing",
 398             "tools",
 399             "websocketprocessbridge",
 400             "websocketprocessbridge_requirements_3.txt",
 401         )
 402         command_context.virtualenv_manager.activate()
 403         command_context.virtualenv_manager.install_pip_requirements(
 404             req, require_hashes=False
 405         )
 406
 407         # sys.executable is used to start the websocketprocessbridge, though for some
 408         # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
 409         sys.executable = command_context.virtualenv_manager.python_path
 410
 411     # This is a hack to introduce an option in mach to not send
 412     # filtered tests to the mochitest harness. Mochitest harness will read
 413     # the master manifest in that case.
 414     if not resolve_tests:
 415         for flavor in flavors:
 416             key = (flavor, kwargs.get("subsuite"))
 417             suites[key] = []
 418
 419     if not suites:
 420         # Make it very clear why no tests were found
 421         if not unsupported:
 422             print(
 423                 TESTS_NOT_FOUND.format(
 424                     "\n".join(sorted(list(test_paths or test_objects)))
 425                 )
 426             )
 427             return 1
 428
 429         msg = []
 430         for f, s in unsupported:
 431             fobj = ALL_FLAVORS[f]
 432             apps = fobj["enabled_apps"]
 433             name = fobj["aliases"][0]
 434             if s:
 435                 name = "{} --subsuite {}".format(name, s)
 436
 437             if buildapp not in apps:
 438                 reason = "requires {}".format(" or ".join(apps))
 439             else:
 440                 reason = "excluded by the command line"
 441             msg.append("    mochitest -f {} ({})".format(name, reason))
 442         print(SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
 443         return 1
 444
 445     if buildapp == "android":
 446         from mozrunner.devices.android_device import (
 447             InstallIntent,
 448             get_adb_path,
 449             verify_android_device,
 450         )
 451
 452         app = kwargs.get("app")
 453         if not app:
 454             app = "org.mozilla.geckoview.test_runner"
 455         device_serial = kwargs.get("deviceSerial")
 456         install = InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES
 457         aab = kwargs.get("aab")
 458
 459         # verify installation
 460         verify_android_device(
 461             command_context,
 462             install=install,
 463             xre=False,
 464             network=True,
 465             app=app,
 466             aab=aab,
 467             device_serial=device_serial,
 468         )
 469
 470         if not kwargs["adbPath"]:
 471             kwargs["adbPath"] = get_adb_path(command_context)
 472
 473         run_mochitest = mochitest.run_android_test
 474     else:
 475         run_mochitest = mochitest.run_desktop_test
 476
 477     overall = None
 478     for (flavor, subsuite), tests in sorted(suites.items()):
 479         suite_name, suite = get_suite_definition(flavor, subsuite)
 480         if "test_paths" in suite["kwargs"]:
 481             del suite["kwargs"]["test_paths"]
 482
 483         harness_args = kwargs.copy()
 484         harness_args.update(suite["kwargs"])
 485         # Pass in the full suite name as defined in moztest/resolve.py in case
 486         # chunk-by-runtime is called, in which case runtime information for
 487         # specific mochitest suite has to be loaded. See Bug 1637463.
 488         harness_args.update({"suite_name": suite_name})
 489
 490         result = run_mochitest(
 491             command_context._mach_context, tests=tests, **harness_args
 492         )
 493
 494         if result:
 495             overall = result
 496
 497         # Halt tests on keyboard interrupt
 498         if result == -1:
 499             break
 500
 501     # Only shutdown the logger if we created it
 502     if kwargs["log"].name == "mach-mochitest":
 503         kwargs["log"].shutdown()
 504
 505     return overall
 506
 507
 508 @Command(
 509     "geckoview-junit",
 510     category="testing",
 511     conditions=[conditions.is_android],
 512     description="Run remote geckoview junit tests.",
 513     parser=setup_junit_argument_parser,
 514 )
 515 @CommandArgument(
 516     "--no-install",
 517     help="Do not try to install application on device before "
 518     + "running (default: False)",
 519     action="store_true",
 520     default=False,
 521 )
 522 def run_junit(command_context, no_install, **kwargs):
 523     command_context._ensure_state_subdir_exists(".")
 524
 525     from mozrunner.devices.android_device import (
 526         InstallIntent,
 527         get_adb_path,
 528         verify_android_device,
 529     )
 530
 531     # verify installation
 532     app = kwargs.get("app")
 533     device_serial = kwargs.get("deviceSerial")
 534     verify_android_device(
 535         command_context,
 536         install=InstallIntent.NO if no_install else InstallIntent.YES,
 537         xre=False,
 538         app=app,
 539         device_serial=device_serial,
 540     )
 541
 542     if not kwargs.get("adbPath"):
 543         kwargs["adbPath"] = get_adb_path(command_context)
 544
 545     if not kwargs.get("log"):
 546         from mozlog.commandline import setup_logging
 547
 548         format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 549         default_format = command_context._mach_context.settings["test"]["format"]
 550         kwargs["log"] = setup_logging(
 551             "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 552         )
 553
 554     mochitest = command_context._spawn(MochitestRunner)
 555     return mochitest.run_geckoview_junit_test(command_context._mach_context, **kwargs)