testing/mochitest/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import functools
   6 import logging
   7 import os
   8 import sys
   9 import warnings
  10 from argparse import Namespace
  11 from collections import defaultdict
  12
  13 import six
  14 from mach.decorators import Command, CommandArgument
  15 from mozbuild.base import MachCommandConditions as conditions
  16 from mozbuild.base import MozbuildObject
  17 from mozfile import load_source
  18
  19 here = os.path.abspath(os.path.dirname(__file__))
  20
  21
  22 ENG_BUILD_REQUIRED = """
  23 The mochitest command requires an engineering build. It may be the case that
  24 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
  25
  26     $ VARIANT=eng ./build.sh
  27
  28 There should be an app called 'test-container.gaiamobile.org' located in
  29 {}.
  30 """.lstrip()
  31
  32 SUPPORTED_TESTS_NOT_FOUND = """
  33 The mochitest command could not find any supported tests to run! The
  34 following flavors and subsuites were found, but are either not supported on
  35 {} builds, or were excluded on the command line:
  36
  37 {}
  38
  39 Double check the command line you used, and make sure you are running in
  40 context of the proper build. To switch build contexts, either run |mach|
  41 from the appropriate objdir, or export the correct mozconfig:
  42
  43     $ export MOZCONFIG=path/to/mozconfig
  44 """.lstrip()
  45
  46 TESTS_NOT_FOUND = """
  47 The mochitest command could not find any mochitests under the following
  48 test path(s):
  49
  50 {}
  51
  52 Please check spelling and make sure there are mochitests living there.
  53 """.lstrip()
  54
  55 SUPPORTED_APPS = ["firefox", "android", "thunderbird"]
  56
  57 parser = None
  58
  59
  60 class MochitestRunner(MozbuildObject):
  61
  62     """Easily run mochitests.
  63
  64     This currently contains just the basics for running mochitests. We may want
  65     to hook up result parsing, etc.
  66     """
  67
  68     def __init__(self, *args, **kwargs):
  69         MozbuildObject.__init__(self, *args, **kwargs)
  70
  71         # TODO Bug 794506 remove once mach integrates with virtualenv.
  72         build_path = os.path.join(self.topobjdir, "build")
  73         if build_path not in sys.path:
  74             sys.path.append(build_path)
  75
  76         self.tests_dir = os.path.join(self.topobjdir, "_tests")
  77         self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest")
  78         self.bin_dir = os.path.join(self.topobjdir, "dist", "bin")
  79
  80     def resolve_tests(self, test_paths, test_objects=None, cwd=None):
  81         if test_objects:
  82             return test_objects
  83
  84         from moztest.resolve import TestResolver
  85
  86         resolver = self._spawn(TestResolver)
  87         tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
  88         return tests
  89
  90     def run_desktop_test(self, command_context, tests=None, **kwargs):
  91         """Runs a mochitest."""
  92         # runtests.py is ambiguous, so we load the file/module manually.
  93         if "mochitest" not in sys.modules:
  94             path = os.path.join(self.mochitest_dir, "runtests.py")
  95             load_source("mochitest", path)
  96
  97         import mochitest
  98
  99         # This is required to make other components happy. Sad, isn't it?
 100         os.chdir(self.topobjdir)
 101
 102         # Automation installs its own stream handler to stdout. Since we want
 103         # all logging to go through us, we just remove their handler.
 104         remove_handlers = [
 105             l
 106             for l in logging.getLogger().handlers
 107             if isinstance(l, logging.StreamHandler)
 108         ]
 109         for handler in remove_handlers:
 110             logging.getLogger().removeHandler(handler)
 111
 112         options = Namespace(**kwargs)
 113         options.topsrcdir = self.topsrcdir
 114         options.topobjdir = self.topobjdir
 115
 116         from manifestparser import TestManifest
 117
 118         if tests and not options.manifestFile:
 119             manifest = TestManifest()
 120             manifest.tests.extend(tests)
 121             options.manifestFile = manifest
 122
 123             # When developing mochitest-plain tests, it's often useful to be able to
 124             # refresh the page to pick up modifications. Therefore leave the browser
 125             # open if only running a single mochitest-plain test. This behaviour can
 126             # be overridden by passing in --keep-open=false.
 127             if (
 128                 len(tests) == 1
 129                 and options.keep_open is None
 130                 and not options.headless
 131                 and getattr(options, "flavor", "plain") == "plain"
 132             ):
 133                 options.keep_open = True
 134
 135         # We need this to enable colorization of output.
 136         self.log_manager.enable_unstructured()
 137         result = mochitest.run_test_harness(parser, options)
 138         self.log_manager.disable_unstructured()
 139         return result
 140
 141     def run_android_test(self, command_context, tests, **kwargs):
 142         host_ret = verify_host_bin()
 143         if host_ret != 0:
 144             return host_ret
 145
 146         path = os.path.join(self.mochitest_dir, "runtestsremote.py")
 147         load_source("runtestsremote", path)
 148
 149         import runtestsremote
 150
 151         options = Namespace(**kwargs)
 152
 153         from manifestparser import TestManifest
 154
 155         if tests and not options.manifestFile:
 156             manifest = TestManifest()
 157             manifest.tests.extend(tests)
 158             options.manifestFile = manifest
 159
 160         return runtestsremote.run_test_harness(parser, options)
 161
 162     def run_geckoview_junit_test(self, context, **kwargs):
 163         host_ret = verify_host_bin()
 164         if host_ret != 0:
 165             return host_ret
 166
 167         import runjunit
 168
 169         options = Namespace(**kwargs)
 170
 171         return runjunit.run_test_harness(parser, options)
 172
 173
 174 # parser
 175
 176
 177 def setup_argument_parser():
 178     build_obj = MozbuildObject.from_environment(cwd=here)
 179
 180     build_path = os.path.join(build_obj.topobjdir, "build")
 181     if build_path not in sys.path:
 182         sys.path.append(build_path)
 183
 184     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 185
 186     with warnings.catch_warnings():
 187         warnings.simplefilter("ignore")
 188
 189         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 190         if not os.path.exists(path):
 191             path = os.path.join(here, "runtests.py")
 192
 193         load_source("mochitest", path)
 194
 195         from mochitest_options import MochitestArgumentParser
 196
 197     if conditions.is_android(build_obj):
 198         # On Android, check for a connected device (and offer to start an
 199         # emulator if appropriate) before running tests. This check must
 200         # be done in this admittedly awkward place because
 201         # MochitestArgumentParser initialization fails if no device is found.
 202         from mozrunner.devices.android_device import (
 203             InstallIntent,
 204             verify_android_device,
 205         )
 206
 207         # verify device and xre
 208         verify_android_device(build_obj, install=InstallIntent.NO, xre=True)
 209
 210     global parser
 211     parser = MochitestArgumentParser()
 212     return parser
 213
 214
 215 def setup_junit_argument_parser():
 216     build_obj = MozbuildObject.from_environment(cwd=here)
 217
 218     build_path = os.path.join(build_obj.topobjdir, "build")
 219     if build_path not in sys.path:
 220         sys.path.append(build_path)
 221
 222     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 223
 224     with warnings.catch_warnings():
 225         warnings.simplefilter("ignore")
 226
 227         # runtests.py contains MochitestDesktop, required by runjunit
 228         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 229         if not os.path.exists(path):
 230             path = os.path.join(here, "runtests.py")
 231
 232         load_source("mochitest", path)
 233
 234         import runjunit
 235         from mozrunner.devices.android_device import (
 236             InstallIntent,
 237             verify_android_device,
 238         )
 239
 240         verify_android_device(
 241             build_obj, install=InstallIntent.NO, xre=True, network=True
 242         )
 243
 244     global parser
 245     parser = runjunit.JunitArgumentParser()
 246     return parser
 247
 248
 249 def verify_host_bin():
 250     # validate MOZ_HOST_BIN environment variables for Android tests
 251     xpcshell_binary = "xpcshell"
 252     if os.name == "nt":
 253         xpcshell_binary = "xpcshell.exe"
 254     MOZ_HOST_BIN = os.environ.get("MOZ_HOST_BIN")
 255     if not MOZ_HOST_BIN:
 256         print(
 257             "environment variable MOZ_HOST_BIN must be set to a directory containing host "
 258             "%s" % xpcshell_binary
 259         )
 260         return 1
 261     elif not os.path.isdir(MOZ_HOST_BIN):
 262         print("$MOZ_HOST_BIN does not specify a directory")
 263         return 1
 264     elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, xpcshell_binary)):
 265         print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary)
 266         return 1
 267     return 0
 268
 269
 270 @Command(
 271     "mochitest",
 272     category="testing",
 273     conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
 274     description="Run any flavor of mochitest (integration test).",
 275     parser=setup_argument_parser,
 276 )
 277 def run_mochitest_general(
 278     command_context, flavor=None, test_objects=None, resolve_tests=True, **kwargs
 279 ):
 280     from mochitest_options import ALL_FLAVORS
 281     from mozlog.commandline import setup_logging
 282     from mozlog.handlers import StreamHandler
 283     from moztest.resolve import get_suite_definition
 284
 285     # TODO: This is only strictly necessary while mochitest is using Python
 286     # 2 and can be removed once the command is migrated to Python 3.
 287     command_context.activate_virtualenv()
 288
 289     buildapp = None
 290     for app in SUPPORTED_APPS:
 291         if conditions.is_buildapp_in(command_context, apps=[app]):
 292             buildapp = app
 293             break
 294
 295     flavors = None
 296     if flavor:
 297         for fname, fobj in six.iteritems(ALL_FLAVORS):
 298             if flavor in fobj["aliases"]:
 299                 if buildapp not in fobj["enabled_apps"]:
 300                     continue
 301                 flavors = [fname]
 302                 break
 303     else:
 304         flavors = [
 305             f for f, v in six.iteritems(ALL_FLAVORS) if buildapp in v["enabled_apps"]
 306         ]
 307
 308     from mozbuild.controller.building import BuildDriver
 309
 310     command_context._ensure_state_subdir_exists(".")
 311
 312     test_paths = kwargs["test_paths"]
 313     kwargs["test_paths"] = []
 314
 315     if kwargs.get("debugger", None):
 316         import mozdebug
 317
 318         if not mozdebug.get_debugger_info(kwargs.get("debugger")):
 319             sys.exit(1)
 320
 321     mochitest = command_context._spawn(MochitestRunner)
 322     tests = []
 323     if resolve_tests:
 324         tests = mochitest.resolve_tests(
 325             test_paths, test_objects, cwd=command_context._mach_context.cwd
 326         )
 327
 328     if not kwargs.get("log"):
 329         # Create shared logger
 330         format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 331         if len(tests) == 1:
 332             format_args["verbose"] = True
 333             format_args["compact"] = False
 334
 335         default_format = command_context._mach_context.settings["test"]["format"]
 336         kwargs["log"] = setup_logging(
 337             "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 338         )
 339         for handler in kwargs["log"].handlers:
 340             if isinstance(handler, StreamHandler):
 341                 handler.formatter.inner.summary_on_shutdown = True
 342
 343     driver = command_context._spawn(BuildDriver)
 344     driver.install_tests()
 345
 346     subsuite = kwargs.get("subsuite")
 347     if subsuite == "default":
 348         kwargs["subsuite"] = None
 349
 350     suites = defaultdict(list)
 351     is_webrtc_tag_present = False
 352     unsupported = set()
 353     for test in tests:
 354         # Check if we're running a webrtc test so we can enable webrtc
 355         # specific test logic later if needed.
 356         if "webrtc" in test.get("tags", ""):
 357             is_webrtc_tag_present = True
 358
 359         # Filter out non-mochitests and unsupported flavors.
 360         if test["flavor"] not in ALL_FLAVORS:
 361             continue
 362
 363         key = (test["flavor"], test.get("subsuite", ""))
 364         if test["flavor"] not in flavors:
 365             unsupported.add(key)
 366             continue
 367
 368         if subsuite == "default":
 369             # "--subsuite default" means only run tests that don't have a subsuite
 370             if test.get("subsuite"):
 371                 unsupported.add(key)
 372                 continue
 373         elif subsuite and test.get("subsuite", "") != subsuite:
 374             unsupported.add(key)
 375             continue
 376
 377         suites[key].append(test)
 378
 379     # Only webrtc mochitests in the media suite need the websocketprocessbridge.
 380     if ("mochitest", "media") in suites and is_webrtc_tag_present:
 381         req = os.path.join(
 382             "testing",
 383             "tools",
 384             "websocketprocessbridge",
 385             "websocketprocessbridge_requirements_3.txt",
 386         )
 387         command_context.virtualenv_manager.activate()
 388         command_context.virtualenv_manager.install_pip_requirements(
 389             req, require_hashes=False
 390         )
 391
 392         # sys.executable is used to start the websocketprocessbridge, though for some
 393         # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
 394         sys.executable = command_context.virtualenv_manager.python_path
 395
 396     if ("browser-chrome", "a11y") in suites and sys.platform == "win32":
 397         # Only Windows a11y browser tests need this.
 398         req = os.path.join(
 399             "accessible",
 400             "tests",
 401             "browser",
 402             "windows",
 403             "a11y_setup_requirements.txt",
 404         )
 405         command_context.virtualenv_manager.activate()
 406         command_context.virtualenv_manager.install_pip_requirements(
 407             req, require_hashes=False
 408         )
 409
 410     # This is a hack to introduce an option in mach to not send
 411     # filtered tests to the mochitest harness. Mochitest harness will read
 412     # the master manifest in that case.
 413     if not resolve_tests:
 414         for flavor in flavors:
 415             key = (flavor, kwargs.get("subsuite"))
 416             suites[key] = []
 417
 418     if not suites:
 419         # Make it very clear why no tests were found
 420         if not unsupported:
 421             print(
 422                 TESTS_NOT_FOUND.format(
 423                     "\n".join(sorted(list(test_paths or test_objects)))
 424                 )
 425             )
 426             return 1
 427
 428         msg = []
 429         for f, s in unsupported:
 430             fobj = ALL_FLAVORS[f]
 431             apps = fobj["enabled_apps"]
 432             name = fobj["aliases"][0]
 433             if s:
 434                 name = "{} --subsuite {}".format(name, s)
 435
 436             if buildapp not in apps:
 437                 reason = "requires {}".format(" or ".join(apps))
 438             else:
 439                 reason = "excluded by the command line"
 440             msg.append("    mochitest -f {} ({})".format(name, reason))
 441         print(SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
 442         return 1
 443
 444     if buildapp == "android":
 445         from mozrunner.devices.android_device import (
 446             InstallIntent,
 447             get_adb_path,
 448             verify_android_device,
 449         )
 450
 451         app = kwargs.get("app")
 452         if not app:
 453             app = "org.mozilla.geckoview.test_runner"
 454         device_serial = kwargs.get("deviceSerial")
 455         install = InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES
 456         aab = kwargs.get("aab")
 457
 458         # verify installation
 459         verify_android_device(
 460             command_context,
 461             install=install,
 462             xre=False,
 463             network=True,
 464             app=app,
 465             aab=aab,
 466             device_serial=device_serial,
 467         )
 468
 469         if not kwargs["adbPath"]:
 470             kwargs["adbPath"] = get_adb_path(command_context)
 471
 472         run_mochitest = mochitest.run_android_test
 473     else:
 474         run_mochitest = mochitest.run_desktop_test
 475
 476     overall = None
 477     for (flavor, subsuite), tests in sorted(suites.items()):
 478         suite_name, suite = get_suite_definition(flavor, subsuite)
 479         if "test_paths" in suite["kwargs"]:
 480             del suite["kwargs"]["test_paths"]
 481
 482         harness_args = kwargs.copy()
 483         harness_args.update(suite["kwargs"])
 484         # Pass in the full suite name as defined in moztest/resolve.py in case
 485         # chunk-by-runtime is called, in which case runtime information for
 486         # specific mochitest suite has to be loaded. See Bug 1637463.
 487         harness_args.update({"suite_name": suite_name})
 488
 489         result = run_mochitest(
 490             command_context._mach_context, tests=tests, **harness_args
 491         )
 492
 493         if result:
 494             overall = result
 495
 496         # Halt tests on keyboard interrupt
 497         if result == -1:
 498             break
 499
 500     # Only shutdown the logger if we created it
 501     if kwargs["log"].name == "mach-mochitest":
 502         kwargs["log"].shutdown()
 503
 504     return overall
 505
 506
 507 @Command(
 508     "geckoview-junit",
 509     category="testing",
 510     conditions=[conditions.is_android],
 511     description="Run remote geckoview junit tests.",
 512     parser=setup_junit_argument_parser,
 513 )
 514 @CommandArgument(
 515     "--no-install",
 516     help="Do not try to install application on device before "
 517     + "running (default: False)",
 518     action="store_true",
 519     default=False,
 520 )
 521 def run_junit(command_context, no_install, **kwargs):
 522     command_context._ensure_state_subdir_exists(".")
 523
 524     from mozrunner.devices.android_device import (
 525         InstallIntent,
 526         get_adb_path,
 527         verify_android_device,
 528     )
 529
 530     # verify installation
 531     app = kwargs.get("app")
 532     device_serial = kwargs.get("deviceSerial")
 533     verify_android_device(
 534         command_context,
 535         install=InstallIntent.NO if no_install else InstallIntent.YES,
 536         xre=False,
 537         app=app,
 538         device_serial=device_serial,
 539     )
 540
 541     if not kwargs.get("adbPath"):
 542         kwargs["adbPath"] = get_adb_path(command_context)
 543
 544     if not kwargs.get("log"):
 545         from mozlog.commandline import setup_logging
 546
 547         format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 548         default_format = command_context._mach_context.settings["test"]["format"]
 549         kwargs["log"] = setup_logging(
 550             "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 551         )
 552
 553     mochitest = command_context._spawn(MochitestRunner)
 554     return mochitest.run_geckoview_junit_test(command_context._mach_context, **kwargs)