testing/mochitest/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 from argparse import Namespace
   8 from collections import defaultdict
   9 import functools
  10 import logging
  11 import os
  12 import six
  13 import sys
  14 import warnings
  15
  16 from mozbuild.base import (
  17     MachCommandBase,
  18     MachCommandConditions as conditions,
  19     MozbuildObject,
  20 )
  21
  22 from mach.decorators import (
  23     CommandArgument,
  24     CommandProvider,
  25     Command,
  26 )
  27
  28 here = os.path.abspath(os.path.dirname(__file__))
  29
  30
  31 ENG_BUILD_REQUIRED = """
  32 The mochitest command requires an engineering build. It may be the case that
  33 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
  34
  35     $ VARIANT=eng ./build.sh
  36
  37 There should be an app called 'test-container.gaiamobile.org' located in
  38 {}.
  39 """.lstrip()
  40
  41 SUPPORTED_TESTS_NOT_FOUND = """
  42 The mochitest command could not find any supported tests to run! The
  43 following flavors and subsuites were found, but are either not supported on
  44 {} builds, or were excluded on the command line:
  45
  46 {}
  47
  48 Double check the command line you used, and make sure you are running in
  49 context of the proper build. To switch build contexts, either run |mach|
  50 from the appropriate objdir, or export the correct mozconfig:
  51
  52     $ export MOZCONFIG=path/to/mozconfig
  53 """.lstrip()
  54
  55 TESTS_NOT_FOUND = """
  56 The mochitest command could not find any mochitests under the following
  57 test path(s):
  58
  59 {}
  60
  61 Please check spelling and make sure there are mochitests living there.
  62 """.lstrip()
  63
  64 SUPPORTED_APPS = ["firefox", "android", "thunderbird"]
  65
  66 parser = None
  67
  68
  69 class MochitestRunner(MozbuildObject):
  70
  71     """Easily run mochitests.
  72
  73     This currently contains just the basics for running mochitests. We may want
  74     to hook up result parsing, etc.
  75     """
  76
  77     def __init__(self, *args, **kwargs):
  78         MozbuildObject.__init__(self, *args, **kwargs)
  79
  80         # TODO Bug 794506 remove once mach integrates with virtualenv.
  81         build_path = os.path.join(self.topobjdir, "build")
  82         if build_path not in sys.path:
  83             sys.path.append(build_path)
  84
  85         self.tests_dir = os.path.join(self.topobjdir, "_tests")
  86         self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest")
  87         self.bin_dir = os.path.join(self.topobjdir, "dist", "bin")
  88
  89     def resolve_tests(self, test_paths, test_objects=None, cwd=None):
  90         if test_objects:
  91             return test_objects
  92
  93         from moztest.resolve import TestResolver
  94
  95         resolver = self._spawn(TestResolver)
  96         tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
  97         return tests
  98
  99     def run_desktop_test(self, context, tests=None, **kwargs):
 100         """Runs a mochitest."""
 101         # runtests.py is ambiguous, so we load the file/module manually.
 102         if "mochitest" not in sys.modules:
 103             import imp
 104
 105             path = os.path.join(self.mochitest_dir, "runtests.py")
 106             with open(path, "r") as fh:
 107                 imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 108
 109         import mochitest
 110
 111         # This is required to make other components happy. Sad, isn't it?
 112         os.chdir(self.topobjdir)
 113
 114         # Automation installs its own stream handler to stdout. Since we want
 115         # all logging to go through us, we just remove their handler.
 116         remove_handlers = [
 117             l
 118             for l in logging.getLogger().handlers
 119             if isinstance(l, logging.StreamHandler)
 120         ]
 121         for handler in remove_handlers:
 122             logging.getLogger().removeHandler(handler)
 123
 124         options = Namespace(**kwargs)
 125         options.topsrcdir = self.topsrcdir
 126         options.topobjdir = self.topobjdir
 127
 128         from manifestparser import TestManifest
 129
 130         if tests and not options.manifestFile:
 131             manifest = TestManifest()
 132             manifest.tests.extend(tests)
 133             options.manifestFile = manifest
 134
 135             # When developing mochitest-plain tests, it's often useful to be able to
 136             # refresh the page to pick up modifications. Therefore leave the browser
 137             # open if only running a single mochitest-plain test. This behaviour can
 138             # be overridden by passing in --keep-open=false.
 139             if (
 140                 len(tests) == 1
 141                 and options.keep_open is None
 142                 and not options.headless
 143                 and getattr(options, "flavor", "plain") == "plain"
 144             ):
 145                 options.keep_open = True
 146
 147         # We need this to enable colorization of output.
 148         self.log_manager.enable_unstructured()
 149         result = mochitest.run_test_harness(parser, options)
 150         self.log_manager.disable_unstructured()
 151         return result
 152
 153     def run_android_test(self, context, tests, **kwargs):
 154         host_ret = verify_host_bin()
 155         if host_ret != 0:
 156             return host_ret
 157
 158         import imp
 159
 160         path = os.path.join(self.mochitest_dir, "runtestsremote.py")
 161         with open(path, "r") as fh:
 162             imp.load_module("runtestsremote", fh, path, (".py", "r", imp.PY_SOURCE))
 163         import runtestsremote
 164
 165         from mozrunner.devices.android_device import get_adb_path
 166
 167         if not kwargs["adbPath"]:
 168             kwargs["adbPath"] = get_adb_path(command_context)
 169
 170         options = Namespace(**kwargs)
 171
 172         from manifestparser import TestManifest
 173
 174         if tests and not options.manifestFile:
 175             manifest = TestManifest()
 176             manifest.tests.extend(tests)
 177             options.manifestFile = manifest
 178
 179         # Firefox for Android doesn't use e10s
 180         if options.app is not None and "geckoview" not in options.app:
 181             options.e10s = False
 182             print("using e10s=False for non-geckoview app")
 183
 184         return runtestsremote.run_test_harness(parser, options)
 185
 186     def run_geckoview_junit_test(self, context, **kwargs):
 187         host_ret = verify_host_bin()
 188         if host_ret != 0:
 189             return host_ret
 190
 191         import runjunit
 192
 193         options = Namespace(**kwargs)
 194         return runjunit.run_test_harness(parser, options)
 195
 196
 197 # parser
 198
 199
 200 def setup_argument_parser():
 201     build_obj = MozbuildObject.from_environment(cwd=here)
 202
 203     build_path = os.path.join(build_obj.topobjdir, "build")
 204     if build_path not in sys.path:
 205         sys.path.append(build_path)
 206
 207     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 208
 209     with warnings.catch_warnings():
 210         warnings.simplefilter("ignore")
 211
 212         import imp
 213
 214         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 215         if not os.path.exists(path):
 216             path = os.path.join(here, "runtests.py")
 217
 218         with open(path, "r") as fh:
 219             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 220
 221         from mochitest_options import MochitestArgumentParser
 222
 223     if conditions.is_android(build_obj):
 224         # On Android, check for a connected device (and offer to start an
 225         # emulator if appropriate) before running tests. This check must
 226         # be done in this admittedly awkward place because
 227         # MochitestArgumentParser initialization fails if no device is found.
 228         from mozrunner.devices.android_device import (
 229             verify_android_device,
 230             InstallIntent,
 231         )
 232
 233         # verify device and xre
 234         verify_android_device(build_obj, install=InstallIntent.NO, xre=True)
 235
 236     global parser
 237     parser = MochitestArgumentParser()
 238     return parser
 239
 240
 241 def setup_junit_argument_parser():
 242     build_obj = MozbuildObject.from_environment(cwd=here)
 243
 244     build_path = os.path.join(build_obj.topobjdir, "build")
 245     if build_path not in sys.path:
 246         sys.path.append(build_path)
 247
 248     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 249
 250     with warnings.catch_warnings():
 251         warnings.simplefilter("ignore")
 252
 253         # runtests.py contains MochitestDesktop, required by runjunit
 254         import imp
 255
 256         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 257         if not os.path.exists(path):
 258             path = os.path.join(here, "runtests.py")
 259
 260         with open(path, "r") as fh:
 261             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 262
 263         import runjunit
 264
 265         from mozrunner.devices.android_device import (
 266             verify_android_device,
 267             InstallIntent,
 268         )
 269
 270         verify_android_device(
 271             build_obj, install=InstallIntent.NO, xre=True, network=True
 272         )
 273
 274     global parser
 275     parser = runjunit.JunitArgumentParser()
 276     return parser
 277
 278
 279 def verify_host_bin():
 280     # validate MOZ_HOST_BIN environment variables for Android tests
 281     xpcshell_binary = "xpcshell"
 282     if os.name == "nt":
 283         xpcshell_binary = "xpcshell.exe"
 284     MOZ_HOST_BIN = os.environ.get("MOZ_HOST_BIN")
 285     if not MOZ_HOST_BIN:
 286         print(
 287             "environment variable MOZ_HOST_BIN must be set to a directory containing host "
 288             "%s" % xpcshell_binary
 289         )
 290         return 1
 291     elif not os.path.isdir(MOZ_HOST_BIN):
 292         print("$MOZ_HOST_BIN does not specify a directory")
 293         return 1
 294     elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, xpcshell_binary)):
 295         print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary)
 296         return 1
 297     return 0
 298
 299
 300 @CommandProvider
 301 class MachCommands(MachCommandBase):
 302     @Command(
 303         "mochitest",
 304         category="testing",
 305         conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
 306         description="Run any flavor of mochitest (integration test).",
 307         parser=setup_argument_parser,
 308     )
 309     def run_mochitest_general(
 310         self,
 311         command_context,
 312         flavor=None,
 313         test_objects=None,
 314         resolve_tests=True,
 315         **kwargs
 316     ):
 317         from mochitest_options import ALL_FLAVORS
 318         from mozlog.commandline import setup_logging
 319         from mozlog.handlers import StreamHandler
 320         from moztest.resolve import get_suite_definition
 321
 322         # TODO: This is only strictly necessary while mochitest is using Python
 323         # 2 and can be removed once the command is migrated to Python 3.
 324         command_context.activate_virtualenv()
 325
 326         buildapp = None
 327         for app in SUPPORTED_APPS:
 328             if conditions.is_buildapp_in(command_context, apps=[app]):
 329                 buildapp = app
 330                 break
 331
 332         flavors = None
 333         if flavor:
 334             for fname, fobj in six.iteritems(ALL_FLAVORS):
 335                 if flavor in fobj["aliases"]:
 336                     if buildapp not in fobj["enabled_apps"]:
 337                         continue
 338                     flavors = [fname]
 339                     break
 340         else:
 341             flavors = [
 342                 f
 343                 for f, v in six.iteritems(ALL_FLAVORS)
 344                 if buildapp in v["enabled_apps"]
 345             ]
 346
 347         from mozbuild.controller.building import BuildDriver
 348
 349         command_context._ensure_state_subdir_exists(".")
 350
 351         test_paths = kwargs["test_paths"]
 352         kwargs["test_paths"] = []
 353
 354         if kwargs.get("debugger", None):
 355             import mozdebug
 356
 357             if not mozdebug.get_debugger_info(kwargs.get("debugger")):
 358                 sys.exit(1)
 359
 360         mochitest = command_context._spawn(MochitestRunner)
 361         tests = []
 362         if resolve_tests:
 363             tests = mochitest.resolve_tests(
 364                 test_paths, test_objects, cwd=command_context._mach_context.cwd
 365             )
 366
 367         if not kwargs.get("log"):
 368             # Create shared logger
 369             format_args = {
 370                 "level": command_context._mach_context.settings["test"]["level"]
 371             }
 372             if len(tests) == 1:
 373                 format_args["verbose"] = True
 374                 format_args["compact"] = False
 375
 376             default_format = command_context._mach_context.settings["test"]["format"]
 377             kwargs["log"] = setup_logging(
 378                 "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 379             )
 380             for handler in kwargs["log"].handlers:
 381                 if isinstance(handler, StreamHandler):
 382                     handler.formatter.inner.summary_on_shutdown = True
 383
 384         driver = command_context._spawn(BuildDriver)
 385         driver.install_tests()
 386
 387         subsuite = kwargs.get("subsuite")
 388         if subsuite == "default":
 389             kwargs["subsuite"] = None
 390
 391         suites = defaultdict(list)
 392         is_webrtc_tag_present = False
 393         unsupported = set()
 394         for test in tests:
 395             # Check if we're running a webrtc test so we can enable webrtc
 396             # specific test logic later if needed.
 397             if "webrtc" in test.get("tags", ""):
 398                 is_webrtc_tag_present = True
 399
 400             # Filter out non-mochitests and unsupported flavors.
 401             if test["flavor"] not in ALL_FLAVORS:
 402                 continue
 403
 404             key = (test["flavor"], test.get("subsuite", ""))
 405             if test["flavor"] not in flavors:
 406                 unsupported.add(key)
 407                 continue
 408
 409             if subsuite == "default":
 410                 # "--subsuite default" means only run tests that don't have a subsuite
 411                 if test.get("subsuite"):
 412                     unsupported.add(key)
 413                     continue
 414             elif subsuite and test.get("subsuite", "") != subsuite:
 415                 unsupported.add(key)
 416                 continue
 417
 418             suites[key].append(test)
 419
 420         # Only webrtc mochitests in the media suite need the websocketprocessbridge.
 421         if ("mochitest", "media") in suites and is_webrtc_tag_present:
 422             req = os.path.join(
 423                 "testing",
 424                 "tools",
 425                 "websocketprocessbridge",
 426                 "websocketprocessbridge_requirements_3.txt",
 427             )
 428             command_context.virtualenv_manager.activate()
 429             command_context.virtualenv_manager.install_pip_requirements(
 430                 req, require_hashes=False
 431             )
 432
 433             # sys.executable is used to start the websocketprocessbridge, though for some
 434             # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
 435             sys.executable = command_context.virtualenv_manager.python_path
 436
 437         # This is a hack to introduce an option in mach to not send
 438         # filtered tests to the mochitest harness. Mochitest harness will read
 439         # the master manifest in that case.
 440         if not resolve_tests:
 441             for flavor in flavors:
 442                 key = (flavor, kwargs.get("subsuite"))
 443                 suites[key] = []
 444
 445         if not suites:
 446             # Make it very clear why no tests were found
 447             if not unsupported:
 448                 print(
 449                     TESTS_NOT_FOUND.format(
 450                         "\n".join(sorted(list(test_paths or test_objects)))
 451                     )
 452                 )
 453                 return 1
 454
 455             msg = []
 456             for f, s in unsupported:
 457                 fobj = ALL_FLAVORS[f]
 458                 apps = fobj["enabled_apps"]
 459                 name = fobj["aliases"][0]
 460                 if s:
 461                     name = "{} --subsuite {}".format(name, s)
 462
 463                 if buildapp not in apps:
 464                     reason = "requires {}".format(" or ".join(apps))
 465                 else:
 466                     reason = "excluded by the command line"
 467                 msg.append("    mochitest -f {} ({})".format(name, reason))
 468             print(SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
 469             return 1
 470
 471         if buildapp == "android":
 472             from mozrunner.devices.android_device import (
 473                 verify_android_device,
 474                 InstallIntent,
 475             )
 476
 477             app = kwargs.get("app")
 478             if not app:
 479                 app = "org.mozilla.geckoview.test"
 480             device_serial = kwargs.get("deviceSerial")
 481             install = (
 482                 InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES
 483             )
 484
 485             # verify installation
 486             verify_android_device(
 487                 command_context,
 488                 install=install,
 489                 xre=False,
 490                 network=True,
 491                 app=app,
 492                 device_serial=device_serial,
 493             )
 494             run_mochitest = mochitest.run_android_test
 495         else:
 496             run_mochitest = mochitest.run_desktop_test
 497
 498         overall = None
 499         for (flavor, subsuite), tests in sorted(suites.items()):
 500             suite_name, suite = get_suite_definition(flavor, subsuite)
 501             if "test_paths" in suite["kwargs"]:
 502                 del suite["kwargs"]["test_paths"]
 503
 504             harness_args = kwargs.copy()
 505             harness_args.update(suite["kwargs"])
 506             # Pass in the full suite name as defined in moztest/resolve.py in case
 507             # chunk-by-runtime is called, in which case runtime information for
 508             # specific mochitest suite has to be loaded. See Bug 1637463.
 509             harness_args.update({"suite_name": suite_name})
 510
 511             result = run_mochitest(
 512                 command_context._mach_context, tests=tests, **harness_args
 513             )
 514
 515             if result:
 516                 overall = result
 517
 518             # Halt tests on keyboard interrupt
 519             if result == -1:
 520                 break
 521
 522         # Only shutdown the logger if we created it
 523         if kwargs["log"].name == "mach-mochitest":
 524             kwargs["log"].shutdown()
 525
 526         return overall
 527
 528
 529 @CommandProvider
 530 class GeckoviewJunitCommands(MachCommandBase):
 531     @Command(
 532         "geckoview-junit",
 533         category="testing",
 534         conditions=[conditions.is_android],
 535         description="Run remote geckoview junit tests.",
 536         parser=setup_junit_argument_parser,
 537     )
 538     @CommandArgument(
 539         "--no-install",
 540         help="Do not try to install application on device before "
 541         + "running (default: False)",
 542         action="store_true",
 543         default=False,
 544     )
 545     def run_junit(self, command_context, no_install, **kwargs):
 546         command_context._ensure_state_subdir_exists(".")
 547
 548         from mozrunner.devices.android_device import (
 549             get_adb_path,
 550             verify_android_device,
 551             InstallIntent,
 552         )
 553
 554         # verify installation
 555         app = kwargs.get("app")
 556         device_serial = kwargs.get("deviceSerial")
 557         verify_android_device(
 558             command_context,
 559             install=InstallIntent.NO if no_install else InstallIntent.YES,
 560             xre=False,
 561             app=app,
 562             device_serial=device_serial,
 563         )
 564
 565         if not kwargs.get("adbPath"):
 566             kwargs["adbPath"] = get_adb_path(command_context)
 567
 568         if not kwargs.get("log"):
 569             from mozlog.commandline import setup_logging
 570
 571             format_args = {
 572                 "level": command_context._mach_context.settings["test"]["level"]
 573             }
 574             default_format = command_context._mach_context.settings["test"]["format"]
 575             kwargs["log"] = setup_logging(
 576                 "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 577             )
 578
 579         mochitest = command_context._spawn(MochitestRunner)
 580         return mochitest.run_geckoview_junit_test(
 581             command_context._mach_context, **kwargs
 582         )