testing/mochitest/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 from argparse import Namespace
   8 from collections import defaultdict
   9 import functools
  10 import logging
  11 import os
  12 import six
  13 import sys
  14 import warnings
  15
  16 from mozbuild.base import (
  17     MachCommandBase,
  18     MachCommandConditions as conditions,
  19     MozbuildObject,
  20 )
  21
  22 from mach.decorators import (
  23     CommandArgument,
  24     CommandProvider,
  25     Command,
  26 )
  27
  28 here = os.path.abspath(os.path.dirname(__file__))
  29
  30
  31 ENG_BUILD_REQUIRED = """
  32 The mochitest command requires an engineering build. It may be the case that
  33 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
  34
  35     $ VARIANT=eng ./build.sh
  36
  37 There should be an app called 'test-container.gaiamobile.org' located in
  38 {}.
  39 """.lstrip()
  40
  41 SUPPORTED_TESTS_NOT_FOUND = """
  42 The mochitest command could not find any supported tests to run! The
  43 following flavors and subsuites were found, but are either not supported on
  44 {} builds, or were excluded on the command line:
  45
  46 {}
  47
  48 Double check the command line you used, and make sure you are running in
  49 context of the proper build. To switch build contexts, either run |mach|
  50 from the appropriate objdir, or export the correct mozconfig:
  51
  52     $ export MOZCONFIG=path/to/mozconfig
  53 """.lstrip()
  54
  55 TESTS_NOT_FOUND = """
  56 The mochitest command could not find any mochitests under the following
  57 test path(s):
  58
  59 {}
  60
  61 Please check spelling and make sure there are mochitests living there.
  62 """.lstrip()
  63
  64 SUPPORTED_APPS = ["firefox", "android", "thunderbird"]
  65
  66 parser = None
  67
  68
  69 class MochitestRunner(MozbuildObject):
  70
  71     """Easily run mochitests.
  72
  73     This currently contains just the basics for running mochitests. We may want
  74     to hook up result parsing, etc.
  75     """
  76
  77     def __init__(self, *args, **kwargs):
  78         MozbuildObject.__init__(self, *args, **kwargs)
  79
  80         # TODO Bug 794506 remove once mach integrates with virtualenv.
  81         build_path = os.path.join(self.topobjdir, "build")
  82         if build_path not in sys.path:
  83             sys.path.append(build_path)
  84
  85         self.tests_dir = os.path.join(self.topobjdir, "_tests")
  86         self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest")
  87         self.bin_dir = os.path.join(self.topobjdir, "dist", "bin")
  88
  89     def resolve_tests(self, test_paths, test_objects=None, cwd=None):
  90         if test_objects:
  91             return test_objects
  92
  93         from moztest.resolve import TestResolver
  94
  95         resolver = self._spawn(TestResolver)
  96         tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd))
  97         return tests
  98
  99     def run_desktop_test(self, context, tests=None, **kwargs):
 100         """Runs a mochitest."""
 101         # runtests.py is ambiguous, so we load the file/module manually.
 102         if "mochitest" not in sys.modules:
 103             import imp
 104
 105             path = os.path.join(self.mochitest_dir, "runtests.py")
 106             with open(path, "r") as fh:
 107                 imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 108
 109         import mochitest
 110
 111         # This is required to make other components happy. Sad, isn't it?
 112         os.chdir(self.topobjdir)
 113
 114         # Automation installs its own stream handler to stdout. Since we want
 115         # all logging to go through us, we just remove their handler.
 116         remove_handlers = [
 117             l
 118             for l in logging.getLogger().handlers
 119             if isinstance(l, logging.StreamHandler)
 120         ]
 121         for handler in remove_handlers:
 122             logging.getLogger().removeHandler(handler)
 123
 124         options = Namespace(**kwargs)
 125         options.topsrcdir = self.topsrcdir
 126         options.topobjdir = self.topobjdir
 127
 128         from manifestparser import TestManifest
 129
 130         if tests and not options.manifestFile:
 131             manifest = TestManifest()
 132             manifest.tests.extend(tests)
 133             options.manifestFile = manifest
 134
 135             # When developing mochitest-plain tests, it's often useful to be able to
 136             # refresh the page to pick up modifications. Therefore leave the browser
 137             # open if only running a single mochitest-plain test. This behaviour can
 138             # be overridden by passing in --keep-open=false.
 139             if (
 140                 len(tests) == 1
 141                 and options.keep_open is None
 142                 and not options.headless
 143                 and getattr(options, "flavor", "plain") == "plain"
 144             ):
 145                 options.keep_open = True
 146
 147         # We need this to enable colorization of output.
 148         self.log_manager.enable_unstructured()
 149         result = mochitest.run_test_harness(parser, options)
 150         self.log_manager.disable_unstructured()
 151         return result
 152
 153     def run_android_test(self, context, tests, **kwargs):
 154         host_ret = verify_host_bin()
 155         if host_ret != 0:
 156             return host_ret
 157
 158         import imp
 159
 160         path = os.path.join(self.mochitest_dir, "runtestsremote.py")
 161         with open(path, "r") as fh:
 162             imp.load_module("runtestsremote", fh, path, (".py", "r", imp.PY_SOURCE))
 163         import runtestsremote
 164
 165         from mozrunner.devices.android_device import get_adb_path
 166
 167         if not kwargs["adbPath"]:
 168             kwargs["adbPath"] = get_adb_path(self)
 169
 170         options = Namespace(**kwargs)
 171
 172         from manifestparser import TestManifest
 173
 174         if tests and not options.manifestFile:
 175             manifest = TestManifest()
 176             manifest.tests.extend(tests)
 177             options.manifestFile = manifest
 178
 179         # Firefox for Android doesn't use e10s
 180         if options.app is not None and "geckoview" not in options.app:
 181             options.e10s = False
 182             print("using e10s=False for non-geckoview app")
 183
 184         return runtestsremote.run_test_harness(parser, options)
 185
 186     def run_geckoview_junit_test(self, context, **kwargs):
 187         host_ret = verify_host_bin()
 188         if host_ret != 0:
 189             return host_ret
 190
 191         import runjunit
 192
 193         options = Namespace(**kwargs)
 194         return runjunit.run_test_harness(parser, options)
 195
 196
 197 # parser
 198
 199
 200 def setup_argument_parser():
 201     build_obj = MozbuildObject.from_environment(cwd=here)
 202
 203     build_path = os.path.join(build_obj.topobjdir, "build")
 204     if build_path not in sys.path:
 205         sys.path.append(build_path)
 206
 207     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 208
 209     with warnings.catch_warnings():
 210         warnings.simplefilter("ignore")
 211
 212         import imp
 213
 214         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 215         if not os.path.exists(path):
 216             path = os.path.join(here, "runtests.py")
 217
 218         with open(path, "r") as fh:
 219             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 220
 221         from mochitest_options import MochitestArgumentParser
 222
 223     if conditions.is_android(build_obj):
 224         # On Android, check for a connected device (and offer to start an
 225         # emulator if appropriate) before running tests. This check must
 226         # be done in this admittedly awkward place because
 227         # MochitestArgumentParser initialization fails if no device is found.
 228         from mozrunner.devices.android_device import (
 229             verify_android_device,
 230             InstallIntent,
 231         )
 232
 233         # verify device and xre
 234         verify_android_device(build_obj, install=InstallIntent.NO, xre=True)
 235
 236     global parser
 237     parser = MochitestArgumentParser()
 238     return parser
 239
 240
 241 def setup_junit_argument_parser():
 242     build_obj = MozbuildObject.from_environment(cwd=here)
 243
 244     build_path = os.path.join(build_obj.topobjdir, "build")
 245     if build_path not in sys.path:
 246         sys.path.append(build_path)
 247
 248     mochitest_dir = os.path.join(build_obj.topobjdir, "_tests", "testing", "mochitest")
 249
 250     with warnings.catch_warnings():
 251         warnings.simplefilter("ignore")
 252
 253         # runtests.py contains MochitestDesktop, required by runjunit
 254         import imp
 255
 256         path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
 257         if not os.path.exists(path):
 258             path = os.path.join(here, "runtests.py")
 259
 260         with open(path, "r") as fh:
 261             imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
 262
 263         import runjunit
 264
 265         from mozrunner.devices.android_device import (
 266             verify_android_device,
 267             InstallIntent,
 268         )
 269
 270         verify_android_device(
 271             build_obj, install=InstallIntent.NO, xre=True, network=True
 272         )
 273
 274     global parser
 275     parser = runjunit.JunitArgumentParser()
 276     return parser
 277
 278
 279 def verify_host_bin():
 280     # validate MOZ_HOST_BIN environment variables for Android tests
 281     xpcshell_binary = "xpcshell"
 282     if os.name == "nt":
 283         xpcshell_binary = "xpcshell.exe"
 284     MOZ_HOST_BIN = os.environ.get("MOZ_HOST_BIN")
 285     if not MOZ_HOST_BIN:
 286         print(
 287             "environment variable MOZ_HOST_BIN must be set to a directory containing host "
 288             "%s" % xpcshell_binary
 289         )
 290         return 1
 291     elif not os.path.isdir(MOZ_HOST_BIN):
 292         print("$MOZ_HOST_BIN does not specify a directory")
 293         return 1
 294     elif not os.path.isfile(os.path.join(MOZ_HOST_BIN, xpcshell_binary)):
 295         print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary)
 296         return 1
 297     return 0
 298
 299
 300 @CommandProvider
 301 class MachCommands(MachCommandBase):
 302     @Command(
 303         "mochitest",
 304         category="testing",
 305         conditions=[functools.partial(conditions.is_buildapp_in, apps=SUPPORTED_APPS)],
 306         description="Run any flavor of mochitest (integration test).",
 307         parser=setup_argument_parser,
 308     )
 309     def run_mochitest_general(
 310         self,
 311         command_context,
 312         flavor=None,
 313         test_objects=None,
 314         resolve_tests=True,
 315         **kwargs
 316     ):
 317         from mochitest_options import ALL_FLAVORS
 318         from mozlog.commandline import setup_logging
 319         from mozlog.handlers import StreamHandler
 320         from moztest.resolve import get_suite_definition
 321
 322         # TODO: This is only strictly necessary while mochitest is using Python
 323         # 2 and can be removed once the command is migrated to Python 3.
 324         self.activate_virtualenv()
 325
 326         buildapp = None
 327         for app in SUPPORTED_APPS:
 328             if conditions.is_buildapp_in(self, apps=[app]):
 329                 buildapp = app
 330                 break
 331
 332         flavors = None
 333         if flavor:
 334             for fname, fobj in six.iteritems(ALL_FLAVORS):
 335                 if flavor in fobj["aliases"]:
 336                     if buildapp not in fobj["enabled_apps"]:
 337                         continue
 338                     flavors = [fname]
 339                     break
 340         else:
 341             flavors = [
 342                 f
 343                 for f, v in six.iteritems(ALL_FLAVORS)
 344                 if buildapp in v["enabled_apps"]
 345             ]
 346
 347         from mozbuild.controller.building import BuildDriver
 348
 349         self._ensure_state_subdir_exists(".")
 350
 351         test_paths = kwargs["test_paths"]
 352         kwargs["test_paths"] = []
 353
 354         if kwargs.get("debugger", None):
 355             import mozdebug
 356
 357             if not mozdebug.get_debugger_info(kwargs.get("debugger")):
 358                 sys.exit(1)
 359
 360         mochitest = self._spawn(MochitestRunner)
 361         tests = []
 362         if resolve_tests:
 363             tests = mochitest.resolve_tests(
 364                 test_paths, test_objects, cwd=self._mach_context.cwd
 365             )
 366
 367         if not kwargs.get("log"):
 368             # Create shared logger
 369             format_args = {"level": self._mach_context.settings["test"]["level"]}
 370             if len(tests) == 1:
 371                 format_args["verbose"] = True
 372                 format_args["compact"] = False
 373
 374             default_format = self._mach_context.settings["test"]["format"]
 375             kwargs["log"] = setup_logging(
 376                 "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 377             )
 378             for handler in kwargs["log"].handlers:
 379                 if isinstance(handler, StreamHandler):
 380                     handler.formatter.inner.summary_on_shutdown = True
 381
 382         driver = self._spawn(BuildDriver)
 383         driver.install_tests()
 384
 385         subsuite = kwargs.get("subsuite")
 386         if subsuite == "default":
 387             kwargs["subsuite"] = None
 388
 389         suites = defaultdict(list)
 390         is_webrtc_tag_present = False
 391         unsupported = set()
 392         for test in tests:
 393             # Check if we're running a webrtc test so we can enable webrtc
 394             # specific test logic later if needed.
 395             if "webrtc" in test.get("tags", ""):
 396                 is_webrtc_tag_present = True
 397
 398             # Filter out non-mochitests and unsupported flavors.
 399             if test["flavor"] not in ALL_FLAVORS:
 400                 continue
 401
 402             key = (test["flavor"], test.get("subsuite", ""))
 403             if test["flavor"] not in flavors:
 404                 unsupported.add(key)
 405                 continue
 406
 407             if subsuite == "default":
 408                 # "--subsuite default" means only run tests that don't have a subsuite
 409                 if test.get("subsuite"):
 410                     unsupported.add(key)
 411                     continue
 412             elif subsuite and test.get("subsuite", "") != subsuite:
 413                 unsupported.add(key)
 414                 continue
 415
 416             suites[key].append(test)
 417
 418         # Only webrtc mochitests in the media suite need the websocketprocessbridge.
 419         if ("mochitest", "media") in suites and is_webrtc_tag_present:
 420             req = os.path.join(
 421                 "testing",
 422                 "tools",
 423                 "websocketprocessbridge",
 424                 "websocketprocessbridge_requirements_3.txt",
 425             )
 426             self.virtualenv_manager.activate()
 427             self.virtualenv_manager.install_pip_requirements(req, require_hashes=False)
 428
 429             # sys.executable is used to start the websocketprocessbridge, though for some
 430             # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
 431             sys.executable = self.virtualenv_manager.python_path
 432
 433         # This is a hack to introduce an option in mach to not send
 434         # filtered tests to the mochitest harness. Mochitest harness will read
 435         # the master manifest in that case.
 436         if not resolve_tests:
 437             for flavor in flavors:
 438                 key = (flavor, kwargs.get("subsuite"))
 439                 suites[key] = []
 440
 441         if not suites:
 442             # Make it very clear why no tests were found
 443             if not unsupported:
 444                 print(
 445                     TESTS_NOT_FOUND.format(
 446                         "\n".join(sorted(list(test_paths or test_objects)))
 447                     )
 448                 )
 449                 return 1
 450
 451             msg = []
 452             for f, s in unsupported:
 453                 fobj = ALL_FLAVORS[f]
 454                 apps = fobj["enabled_apps"]
 455                 name = fobj["aliases"][0]
 456                 if s:
 457                     name = "{} --subsuite {}".format(name, s)
 458
 459                 if buildapp not in apps:
 460                     reason = "requires {}".format(" or ".join(apps))
 461                 else:
 462                     reason = "excluded by the command line"
 463                 msg.append("    mochitest -f {} ({})".format(name, reason))
 464             print(SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
 465             return 1
 466
 467         if buildapp == "android":
 468             from mozrunner.devices.android_device import (
 469                 verify_android_device,
 470                 InstallIntent,
 471             )
 472
 473             app = kwargs.get("app")
 474             if not app:
 475                 app = "org.mozilla.geckoview.test"
 476             device_serial = kwargs.get("deviceSerial")
 477             install = (
 478                 InstallIntent.NO if kwargs.get("no_install") else InstallIntent.YES
 479             )
 480
 481             # verify installation
 482             verify_android_device(
 483                 self,
 484                 install=install,
 485                 xre=False,
 486                 network=True,
 487                 app=app,
 488                 device_serial=device_serial,
 489             )
 490             run_mochitest = mochitest.run_android_test
 491         else:
 492             run_mochitest = mochitest.run_desktop_test
 493
 494         overall = None
 495         for (flavor, subsuite), tests in sorted(suites.items()):
 496             suite_name, suite = get_suite_definition(flavor, subsuite)
 497             if "test_paths" in suite["kwargs"]:
 498                 del suite["kwargs"]["test_paths"]
 499
 500             harness_args = kwargs.copy()
 501             harness_args.update(suite["kwargs"])
 502             # Pass in the full suite name as defined in moztest/resolve.py in case
 503             # chunk-by-runtime is called, in which case runtime information for
 504             # specific mochitest suite has to be loaded. See Bug 1637463.
 505             harness_args.update({"suite_name": suite_name})
 506
 507             result = run_mochitest(self._mach_context, tests=tests, **harness_args)
 508
 509             if result:
 510                 overall = result
 511
 512             # Halt tests on keyboard interrupt
 513             if result == -1:
 514                 break
 515
 516         # Only shutdown the logger if we created it
 517         if kwargs["log"].name == "mach-mochitest":
 518             kwargs["log"].shutdown()
 519
 520         return overall
 521
 522
 523 @CommandProvider
 524 class GeckoviewJunitCommands(MachCommandBase):
 525     @Command(
 526         "geckoview-junit",
 527         category="testing",
 528         conditions=[conditions.is_android],
 529         description="Run remote geckoview junit tests.",
 530         parser=setup_junit_argument_parser,
 531     )
 532     @CommandArgument(
 533         "--no-install",
 534         help="Do not try to install application on device before "
 535         + "running (default: False)",
 536         action="store_true",
 537         default=False,
 538     )
 539     def run_junit(self, command_context, no_install, **kwargs):
 540         self._ensure_state_subdir_exists(".")
 541
 542         from mozrunner.devices.android_device import (
 543             get_adb_path,
 544             verify_android_device,
 545             InstallIntent,
 546         )
 547
 548         # verify installation
 549         app = kwargs.get("app")
 550         device_serial = kwargs.get("deviceSerial")
 551         verify_android_device(
 552             self,
 553             install=InstallIntent.NO if no_install else InstallIntent.YES,
 554             xre=False,
 555             app=app,
 556             device_serial=device_serial,
 557         )
 558
 559         if not kwargs.get("adbPath"):
 560             kwargs["adbPath"] = get_adb_path(self)
 561
 562         if not kwargs.get("log"):
 563             from mozlog.commandline import setup_logging
 564
 565             format_args = {"level": self._mach_context.settings["test"]["level"]}
 566             default_format = self._mach_context.settings["test"]["format"]
 567             kwargs["log"] = setup_logging(
 568                 "mach-mochitest", kwargs, {default_format: sys.stdout}, format_args
 569             )
 570
 571         mochitest = self._spawn(MochitestRunner)
 572         return mochitest.run_geckoview_junit_test(self._mach_context, **kwargs)