testing/xpcshell/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the xpcshell test runner with mach.
   6
   7 import errno
   8 import logging
   9 import os
  10 import sys
  11 from multiprocessing import cpu_count
  12
  13 from mach.decorators import Command
  14 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  15 from mozbuild.base import MachCommandConditions as conditions
  16 from mozlog import structured
  17 from xpcshellcommandline import parser_desktop, parser_remote
  18
  19 here = os.path.abspath(os.path.dirname(__file__))
  20
  21
  22 # This should probably be consolidated with similar classes in other test
  23 # runners.
  24 class InvalidTestPathError(Exception):
  25     """Exception raised when the test path is not valid."""
  26
  27
  28 class XPCShellRunner(MozbuildObject):
  29     """Run xpcshell tests."""
  30
  31     def run_suite(self, **kwargs):
  32         return self._run_xpcshell_harness(**kwargs)
  33
  34     def run_test(self, **kwargs):
  35         """Runs an individual xpcshell test."""
  36
  37         # TODO Bug 794506 remove once mach integrates with virtualenv.
  38         build_path = os.path.join(self.topobjdir, "build")
  39         if build_path not in sys.path:
  40             sys.path.append(build_path)
  41
  42         src_build_path = os.path.join(self.topsrcdir, "mozilla", "build")
  43         if os.path.isdir(src_build_path):
  44             sys.path.append(src_build_path)
  45
  46         return self.run_suite(**kwargs)
  47
  48     def _run_xpcshell_harness(self, **kwargs):
  49         # Obtain a reference to the xpcshell test runner.
  50         import runxpcshelltests
  51
  52         log = kwargs.pop("log")
  53
  54         xpcshell = runxpcshelltests.XPCShellTests(log=log)
  55         self.log_manager.enable_unstructured()
  56
  57         tests_dir = os.path.join(self.topobjdir, "_tests", "xpcshell")
  58         # We want output from the test to be written immediately if we are only
  59         # running a single test.
  60         single_test = (
  61             len(kwargs["testPaths"]) == 1
  62             and os.path.isfile(kwargs["testPaths"][0])
  63             or kwargs["manifest"]
  64             and (len(kwargs["manifest"].test_paths()) == 1)
  65         )
  66
  67         if single_test:
  68             kwargs["verbose"] = True
  69
  70         if kwargs["xpcshell"] is None:
  71             try:
  72                 kwargs["xpcshell"] = self.get_binary_path("xpcshell")
  73             except BinaryNotFoundException as e:
  74                 self.log(
  75                     logging.ERROR, "xpcshell-test", {"error": str(e)}, "ERROR: {error}"
  76                 )
  77                 self.log(logging.INFO, "xpcshell-test", {"help": e.help()}, "{help}")
  78                 return 1
  79
  80         if kwargs["mozInfo"] is None:
  81             kwargs["mozInfo"] = os.path.join(self.topobjdir, "mozinfo.json")
  82
  83         if kwargs["symbolsPath"] is None:
  84             kwargs["symbolsPath"] = os.path.join(self.distdir, "crashreporter-symbols")
  85
  86         if kwargs["logfiles"] is None:
  87             kwargs["logfiles"] = False
  88
  89         if kwargs["profileName"] is None:
  90             kwargs["profileName"] = "firefox"
  91
  92         if kwargs["testingModulesDir"] is None:
  93             kwargs["testingModulesDir"] = os.path.join(self.topobjdir, "_tests/modules")
  94
  95         if kwargs["utility_path"] is None:
  96             kwargs["utility_path"] = self.bindir
  97
  98         if kwargs["manifest"] is None:
  99             kwargs["manifest"] = os.path.join(tests_dir, "xpcshell.toml")
 100
 101         if kwargs["failure_manifest"] is None:
 102             kwargs["failure_manifest"] = os.path.join(
 103                 self.statedir, "xpcshell.failures.toml"
 104             )
 105
 106         # Use the object directory for the temp directory to minimize the chance
 107         # of file scanning. The overhead from e.g. search indexers and anti-virus
 108         # scanners like Windows Defender can add tons of overhead to test execution.
 109         # We encourage people to disable these things in the object directory.
 110         temp_dir = os.path.join(self.topobjdir, "temp")
 111         try:
 112             os.mkdir(temp_dir)
 113         except OSError as e:
 114             if e.errno != errno.EEXIST:
 115                 raise
 116         kwargs["tempDir"] = temp_dir
 117
 118         result = xpcshell.runTests(kwargs)
 119
 120         self.log_manager.disable_unstructured()
 121
 122         if not result and not xpcshell.sequential:
 123             print(
 124                 "Tests were run in parallel. Try running with --sequential "
 125                 "to make sure the failures were not caused by this."
 126             )
 127         return int(not result)
 128
 129
 130 class AndroidXPCShellRunner(MozbuildObject):
 131     """Run Android xpcshell tests."""
 132
 133     def run_test(self, **kwargs):
 134         # TODO Bug 794506 remove once mach integrates with virtualenv.
 135         build_path = os.path.join(self.topobjdir, "build")
 136         if build_path not in sys.path:
 137             sys.path.append(build_path)
 138
 139         import remotexpcshelltests
 140
 141         log = kwargs.pop("log")
 142         self.log_manager.enable_unstructured()
 143
 144         if kwargs["xpcshell"] is None:
 145             kwargs["xpcshell"] = "xpcshell"
 146
 147         if not kwargs["objdir"]:
 148             kwargs["objdir"] = self.topobjdir
 149
 150         if not kwargs["localBin"]:
 151             kwargs["localBin"] = os.path.join(self.topobjdir, "dist/bin")
 152
 153         if not kwargs["testingModulesDir"]:
 154             kwargs["testingModulesDir"] = os.path.join(self.topobjdir, "_tests/modules")
 155
 156         if not kwargs["mozInfo"]:
 157             kwargs["mozInfo"] = os.path.join(self.topobjdir, "mozinfo.json")
 158
 159         if not kwargs["manifest"]:
 160             kwargs["manifest"] = os.path.join(
 161                 self.topobjdir, "_tests/xpcshell/xpcshell.toml"
 162             )
 163
 164         if not kwargs["symbolsPath"]:
 165             kwargs["symbolsPath"] = os.path.join(self.distdir, "crashreporter-symbols")
 166
 167         if self.substs.get("MOZ_BUILD_APP") == "b2g":
 168             kwargs["localAPK"] = None
 169         elif not kwargs["localAPK"]:
 170             for root, _, paths in os.walk(os.path.join(kwargs["objdir"], "gradle")):
 171                 for file_name in paths:
 172                     if file_name.endswith(".apk") and file_name.startswith(
 173                         "test_runner-withGeckoBinaries"
 174                     ):
 175                         kwargs["localAPK"] = os.path.join(root, file_name)
 176                         print("using APK: %s" % kwargs["localAPK"])
 177                         break
 178                 if kwargs["localAPK"]:
 179                     break
 180             else:
 181                 raise Exception("APK not found in objdir. You must specify an APK.")
 182
 183         xpcshell = remotexpcshelltests.XPCShellRemote(kwargs, log)
 184
 185         result = xpcshell.runTests(
 186             kwargs,
 187             testClass=remotexpcshelltests.RemoteXPCShellTestThread,
 188             mobileArgs=xpcshell.mobileArgs,
 189         )
 190
 191         self.log_manager.disable_unstructured()
 192
 193         return int(not result)
 194
 195
 196 def get_parser():
 197     build_obj = MozbuildObject.from_environment(cwd=here)
 198     if (
 199         conditions.is_android(build_obj)
 200         or build_obj.substs.get("MOZ_BUILD_APP") == "b2g"
 201     ):
 202         return parser_remote()
 203     else:
 204         return parser_desktop()
 205
 206
 207 @Command(
 208     "xpcshell-test",
 209     category="testing",
 210     description="Run XPCOM Shell tests (API direct unit testing)",
 211     conditions=[lambda *args: True],
 212     parser=get_parser,
 213 )
 214 def run_xpcshell_test(command_context, test_objects=None, **params):
 215     from mozbuild.controller.building import BuildDriver
 216
 217     if test_objects is not None:
 218         from manifestparser import TestManifest
 219
 220         m = TestManifest()
 221         m.tests.extend(test_objects)
 222         params["manifest"] = m
 223
 224     driver = command_context._spawn(BuildDriver)
 225     driver.install_tests()
 226
 227     # We should probably have a utility function to ensure the tree is
 228     # ready to run tests. Until then, we just create the state dir (in
 229     # case the tree wasn't built with mach).
 230     command_context._ensure_state_subdir_exists(".")
 231
 232     if not params.get("log"):
 233         log_defaults = {
 234             command_context._mach_context.settings["test"]["format"]: sys.stdout
 235         }
 236         fmt_defaults = {
 237             "level": command_context._mach_context.settings["test"]["level"],
 238             "verbose": True,
 239         }
 240         params["log"] = structured.commandline.setup_logging(
 241             "XPCShellTests", params, log_defaults, fmt_defaults
 242         )
 243
 244     if not params["threadCount"]:
 245         # pylint --py3k W1619
 246         params["threadCount"] = int((cpu_count() * 3) / 2)
 247
 248     if (
 249         conditions.is_android(command_context)
 250         or command_context.substs.get("MOZ_BUILD_APP") == "b2g"
 251     ):
 252         from mozrunner.devices.android_device import (
 253             InstallIntent,
 254             get_adb_path,
 255             verify_android_device,
 256         )
 257
 258         install = InstallIntent.YES if params["setup"] else InstallIntent.NO
 259         device_serial = params.get("deviceSerial")
 260         verify_android_device(
 261             command_context,
 262             network=True,
 263             install=install,
 264             device_serial=device_serial,
 265         )
 266         if not params["adbPath"]:
 267             params["adbPath"] = get_adb_path(command_context)
 268         xpcshell = command_context._spawn(AndroidXPCShellRunner)
 269     else:
 270         xpcshell = command_context._spawn(XPCShellRunner)
 271     xpcshell.cwd = command_context._mach_context.cwd
 272
 273     try:
 274         return xpcshell.run_test(**params)
 275     except InvalidTestPathError as e:
 276         print(str(e))
 277         return 1