testing/xpcshell/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the xpcshell test runner with mach.
   6
   7 import errno
   8 import logging
   9 import os
  10 import sys
  11 from multiprocessing import cpu_count
  12
  13 from mach.decorators import Command
  14 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  15 from mozbuild.base import MachCommandConditions as conditions
  16 from mozlog import structured
  17 from xpcshellcommandline import parser_desktop, parser_remote
  18
  19 here = os.path.abspath(os.path.dirname(__file__))
  20
  21
  22 # This should probably be consolidated with similar classes in other test
  23 # runners.
  24 class InvalidTestPathError(Exception):
  25     """Exception raised when the test path is not valid."""
  26
  27
  28 class XPCShellRunner(MozbuildObject):
  29     """Run xpcshell tests."""
  30
  31     def run_suite(self, **kwargs):
  32         return self._run_xpcshell_harness(**kwargs)
  33
  34     def run_test(self, **kwargs):
  35         """Runs an individual xpcshell test."""
  36
  37         # TODO Bug 794506 remove once mach integrates with virtualenv.
  38         build_path = os.path.join(self.topobjdir, "build")
  39         if build_path not in sys.path:
  40             sys.path.append(build_path)
  41
  42         src_build_path = os.path.join(self.topsrcdir, "mozilla", "build")
  43         if os.path.isdir(src_build_path):
  44             sys.path.append(src_build_path)
  45
  46         return self.run_suite(**kwargs)
  47
  48     def _run_xpcshell_harness(self, **kwargs):
  49         # Obtain a reference to the xpcshell test runner.
  50         import runxpcshelltests
  51
  52         log = kwargs.pop("log")
  53
  54         xpcshell = runxpcshelltests.XPCShellTests(log=log)
  55         self.log_manager.enable_unstructured()
  56
  57         tests_dir = os.path.join(self.topobjdir, "_tests", "xpcshell")
  58         # We want output from the test to be written immediately if we are only
  59         # running a single test.
  60         single_test = (
  61             len(kwargs["testPaths"]) == 1
  62             and os.path.isfile(kwargs["testPaths"][0])
  63             or kwargs["manifest"]
  64             and (len(kwargs["manifest"].test_paths()) == 1)
  65         )
  66
  67         if single_test:
  68             kwargs["verbose"] = True
  69
  70         if kwargs["xpcshell"] is None:
  71             try:
  72                 kwargs["xpcshell"] = self.get_binary_path("xpcshell")
  73             except BinaryNotFoundException as e:
  74                 self.log(
  75                     logging.ERROR, "xpcshell-test", {"error": str(e)}, "ERROR: {error}"
  76                 )
  77                 self.log(logging.INFO, "xpcshell-test", {"help": e.help()}, "{help}")
  78                 return 1
  79
  80         if kwargs["mozInfo"] is None:
  81             kwargs["mozInfo"] = os.path.join(self.topobjdir, "mozinfo.json")
  82
  83         if kwargs["symbolsPath"] is None:
  84             kwargs["symbolsPath"] = os.path.join(self.distdir, "crashreporter-symbols")
  85
  86         if kwargs["logfiles"] is None:
  87             kwargs["logfiles"] = False
  88
  89         if kwargs["profileName"] is None:
  90             kwargs["profileName"] = "firefox"
  91
  92         if kwargs["testingModulesDir"] is None:
  93             kwargs["testingModulesDir"] = os.path.join(self.topobjdir, "_tests/modules")
  94
  95         if kwargs["utility_path"] is None:
  96             kwargs["utility_path"] = self.bindir
  97
  98         if kwargs["manifest"] is None:
  99             kwargs["manifest"] = os.path.join(tests_dir, "xpcshell.toml")
 100
 101         if kwargs["failure_manifest"] is None:
 102             kwargs["failure_manifest"] = os.path.join(
 103                 self.statedir, "xpcshell.failures.toml"
 104             )
 105
 106         # Use the object directory for the temp directory to minimize the chance
 107         # of file scanning. The overhead from e.g. search indexers and anti-virus
 108         # scanners like Windows Defender can add tons of overhead to test execution.
 109         # We encourage people to disable these things in the object directory.
 110         temp_dir = os.path.join(self.topobjdir, "temp")
 111         try:
 112             os.mkdir(temp_dir)
 113         except OSError as e:
 114             if e.errno != errno.EEXIST:
 115                 raise
 116         kwargs["tempDir"] = temp_dir
 117
 118         result = xpcshell.runTests(kwargs)
 119
 120         self.log_manager.disable_unstructured()
 121
 122         if not result and not xpcshell.sequential:
 123             print(
 124                 "Tests were run in parallel. Try running with --sequential "
 125                 "to make sure the failures were not caused by this."
 126             )
 127         return int(not result)
 128
 129
 130 class AndroidXPCShellRunner(MozbuildObject):
 131     """Run Android xpcshell tests."""
 132
 133     def run_test(self, **kwargs):
 134         # TODO Bug 794506 remove once mach integrates with virtualenv.
 135         build_path = os.path.join(self.topobjdir, "build")
 136         if build_path not in sys.path:
 137             sys.path.append(build_path)
 138
 139         import remotexpcshelltests
 140
 141         log = kwargs.pop("log")
 142         self.log_manager.enable_unstructured()
 143
 144         if kwargs["xpcshell"] is None:
 145             kwargs["xpcshell"] = "xpcshell"
 146
 147         if not kwargs["objdir"]:
 148             kwargs["objdir"] = self.topobjdir
 149
 150         if not kwargs["localBin"]:
 151             kwargs["localBin"] = os.path.join(self.topobjdir, "dist/bin")
 152
 153         if not kwargs["testingModulesDir"]:
 154             kwargs["testingModulesDir"] = os.path.join(self.topobjdir, "_tests/modules")
 155
 156         if not kwargs["mozInfo"]:
 157             kwargs["mozInfo"] = os.path.join(self.topobjdir, "mozinfo.json")
 158
 159         if not kwargs["manifest"]:
 160             kwargs["manifest"] = os.path.join(
 161                 self.topobjdir, "_tests/xpcshell/xpcshell.toml"
 162             )
 163
 164         if not kwargs["symbolsPath"]:
 165             kwargs["symbolsPath"] = os.path.join(self.distdir, "crashreporter-symbols")
 166
 167         if not kwargs["localAPK"]:
 168             for root, _, paths in os.walk(os.path.join(kwargs["objdir"], "gradle")):
 169                 for file_name in paths:
 170                     if file_name.endswith(".apk") and file_name.startswith(
 171                         "test_runner-withGeckoBinaries"
 172                     ):
 173                         kwargs["localAPK"] = os.path.join(root, file_name)
 174                         print("using APK: %s" % kwargs["localAPK"])
 175                         break
 176                 if kwargs["localAPK"]:
 177                     break
 178             else:
 179                 raise Exception("APK not found in objdir. You must specify an APK.")
 180
 181         xpcshell = remotexpcshelltests.XPCShellRemote(kwargs, log)
 182
 183         result = xpcshell.runTests(
 184             kwargs,
 185             testClass=remotexpcshelltests.RemoteXPCShellTestThread,
 186             mobileArgs=xpcshell.mobileArgs,
 187         )
 188
 189         self.log_manager.disable_unstructured()
 190
 191         return int(not result)
 192
 193
 194 def get_parser():
 195     build_obj = MozbuildObject.from_environment(cwd=here)
 196     if conditions.is_android(build_obj):
 197         return parser_remote()
 198     else:
 199         return parser_desktop()
 200
 201
 202 @Command(
 203     "xpcshell-test",
 204     category="testing",
 205     description="Run XPCOM Shell tests (API direct unit testing)",
 206     conditions=[lambda *args: True],
 207     parser=get_parser,
 208 )
 209 def run_xpcshell_test(command_context, test_objects=None, **params):
 210     from mozbuild.controller.building import BuildDriver
 211
 212     if test_objects is not None:
 213         from manifestparser import TestManifest
 214
 215         m = TestManifest()
 216         m.tests.extend(test_objects)
 217         params["manifest"] = m
 218
 219     driver = command_context._spawn(BuildDriver)
 220     driver.install_tests()
 221
 222     # We should probably have a utility function to ensure the tree is
 223     # ready to run tests. Until then, we just create the state dir (in
 224     # case the tree wasn't built with mach).
 225     command_context._ensure_state_subdir_exists(".")
 226
 227     if not params.get("log"):
 228         log_defaults = {
 229             command_context._mach_context.settings["test"]["format"]: sys.stdout
 230         }
 231         fmt_defaults = {
 232             "level": command_context._mach_context.settings["test"]["level"],
 233             "verbose": True,
 234         }
 235         params["log"] = structured.commandline.setup_logging(
 236             "XPCShellTests", params, log_defaults, fmt_defaults
 237         )
 238
 239     if not params["threadCount"]:
 240         # pylint --py3k W1619
 241         params["threadCount"] = int((cpu_count() * 3) / 2)
 242
 243     if conditions.is_android(command_context):
 244         from mozrunner.devices.android_device import (
 245             InstallIntent,
 246             get_adb_path,
 247             verify_android_device,
 248         )
 249
 250         install = InstallIntent.YES if params["setup"] else InstallIntent.NO
 251         device_serial = params.get("deviceSerial")
 252         verify_android_device(
 253             command_context,
 254             network=True,
 255             install=install,
 256             device_serial=device_serial,
 257         )
 258         if not params["adbPath"]:
 259             params["adbPath"] = get_adb_path(command_context)
 260         xpcshell = command_context._spawn(AndroidXPCShellRunner)
 261     else:
 262         xpcshell = command_context._spawn(XPCShellRunner)
 263     xpcshell.cwd = command_context._mach_context.cwd
 264
 265     try:
 266         return xpcshell.run_test(**params)
 267     except InvalidTestPathError as e:
 268         print(str(e))
 269         return 1