testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
  34     ):
  35         """
  36         Run a single C++ unit test program.
  37
  38         Arguments:
  39         * prog: The path to the test program to run.
  40         * env: The environment to use for running the program.
  41         * symbols_path: A path to a directory containing Breakpad-formatted
  42                         symbol files for producing stack traces on crash.
  43         * timeout_factor: An optional test-specific timeout multiplier.
  44
  45         Return True if the program exits with a zero status, False otherwise.
  46         """
  47         basename = os.path.basename(prog)
  48         self.log.test_start(basename)
  49         with mozfile.TemporaryDirectory() as tempdir:
  50             if interactive:
  51                 # For tests run locally, via mach, print output directly
  52                 proc = mozprocess.ProcessHandler(
  53                     [prog],
  54                     cwd=tempdir,
  55                     env=env,
  56                     storeOutput=False,
  57                     universal_newlines=True,
  58                 )
  59             else:
  60                 proc = mozprocess.ProcessHandler(
  61                     [prog],
  62                     cwd=tempdir,
  63                     env=env,
  64                     storeOutput=True,
  65                     processOutputLine=lambda _: None,
  66                     universal_newlines=True,
  67                 )
  68             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  69             # instead use a timeout in .wait() and then kill to get a stack.
  70             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  71             proc.run(
  72                 timeout=test_timeout,
  73                 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  74             )
  75             proc.wait()
  76             if proc.output:
  77                 if self.fix_stack:
  78                     procOutput = [self.fix_stack(l) for l in proc.output]
  79                 else:
  80                     procOutput = proc.output
  81
  82                 output = "\n%s" % "\n".join(procOutput)
  83                 self.log.process_output(proc.pid, output, command=[prog])
  84             if proc.timedOut:
  85                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  86                 self.log.test_end(
  87                     basename, status="TIMEOUT", expected="PASS", message=message
  88                 )
  89                 return False
  90             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
  91                 self.log.test_end(basename, status="CRASH", expected="PASS")
  92                 return False
  93             result = proc.proc.returncode == 0
  94             if not result:
  95                 self.log.test_end(
  96                     basename,
  97                     status="FAIL",
  98                     expected="PASS",
  99                     message=("test failed with return code %d" % proc.proc.returncode),
 100                 )
 101             else:
 102                 self.log.test_end(basename, status="PASS", expected="PASS")
 103             return result
 104
 105     def build_core_environment(self, env):
 106         """
 107         Add environment variables likely to be used across all platforms, including remote systems.
 108         """
 109         env["MOZ_XRE_DIR"] = self.xre_path
 110         # TODO: switch this to just abort once all C++ unit tests have
 111         # been fixed to enable crash reporting
 112         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 113         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 114         env["MOZ_CRASHREPORTER"] = "1"
 115         return env
 116
 117     def build_environment(self):
 118         """
 119         Create and return a dictionary of all the appropriate env variables and values.
 120         On a remote system, we overload this to set different values and are missing things
 121         like os.environ and PATH.
 122         """
 123         if not os.path.isdir(self.xre_path):
 124             raise Exception("xre_path does not exist: %s", self.xre_path)
 125         env = dict(os.environ)
 126         env = self.build_core_environment(env)
 127         pathvar = ""
 128         libpath = self.xre_path
 129         if mozinfo.os == "linux":
 130             pathvar = "LD_LIBRARY_PATH"
 131         elif mozinfo.os == "mac":
 132             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 133             if os.path.exists(applibpath):
 134                 # Set the library load path to Contents/MacOS if we're run from
 135                 # the app bundle.
 136                 libpath = applibpath
 137             pathvar = "DYLD_LIBRARY_PATH"
 138         elif mozinfo.os == "win":
 139             pathvar = "PATH"
 140         if pathvar:
 141             if pathvar in env:
 142                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 143             else:
 144                 env[pathvar] = libpath
 145
 146         if mozinfo.info["asan"]:
 147             # Use llvm-symbolizer for ASan if available/required
 148             llvmsym = os.path.join(
 149                 self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 150             )
 151             if os.path.isfile(llvmsym):
 152                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 153                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 154             else:
 155                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 156
 157             # dom/media/webrtc/transport tests statically link in NSS, which
 158             # causes ODR violations. See bug 1215679.
 159             assert "ASAN_OPTIONS" not in env
 160             env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0"
 161
 162         return env
 163
 164     def run_tests(
 165         self,
 166         programs,
 167         xre_path,
 168         symbols_path=None,
 169         utility_path=None,
 170         interactive=False,
 171     ):
 172         """
 173         Run a set of C++ unit test programs.
 174
 175         Arguments:
 176         * programs: An iterable containing (test path, test timeout factor) tuples
 177         * xre_path: A path to a directory containing a XUL Runtime Environment.
 178         * symbols_path: A path to a directory containing Breakpad-formatted
 179                         symbol files for producing stack traces on crash.
 180         * utility_path: A path to a directory containing utility programs
 181                         (xpcshell et al)
 182
 183         Returns True if all test programs exited with a zero status, False
 184         otherwise.
 185         """
 186         self.xre_path = xre_path
 187         self.log = mozlog.get_default_logger()
 188         if utility_path:
 189             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 190                 utility_path, symbols_path
 191             )
 192         self.log.suite_start(programs, name="cppunittest")
 193         env = self.build_environment()
 194         pass_count = 0
 195         fail_count = 0
 196         for prog in programs:
 197             test_path = prog[0]
 198             timeout_factor = prog[1]
 199             single_result = self.run_one_test(
 200                 test_path, env, symbols_path, interactive, timeout_factor
 201             )
 202             if single_result:
 203                 pass_count += 1
 204             else:
 205                 fail_count += 1
 206         self.log.suite_end()
 207
 208         # Mozharness-parseable summary formatting.
 209         self.log.info("Result summary:")
 210         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 211         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 212         return fail_count == 0
 213
 214
 215 class CPPUnittestOptions(OptionParser):
 216     def __init__(self):
 217         OptionParser.__init__(self)
 218         self.add_option(
 219             "--xre-path",
 220             action="store",
 221             type="string",
 222             dest="xre_path",
 223             default=None,
 224             help="absolute path to directory containing XRE (probably xulrunner)",
 225         )
 226         self.add_option(
 227             "--symbols-path",
 228             action="store",
 229             type="string",
 230             dest="symbols_path",
 231             default=None,
 232             help="absolute path to directory containing breakpad symbols, or "
 233             "the URL of a zip file containing symbols",
 234         )
 235         self.add_option(
 236             "--manifest-path",
 237             action="store",
 238             type="string",
 239             dest="manifest_path",
 240             default=None,
 241             help="path to test manifest, if different from the path to test binaries",
 242         )
 243         self.add_option(
 244             "--utility-path",
 245             action="store",
 246             type="string",
 247             dest="utility_path",
 248             default=None,
 249             help="path to directory containing utility programs",
 250         )
 251
 252
 253 def extract_unittests_from_args(args, environ, manifest_path):
 254     """Extract unittests from args, expanding directories as needed"""
 255     mp = manifestparser.TestManifest(strict=True)
 256     tests = []
 257     binary_path = None
 258
 259     if manifest_path:
 260         mp.read(manifest_path)
 261         binary_path = os.path.abspath(args[0])
 262     else:
 263         for p in args:
 264             if os.path.isdir(p):
 265                 try:
 266                     mp.read(os.path.join(p, "cppunittest.ini"))
 267                 except IOError:
 268                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 269                     tests.extend(
 270                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 271                     )
 272             else:
 273                 tests.append((os.path.abspath(p), 1))
 274
 275     # We don't use the manifest parser's existence-check only because it will
 276     # fail on Windows due to the `.exe` suffix.
 277     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 278     suffix = ".exe" if mozinfo.isWin else ""
 279     if binary_path:
 280         tests.extend(
 281             [
 282                 (
 283                     os.path.join(binary_path, test["relpath"] + suffix),
 284                     int(test.get("requesttimeoutfactor", 1)),
 285                 )
 286                 for test in active_tests
 287             ]
 288         )
 289     else:
 290         tests.extend(
 291             [
 292                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 293                 for test in active_tests
 294             ]
 295         )
 296
 297     # Manually confirm that all tests named in the manifest exist.
 298     errors = False
 299     log = mozlog.get_default_logger()
 300     for test in tests:
 301         if not os.path.isfile(test[0]):
 302             errors = True
 303             log.error("test file not found: %s" % test[0])
 304
 305     if errors:
 306         raise RuntimeError("One or more cppunittests not found; aborting.")
 307
 308     return tests
 309
 310
 311 def update_mozinfo():
 312     """walk up directories to find mozinfo.json update the info"""
 313     path = SCRIPT_DIR
 314     dirs = set()
 315     while path != os.path.expanduser("~"):
 316         if path in dirs:
 317             break
 318         dirs.add(path)
 319         path = os.path.split(path)[0]
 320     mozinfo.find_and_update_from_json(*dirs)
 321
 322
 323 def run_test_harness(options, args):
 324     update_mozinfo()
 325     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 326     options.xre_path = os.path.abspath(options.xre_path)
 327     options.utility_path = os.path.abspath(options.utility_path)
 328     tester = CPPUnitTests()
 329     result = tester.run_tests(
 330         progs,
 331         options.xre_path,
 332         options.symbols_path,
 333         options.utility_path,
 334     )
 335
 336     return result
 337
 338
 339 def main():
 340     parser = CPPUnittestOptions()
 341     mozlog.commandline.add_logging_group(parser)
 342     options, args = parser.parse_args()
 343     if not args:
 344         print(
 345             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 346             file=sys.stderr,
 347         )
 348         sys.exit(1)
 349     if not options.xre_path:
 350         print("""Error: --xre-path is required""", file=sys.stderr)
 351         sys.exit(1)
 352     if options.manifest_path and len(args) > 1:
 353         print(
 354             "Error: multiple arguments not supported with --test-manifest",
 355             file=sys.stderr,
 356         )
 357         sys.exit(1)
 358     log = mozlog.commandline.setup_logging(
 359         "cppunittests", options, {"tbpl": sys.stdout}
 360     )
 361     try:
 362         result = run_test_harness(options, args)
 363     except Exception as e:
 364         log.error(str(e))
 365         result = False
 366
 367     sys.exit(0 if result else 1)
 368
 369
 370 if __name__ == "__main__":
 371     main()