testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
  34     ):
  35         """
  36         Run a single C++ unit test program.
  37
  38         Arguments:
  39         * prog: The path to the test program to run.
  40         * env: The environment to use for running the program.
  41         * symbols_path: A path to a directory containing Breakpad-formatted
  42                         symbol files for producing stack traces on crash.
  43         * timeout_factor: An optional test-specific timeout multiplier.
  44
  45         Return True if the program exits with a zero status, False otherwise.
  46         """
  47         basename = os.path.basename(prog)
  48         self.log.test_start(basename)
  49         with mozfile.TemporaryDirectory() as tempdir:
  50             if interactive:
  51                 # For tests run locally, via mach, print output directly
  52                 proc = mozprocess.ProcessHandler(
  53                     [prog],
  54                     cwd=tempdir,
  55                     env=env,
  56                     storeOutput=False,
  57                     universal_newlines=True,
  58                 )
  59             else:
  60                 proc = mozprocess.ProcessHandler(
  61                     [prog],
  62                     cwd=tempdir,
  63                     env=env,
  64                     storeOutput=True,
  65                     processOutputLine=lambda _: None,
  66                     universal_newlines=True,
  67                 )
  68             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  69             # instead use a timeout in .wait() and then kill to get a stack.
  70             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  71             proc.run(
  72                 timeout=test_timeout,
  73                 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  74             )
  75             proc.wait()
  76             if proc.output:
  77                 if self.fix_stack:
  78                     procOutput = [self.fix_stack(l) for l in proc.output]
  79                 else:
  80                     procOutput = proc.output
  81
  82                 output = "\n%s" % "\n".join(procOutput)
  83                 self.log.process_output(proc.pid, output, command=[prog])
  84             if proc.timedOut:
  85                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  86                 self.log.test_end(
  87                     basename, status="TIMEOUT", expected="PASS", message=message
  88                 )
  89                 return False
  90             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
  91                 self.log.test_end(basename, status="CRASH", expected="PASS")
  92                 return False
  93             result = proc.proc.returncode == 0
  94             if not result:
  95                 self.log.test_end(
  96                     basename,
  97                     status="FAIL",
  98                     expected="PASS",
  99                     message=("test failed with return code %d" % proc.proc.returncode),
 100                 )
 101             else:
 102                 self.log.test_end(basename, status="PASS", expected="PASS")
 103             return result
 104
 105     def build_core_environment(self, env):
 106         """
 107         Add environment variables likely to be used across all platforms, including remote systems.
 108         """
 109         env["MOZ_XRE_DIR"] = self.xre_path
 110         # TODO: switch this to just abort once all C++ unit tests have
 111         # been fixed to enable crash reporting
 112         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 113         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 114         env["MOZ_CRASHREPORTER"] = "1"
 115         return env
 116
 117     def build_environment(self):
 118         """
 119         Create and return a dictionary of all the appropriate env variables and values.
 120         On a remote system, we overload this to set different values and are missing things
 121         like os.environ and PATH.
 122         """
 123         if not os.path.isdir(self.xre_path):
 124             raise Exception("xre_path does not exist: %s", self.xre_path)
 125         env = dict(os.environ)
 126         env = self.build_core_environment(env)
 127         pathvar = ""
 128         libpath = self.xre_path
 129         if mozinfo.os == "linux":
 130             pathvar = "LD_LIBRARY_PATH"
 131         elif mozinfo.os == "mac":
 132             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 133             if os.path.exists(applibpath):
 134                 # Set the library load path to Contents/MacOS if we're run from
 135                 # the app bundle.
 136                 libpath = applibpath
 137             pathvar = "DYLD_LIBRARY_PATH"
 138         elif mozinfo.os == "win":
 139             pathvar = "PATH"
 140         if pathvar:
 141             if pathvar in env:
 142                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 143             else:
 144                 env[pathvar] = libpath
 145
 146         if mozinfo.info["asan"]:
 147             # Use llvm-symbolizer for ASan if available/required
 148             if "ASAN_SYMBOLIZER_PATH" in env and os.path.isfile(
 149                 env["ASAN_SYMBOLIZER_PATH"]
 150             ):
 151                 llvmsym = env["ASAN_SYMBOLIZER_PATH"]
 152             else:
 153                 llvmsym = os.path.join(
 154                     self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 155                 )
 156             if os.path.isfile(llvmsym):
 157                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 158                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 159             else:
 160                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 161
 162             # dom/media/webrtc/transport tests statically link in NSS, which
 163             # causes ODR violations. See bug 1215679.
 164             assert "ASAN_OPTIONS" not in env
 165             env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0"
 166
 167         return env
 168
 169     def run_tests(
 170         self,
 171         programs,
 172         xre_path,
 173         symbols_path=None,
 174         utility_path=None,
 175         interactive=False,
 176     ):
 177         """
 178         Run a set of C++ unit test programs.
 179
 180         Arguments:
 181         * programs: An iterable containing (test path, test timeout factor) tuples
 182         * xre_path: A path to a directory containing a XUL Runtime Environment.
 183         * symbols_path: A path to a directory containing Breakpad-formatted
 184                         symbol files for producing stack traces on crash.
 185         * utility_path: A path to a directory containing utility programs
 186                         (xpcshell et al)
 187
 188         Returns True if all test programs exited with a zero status, False
 189         otherwise.
 190         """
 191         self.xre_path = xre_path
 192         self.log = mozlog.get_default_logger()
 193         if utility_path:
 194             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 195                 utility_path, symbols_path
 196             )
 197         self.log.suite_start(programs, name="cppunittest")
 198         env = self.build_environment()
 199         pass_count = 0
 200         fail_count = 0
 201         for prog in programs:
 202             test_path = prog[0]
 203             timeout_factor = prog[1]
 204             single_result = self.run_one_test(
 205                 test_path, env, symbols_path, interactive, timeout_factor
 206             )
 207             if single_result:
 208                 pass_count += 1
 209             else:
 210                 fail_count += 1
 211         self.log.suite_end()
 212
 213         # Mozharness-parseable summary formatting.
 214         self.log.info("Result summary:")
 215         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 216         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 217         return fail_count == 0
 218
 219
 220 class CPPUnittestOptions(OptionParser):
 221     def __init__(self):
 222         OptionParser.__init__(self)
 223         self.add_option(
 224             "--xre-path",
 225             action="store",
 226             type="string",
 227             dest="xre_path",
 228             default=None,
 229             help="absolute path to directory containing XRE (probably xulrunner)",
 230         )
 231         self.add_option(
 232             "--symbols-path",
 233             action="store",
 234             type="string",
 235             dest="symbols_path",
 236             default=None,
 237             help="absolute path to directory containing breakpad symbols, or "
 238             "the URL of a zip file containing symbols",
 239         )
 240         self.add_option(
 241             "--manifest-path",
 242             action="store",
 243             type="string",
 244             dest="manifest_path",
 245             default=None,
 246             help="path to test manifest, if different from the path to test binaries",
 247         )
 248         self.add_option(
 249             "--utility-path",
 250             action="store",
 251             type="string",
 252             dest="utility_path",
 253             default=None,
 254             help="path to directory containing utility programs",
 255         )
 256
 257
 258 def extract_unittests_from_args(args, environ, manifest_path):
 259     """Extract unittests from args, expanding directories as needed"""
 260     mp = manifestparser.TestManifest(strict=True)
 261     tests = []
 262     binary_path = None
 263
 264     if manifest_path:
 265         mp.read(manifest_path)
 266         binary_path = os.path.abspath(args[0])
 267     else:
 268         for p in args:
 269             if os.path.isdir(p):
 270                 try:
 271                     mp.read(os.path.join(p, "cppunittest.ini"))
 272                 except IOError:
 273                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 274                     tests.extend(
 275                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 276                     )
 277             else:
 278                 tests.append((os.path.abspath(p), 1))
 279
 280     # We don't use the manifest parser's existence-check only because it will
 281     # fail on Windows due to the `.exe` suffix.
 282     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 283     suffix = ".exe" if mozinfo.isWin else ""
 284     if binary_path:
 285         tests.extend(
 286             [
 287                 (
 288                     os.path.join(binary_path, test["relpath"] + suffix),
 289                     int(test.get("requesttimeoutfactor", 1)),
 290                 )
 291                 for test in active_tests
 292             ]
 293         )
 294     else:
 295         tests.extend(
 296             [
 297                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 298                 for test in active_tests
 299             ]
 300         )
 301
 302     # Manually confirm that all tests named in the manifest exist.
 303     errors = False
 304     log = mozlog.get_default_logger()
 305     for test in tests:
 306         if not os.path.isfile(test[0]):
 307             errors = True
 308             log.error("test file not found: %s" % test[0])
 309
 310     if errors:
 311         raise RuntimeError("One or more cppunittests not found; aborting.")
 312
 313     return tests
 314
 315
 316 def update_mozinfo():
 317     """walk up directories to find mozinfo.json update the info"""
 318     path = SCRIPT_DIR
 319     dirs = set()
 320     while path != os.path.expanduser("~"):
 321         if path in dirs:
 322             break
 323         dirs.add(path)
 324         path = os.path.split(path)[0]
 325     mozinfo.find_and_update_from_json(*dirs)
 326
 327
 328 def run_test_harness(options, args):
 329     update_mozinfo()
 330     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 331     options.xre_path = os.path.abspath(options.xre_path)
 332     options.utility_path = os.path.abspath(options.utility_path)
 333     tester = CPPUnitTests()
 334     result = tester.run_tests(
 335         progs,
 336         options.xre_path,
 337         options.symbols_path,
 338         options.utility_path,
 339     )
 340
 341     return result
 342
 343
 344 def main():
 345     parser = CPPUnittestOptions()
 346     mozlog.commandline.add_logging_group(parser)
 347     options, args = parser.parse_args()
 348     if not args:
 349         print(
 350             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 351             file=sys.stderr,
 352         )
 353         sys.exit(1)
 354     if not options.xre_path:
 355         print("""Error: --xre-path is required""", file=sys.stderr)
 356         sys.exit(1)
 357     if options.manifest_path and len(args) > 1:
 358         print(
 359             "Error: multiple arguments not supported with --test-manifest",
 360             file=sys.stderr,
 361         )
 362         sys.exit(1)
 363     log = mozlog.commandline.setup_logging(
 364         "cppunittests", options, {"tbpl": sys.stdout}
 365     )
 366     try:
 367         result = run_test_harness(options, args)
 368     except Exception as e:
 369         log.error(str(e))
 370         result = False
 371
 372     sys.exit(0 if result else 1)
 373
 374
 375 if __name__ == "__main__":
 376     main()