testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import os
   8 import sys
   9 from optparse import OptionParser
  10 from os import environ as env
  11
  12 import manifestparser
  13 import mozcrash
  14 import mozfile
  15 import mozinfo
  16 import mozlog
  17 import mozprocess
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self,
  34         prog,
  35         env,
  36         symbols_path=None,
  37         utility_path=None,
  38         interactive=False,
  39         timeout_factor=1,
  40     ):
  41         """
  42         Run a single C++ unit test program.
  43
  44         Arguments:
  45         * prog: The path to the test program to run.
  46         * env: The environment to use for running the program.
  47         * symbols_path: A path to a directory containing Breakpad-formatted
  48                         symbol files for producing stack traces on crash.
  49         * timeout_factor: An optional test-specific timeout multiplier.
  50
  51         Return True if the program exits with a zero status, False otherwise.
  52         """
  53         CPPUnitTests.run_one_test.timed_out = False
  54         output = []
  55
  56         def timeout_handler(proc):
  57             CPPUnitTests.run_one_test.timed_out = True
  58             message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  59             self.log.test_end(
  60                 basename, status="TIMEOUT", expected="PASS", message=message
  61             )
  62             mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path)
  63
  64         def output_timeout_handler(proc):
  65             CPPUnitTests.run_one_test.timed_out = True
  66             message = (
  67                 "timed out after %d seconds without output"
  68                 % CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT
  69             )
  70             self.log.test_end(
  71                 basename, status="TIMEOUT", expected="PASS", message=message
  72             )
  73             mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path)
  74
  75         def output_line_handler(_, line):
  76             fixed_line = self.fix_stack(line) if self.fix_stack else line
  77             if interactive:
  78                 print(fixed_line)
  79             else:
  80                 output.append(fixed_line)
  81
  82         basename = os.path.basename(prog)
  83         self.log.test_start(basename)
  84         with mozfile.TemporaryDirectory() as tempdir:
  85             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  86             proc = mozprocess.run_and_wait(
  87                 [prog],
  88                 cwd=tempdir,
  89                 env=env,
  90                 output_line_handler=output_line_handler,
  91                 timeout=test_timeout,
  92                 timeout_handler=timeout_handler,
  93                 output_timeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  94                 output_timeout_handler=output_timeout_handler,
  95             )
  96
  97             if output:
  98                 output = "\n%s" % "\n".join(output)
  99                 self.log.process_output(proc.pid, output, command=[prog])
 100             if CPPUnitTests.run_one_test.timed_out:
 101                 return False
 102             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
 103                 self.log.test_end(basename, status="CRASH", expected="PASS")
 104                 return False
 105             result = proc.returncode == 0
 106             if not result:
 107                 self.log.test_end(
 108                     basename,
 109                     status="FAIL",
 110                     expected="PASS",
 111                     message=("test failed with return code %d" % proc.returncode),
 112                 )
 113             else:
 114                 self.log.test_end(basename, status="PASS", expected="PASS")
 115             return result
 116
 117     def build_core_environment(self, env):
 118         """
 119         Add environment variables likely to be used across all platforms, including remote systems.
 120         """
 121         env["MOZ_XRE_DIR"] = self.xre_path
 122         # TODO: switch this to just abort once all C++ unit tests have
 123         # been fixed to enable crash reporting
 124         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 125         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 126         env["MOZ_CRASHREPORTER"] = "1"
 127         return env
 128
 129     def build_environment(self):
 130         """
 131         Create and return a dictionary of all the appropriate env variables and values.
 132         On a remote system, we overload this to set different values and are missing things
 133         like os.environ and PATH.
 134         """
 135         if not os.path.isdir(self.xre_path):
 136             raise Exception("xre_path does not exist: %s", self.xre_path)
 137         env = dict(os.environ)
 138         env = self.build_core_environment(env)
 139         pathvar = ""
 140         libpath = self.xre_path
 141         if mozinfo.os == "linux":
 142             pathvar = "LD_LIBRARY_PATH"
 143         elif mozinfo.os == "mac":
 144             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 145             if os.path.exists(applibpath):
 146                 # Set the library load path to Contents/MacOS if we're run from
 147                 # the app bundle.
 148                 libpath = applibpath
 149             pathvar = "DYLD_LIBRARY_PATH"
 150         elif mozinfo.os == "win":
 151             pathvar = "PATH"
 152         if pathvar:
 153             if pathvar in env:
 154                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 155             else:
 156                 env[pathvar] = libpath
 157
 158         symbolizer_path = None
 159         if mozinfo.info["asan"]:
 160             symbolizer_path = "ASAN_SYMBOLIZER_PATH"
 161         elif mozinfo.info["tsan"]:
 162             symbolizer_path = "TSAN_SYMBOLIZER_PATH"
 163
 164         if symbolizer_path is not None:
 165             # Use llvm-symbolizer for ASan/TSan if available/required
 166             if symbolizer_path in env and os.path.isfile(env[symbolizer_path]):
 167                 llvmsym = env[symbolizer_path]
 168             else:
 169                 llvmsym = os.path.join(
 170                     self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 171                 )
 172             if os.path.isfile(llvmsym):
 173                 env[symbolizer_path] = llvmsym
 174                 self.log.info("Using LLVM symbolizer at %s" % llvmsym)
 175             else:
 176                 self.log.info("Failed to find LLVM symbolizer at %s" % llvmsym)
 177
 178         return env
 179
 180     def run_tests(
 181         self,
 182         programs,
 183         xre_path,
 184         symbols_path=None,
 185         utility_path=None,
 186         interactive=False,
 187     ):
 188         """
 189         Run a set of C++ unit test programs.
 190
 191         Arguments:
 192         * programs: An iterable containing (test path, test timeout factor) tuples
 193         * xre_path: A path to a directory containing a XUL Runtime Environment.
 194         * symbols_path: A path to a directory containing Breakpad-formatted
 195                         symbol files for producing stack traces on crash.
 196         * utility_path: A path to a directory containing utility programs
 197                         (xpcshell et al)
 198
 199         Returns True if all test programs exited with a zero status, False
 200         otherwise.
 201         """
 202         self.xre_path = xre_path
 203         self.log = mozlog.get_default_logger()
 204         if utility_path:
 205             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 206                 utility_path, symbols_path
 207             )
 208         self.log.suite_start(programs, name="cppunittest")
 209         env = self.build_environment()
 210         pass_count = 0
 211         fail_count = 0
 212         for prog in programs:
 213             test_path = prog[0]
 214             timeout_factor = prog[1]
 215             single_result = self.run_one_test(
 216                 test_path, env, symbols_path, utility_path, interactive, timeout_factor
 217             )
 218             if single_result:
 219                 pass_count += 1
 220             else:
 221                 fail_count += 1
 222         self.log.suite_end()
 223
 224         # Mozharness-parseable summary formatting.
 225         self.log.info("Result summary:")
 226         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 227         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 228         return fail_count == 0
 229
 230
 231 class CPPUnittestOptions(OptionParser):
 232     def __init__(self):
 233         OptionParser.__init__(self)
 234         self.add_option(
 235             "--xre-path",
 236             action="store",
 237             type="string",
 238             dest="xre_path",
 239             default=None,
 240             help="absolute path to directory containing XRE (probably xulrunner)",
 241         )
 242         self.add_option(
 243             "--symbols-path",
 244             action="store",
 245             type="string",
 246             dest="symbols_path",
 247             default=None,
 248             help="absolute path to directory containing breakpad symbols, or "
 249             "the URL of a zip file containing symbols",
 250         )
 251         self.add_option(
 252             "--manifest-path",
 253             action="store",
 254             type="string",
 255             dest="manifest_path",
 256             default=None,
 257             help="path to test manifest, if different from the path to test binaries",
 258         )
 259         self.add_option(
 260             "--utility-path",
 261             action="store",
 262             type="string",
 263             dest="utility_path",
 264             default=None,
 265             help="path to directory containing utility programs",
 266         )
 267
 268
 269 def extract_unittests_from_args(args, environ, manifest_path):
 270     """Extract unittests from args, expanding directories as needed"""
 271     mp = manifestparser.TestManifest(strict=True)
 272     tests = []
 273     binary_path = None
 274
 275     if manifest_path:
 276         mp.read(manifest_path)
 277         binary_path = os.path.abspath(args[0])
 278     else:
 279         for p in args:
 280             if os.path.isdir(p):
 281                 try:
 282                     mp.read(os.path.join(p, "cppunittest.ini"))
 283                 except IOError:
 284                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 285                     tests.extend(
 286                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 287                     )
 288             else:
 289                 tests.append((os.path.abspath(p), 1))
 290
 291     # We don't use the manifest parser's existence-check only because it will
 292     # fail on Windows due to the `.exe` suffix.
 293     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 294     suffix = ".exe" if mozinfo.isWin else ""
 295     if binary_path:
 296         tests.extend(
 297             [
 298                 (
 299                     os.path.join(binary_path, test["relpath"] + suffix),
 300                     int(test.get("requesttimeoutfactor", 1)),
 301                 )
 302                 for test in active_tests
 303             ]
 304         )
 305     else:
 306         tests.extend(
 307             [
 308                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 309                 for test in active_tests
 310             ]
 311         )
 312
 313     # Manually confirm that all tests named in the manifest exist.
 314     errors = False
 315     log = mozlog.get_default_logger()
 316     for test in tests:
 317         if not os.path.isfile(test[0]):
 318             errors = True
 319             log.error("test file not found: %s" % test[0])
 320
 321     if errors:
 322         raise RuntimeError("One or more cppunittests not found; aborting.")
 323
 324     return tests
 325
 326
 327 def update_mozinfo():
 328     """walk up directories to find mozinfo.json update the info"""
 329     path = SCRIPT_DIR
 330     dirs = set()
 331     while path != os.path.expanduser("~"):
 332         if path in dirs:
 333             break
 334         dirs.add(path)
 335         path = os.path.split(path)[0]
 336     mozinfo.find_and_update_from_json(*dirs)
 337
 338
 339 def run_test_harness(options, args):
 340     update_mozinfo()
 341     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 342     options.xre_path = os.path.abspath(options.xre_path)
 343     options.utility_path = os.path.abspath(options.utility_path)
 344     tester = CPPUnitTests()
 345     result = tester.run_tests(
 346         progs,
 347         options.xre_path,
 348         options.symbols_path,
 349         options.utility_path,
 350     )
 351
 352     return result
 353
 354
 355 def main():
 356     parser = CPPUnittestOptions()
 357     mozlog.commandline.add_logging_group(parser)
 358     options, args = parser.parse_args()
 359     if not args:
 360         print(
 361             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 362             file=sys.stderr,
 363         )
 364         sys.exit(1)
 365     if not options.xre_path:
 366         print("""Error: --xre-path is required""", file=sys.stderr)
 367         sys.exit(1)
 368     if options.manifest_path and len(args) > 1:
 369         print(
 370             "Error: multiple arguments not supported with --test-manifest",
 371             file=sys.stderr,
 372         )
 373         sys.exit(1)
 374     log = mozlog.commandline.setup_logging(
 375         "cppunittests", options, {"tbpl": sys.stdout}
 376     )
 377     try:
 378         result = run_test_harness(options, args)
 379     except Exception as e:
 380         log.error(str(e))
 381         result = False
 382
 383     sys.exit(0 if result else 1)
 384
 385
 386 if __name__ == "__main__":
 387     main()