testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import os
   8 import sys
   9 from optparse import OptionParser
  10 from os import environ as env
  11
  12 import manifestparser
  13 import mozcrash
  14 import mozfile
  15 import mozinfo
  16 import mozlog
  17 import mozprocess
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
  34     ):
  35         """
  36         Run a single C++ unit test program.
  37
  38         Arguments:
  39         * prog: The path to the test program to run.
  40         * env: The environment to use for running the program.
  41         * symbols_path: A path to a directory containing Breakpad-formatted
  42                         symbol files for producing stack traces on crash.
  43         * timeout_factor: An optional test-specific timeout multiplier.
  44
  45         Return True if the program exits with a zero status, False otherwise.
  46         """
  47         basename = os.path.basename(prog)
  48         self.log.test_start(basename)
  49         with mozfile.TemporaryDirectory() as tempdir:
  50             if interactive:
  51                 # For tests run locally, via mach, print output directly
  52                 proc = mozprocess.ProcessHandler(
  53                     [prog],
  54                     cwd=tempdir,
  55                     env=env,
  56                     storeOutput=False,
  57                     universal_newlines=True,
  58                 )
  59             else:
  60                 proc = mozprocess.ProcessHandler(
  61                     [prog],
  62                     cwd=tempdir,
  63                     env=env,
  64                     storeOutput=True,
  65                     processOutputLine=lambda _: None,
  66                     universal_newlines=True,
  67                 )
  68             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  69             # instead use a timeout in .wait() and then kill to get a stack.
  70             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  71             proc.run(
  72                 timeout=test_timeout,
  73                 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  74             )
  75             proc.wait()
  76             if proc.output:
  77                 if self.fix_stack:
  78                     procOutput = [self.fix_stack(l) for l in proc.output]
  79                 else:
  80                     procOutput = proc.output
  81
  82                 output = "\n%s" % "\n".join(procOutput)
  83                 self.log.process_output(proc.pid, output, command=[prog])
  84             if proc.timedOut:
  85                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  86                 self.log.test_end(
  87                     basename, status="TIMEOUT", expected="PASS", message=message
  88                 )
  89                 return False
  90             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
  91                 self.log.test_end(basename, status="CRASH", expected="PASS")
  92                 return False
  93             result = proc.proc.returncode == 0
  94             if not result:
  95                 self.log.test_end(
  96                     basename,
  97                     status="FAIL",
  98                     expected="PASS",
  99                     message=("test failed with return code %d" % proc.proc.returncode),
 100                 )
 101             else:
 102                 self.log.test_end(basename, status="PASS", expected="PASS")
 103             return result
 104
 105     def build_core_environment(self, env):
 106         """
 107         Add environment variables likely to be used across all platforms, including remote systems.
 108         """
 109         env["MOZ_XRE_DIR"] = self.xre_path
 110         # TODO: switch this to just abort once all C++ unit tests have
 111         # been fixed to enable crash reporting
 112         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 113         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 114         env["MOZ_CRASHREPORTER"] = "1"
 115         return env
 116
 117     def build_environment(self):
 118         """
 119         Create and return a dictionary of all the appropriate env variables and values.
 120         On a remote system, we overload this to set different values and are missing things
 121         like os.environ and PATH.
 122         """
 123         if not os.path.isdir(self.xre_path):
 124             raise Exception("xre_path does not exist: %s", self.xre_path)
 125         env = dict(os.environ)
 126         env = self.build_core_environment(env)
 127         pathvar = ""
 128         libpath = self.xre_path
 129         if mozinfo.os == "linux":
 130             pathvar = "LD_LIBRARY_PATH"
 131         elif mozinfo.os == "mac":
 132             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 133             if os.path.exists(applibpath):
 134                 # Set the library load path to Contents/MacOS if we're run from
 135                 # the app bundle.
 136                 libpath = applibpath
 137             pathvar = "DYLD_LIBRARY_PATH"
 138         elif mozinfo.os == "win":
 139             pathvar = "PATH"
 140         if pathvar:
 141             if pathvar in env:
 142                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 143             else:
 144                 env[pathvar] = libpath
 145
 146         symbolizer_path = None
 147         if mozinfo.info["asan"]:
 148             symbolizer_path = "ASAN_SYMBOLIZER_PATH"
 149         elif mozinfo.info["tsan"]:
 150             symbolizer_path = "TSAN_SYMBOLIZER_PATH"
 151
 152         if symbolizer_path is not None:
 153             # Use llvm-symbolizer for ASan/TSan if available/required
 154             if symbolizer_path in env and os.path.isfile(env[symbolizer_path]):
 155                 llvmsym = env[symbolizer_path]
 156             else:
 157                 llvmsym = os.path.join(
 158                     self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 159                 )
 160             if os.path.isfile(llvmsym):
 161                 env[symbolizer_path] = llvmsym
 162                 self.log.info("Using LLVM symbolizer at %s" % llvmsym)
 163             else:
 164                 self.log.info("Failed to find LLVM symbolizer at %s" % llvmsym)
 165
 166         return env
 167
 168     def run_tests(
 169         self,
 170         programs,
 171         xre_path,
 172         symbols_path=None,
 173         utility_path=None,
 174         interactive=False,
 175     ):
 176         """
 177         Run a set of C++ unit test programs.
 178
 179         Arguments:
 180         * programs: An iterable containing (test path, test timeout factor) tuples
 181         * xre_path: A path to a directory containing a XUL Runtime Environment.
 182         * symbols_path: A path to a directory containing Breakpad-formatted
 183                         symbol files for producing stack traces on crash.
 184         * utility_path: A path to a directory containing utility programs
 185                         (xpcshell et al)
 186
 187         Returns True if all test programs exited with a zero status, False
 188         otherwise.
 189         """
 190         self.xre_path = xre_path
 191         self.log = mozlog.get_default_logger()
 192         if utility_path:
 193             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 194                 utility_path, symbols_path
 195             )
 196         self.log.suite_start(programs, name="cppunittest")
 197         env = self.build_environment()
 198         pass_count = 0
 199         fail_count = 0
 200         for prog in programs:
 201             test_path = prog[0]
 202             timeout_factor = prog[1]
 203             single_result = self.run_one_test(
 204                 test_path, env, symbols_path, interactive, timeout_factor
 205             )
 206             if single_result:
 207                 pass_count += 1
 208             else:
 209                 fail_count += 1
 210         self.log.suite_end()
 211
 212         # Mozharness-parseable summary formatting.
 213         self.log.info("Result summary:")
 214         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 215         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 216         return fail_count == 0
 217
 218
 219 class CPPUnittestOptions(OptionParser):
 220     def __init__(self):
 221         OptionParser.__init__(self)
 222         self.add_option(
 223             "--xre-path",
 224             action="store",
 225             type="string",
 226             dest="xre_path",
 227             default=None,
 228             help="absolute path to directory containing XRE (probably xulrunner)",
 229         )
 230         self.add_option(
 231             "--symbols-path",
 232             action="store",
 233             type="string",
 234             dest="symbols_path",
 235             default=None,
 236             help="absolute path to directory containing breakpad symbols, or "
 237             "the URL of a zip file containing symbols",
 238         )
 239         self.add_option(
 240             "--manifest-path",
 241             action="store",
 242             type="string",
 243             dest="manifest_path",
 244             default=None,
 245             help="path to test manifest, if different from the path to test binaries",
 246         )
 247         self.add_option(
 248             "--utility-path",
 249             action="store",
 250             type="string",
 251             dest="utility_path",
 252             default=None,
 253             help="path to directory containing utility programs",
 254         )
 255
 256
 257 def extract_unittests_from_args(args, environ, manifest_path):
 258     """Extract unittests from args, expanding directories as needed"""
 259     mp = manifestparser.TestManifest(strict=True)
 260     tests = []
 261     binary_path = None
 262
 263     if manifest_path:
 264         mp.read(manifest_path)
 265         binary_path = os.path.abspath(args[0])
 266     else:
 267         for p in args:
 268             if os.path.isdir(p):
 269                 try:
 270                     mp.read(os.path.join(p, "cppunittest.ini"))
 271                 except IOError:
 272                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 273                     tests.extend(
 274                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 275                     )
 276             else:
 277                 tests.append((os.path.abspath(p), 1))
 278
 279     # We don't use the manifest parser's existence-check only because it will
 280     # fail on Windows due to the `.exe` suffix.
 281     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 282     suffix = ".exe" if mozinfo.isWin else ""
 283     if binary_path:
 284         tests.extend(
 285             [
 286                 (
 287                     os.path.join(binary_path, test["relpath"] + suffix),
 288                     int(test.get("requesttimeoutfactor", 1)),
 289                 )
 290                 for test in active_tests
 291             ]
 292         )
 293     else:
 294         tests.extend(
 295             [
 296                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 297                 for test in active_tests
 298             ]
 299         )
 300
 301     # Manually confirm that all tests named in the manifest exist.
 302     errors = False
 303     log = mozlog.get_default_logger()
 304     for test in tests:
 305         if not os.path.isfile(test[0]):
 306             errors = True
 307             log.error("test file not found: %s" % test[0])
 308
 309     if errors:
 310         raise RuntimeError("One or more cppunittests not found; aborting.")
 311
 312     return tests
 313
 314
 315 def update_mozinfo():
 316     """walk up directories to find mozinfo.json update the info"""
 317     path = SCRIPT_DIR
 318     dirs = set()
 319     while path != os.path.expanduser("~"):
 320         if path in dirs:
 321             break
 322         dirs.add(path)
 323         path = os.path.split(path)[0]
 324     mozinfo.find_and_update_from_json(*dirs)
 325
 326
 327 def run_test_harness(options, args):
 328     update_mozinfo()
 329     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 330     options.xre_path = os.path.abspath(options.xre_path)
 331     options.utility_path = os.path.abspath(options.utility_path)
 332     tester = CPPUnitTests()
 333     result = tester.run_tests(
 334         progs,
 335         options.xre_path,
 336         options.symbols_path,
 337         options.utility_path,
 338     )
 339
 340     return result
 341
 342
 343 def main():
 344     parser = CPPUnittestOptions()
 345     mozlog.commandline.add_logging_group(parser)
 346     options, args = parser.parse_args()
 347     if not args:
 348         print(
 349             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 350             file=sys.stderr,
 351         )
 352         sys.exit(1)
 353     if not options.xre_path:
 354         print("""Error: --xre-path is required""", file=sys.stderr)
 355         sys.exit(1)
 356     if options.manifest_path and len(args) > 1:
 357         print(
 358             "Error: multiple arguments not supported with --test-manifest",
 359             file=sys.stderr,
 360         )
 361         sys.exit(1)
 362     log = mozlog.commandline.setup_logging(
 363         "cppunittests", options, {"tbpl": sys.stdout}
 364     )
 365     try:
 366         result = run_test_harness(options, args)
 367     except Exception as e:
 368         log.error(str(e))
 369         result = False
 370
 371     sys.exit(0 if result else 1)
 372
 373
 374 if __name__ == "__main__":
 375     main()