testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
  34     ):
  35         """
  36         Run a single C++ unit test program.
  37
  38         Arguments:
  39         * prog: The path to the test program to run.
  40         * env: The environment to use for running the program.
  41         * symbols_path: A path to a directory containing Breakpad-formatted
  42                         symbol files for producing stack traces on crash.
  43         * timeout_factor: An optional test-specific timeout multiplier.
  44
  45         Return True if the program exits with a zero status, False otherwise.
  46         """
  47         basename = os.path.basename(prog)
  48         self.log.test_start(basename)
  49         with mozfile.TemporaryDirectory() as tempdir:
  50             if interactive:
  51                 # For tests run locally, via mach, print output directly
  52                 proc = mozprocess.ProcessHandler(
  53                     [prog],
  54                     cwd=tempdir,
  55                     env=env,
  56                     storeOutput=False,
  57                     universal_newlines=True,
  58                 )
  59             else:
  60                 proc = mozprocess.ProcessHandler(
  61                     [prog],
  62                     cwd=tempdir,
  63                     env=env,
  64                     storeOutput=True,
  65                     processOutputLine=lambda _: None,
  66                     universal_newlines=True,
  67                 )
  68             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  69             # instead use a timeout in .wait() and then kill to get a stack.
  70             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  71             proc.run(
  72                 timeout=test_timeout,
  73                 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  74             )
  75             proc.wait()
  76             if proc.output:
  77                 if self.fix_stack:
  78                     procOutput = [self.fix_stack(l) for l in proc.output]
  79                 else:
  80                     procOutput = proc.output
  81
  82                 output = "\n%s" % "\n".join(procOutput)
  83                 self.log.process_output(proc.pid, output, command=[prog])
  84             if proc.timedOut:
  85                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  86                 self.log.test_end(
  87                     basename, status="TIMEOUT", expected="PASS", message=message
  88                 )
  89                 return False
  90             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
  91                 self.log.test_end(basename, status="CRASH", expected="PASS")
  92                 return False
  93             result = proc.proc.returncode == 0
  94             if not result:
  95                 self.log.test_end(
  96                     basename,
  97                     status="FAIL",
  98                     expected="PASS",
  99                     message=("test failed with return code %d" % proc.proc.returncode),
 100                 )
 101             else:
 102                 self.log.test_end(basename, status="PASS", expected="PASS")
 103             return result
 104
 105     def build_core_environment(self, env, enable_webrender):
 106         """
 107         Add environment variables likely to be used across all platforms, including remote systems.
 108         """
 109         env["MOZ_XRE_DIR"] = self.xre_path
 110         # TODO: switch this to just abort once all C++ unit tests have
 111         # been fixed to enable crash reporting
 112         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 113         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 114         env["MOZ_CRASHREPORTER"] = "1"
 115
 116         if enable_webrender:
 117             env["MOZ_WEBRENDER"] = "1"
 118             env["MOZ_ACCELERATED"] = "1"
 119         else:
 120             env["MOZ_WEBRENDER"] = "0"
 121
 122         return env
 123
 124     def build_environment(self, enable_webrender=False):
 125         """
 126         Create and return a dictionary of all the appropriate env variables and values.
 127         On a remote system, we overload this to set different values and are missing things
 128         like os.environ and PATH.
 129         """
 130         if not os.path.isdir(self.xre_path):
 131             raise Exception("xre_path does not exist: %s", self.xre_path)
 132         env = dict(os.environ)
 133         env = self.build_core_environment(env, enable_webrender)
 134         pathvar = ""
 135         libpath = self.xre_path
 136         if mozinfo.os == "linux":
 137             pathvar = "LD_LIBRARY_PATH"
 138         elif mozinfo.os == "mac":
 139             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 140             if os.path.exists(applibpath):
 141                 # Set the library load path to Contents/MacOS if we're run from
 142                 # the app bundle.
 143                 libpath = applibpath
 144             pathvar = "DYLD_LIBRARY_PATH"
 145         elif mozinfo.os == "win":
 146             pathvar = "PATH"
 147         if pathvar:
 148             if pathvar in env:
 149                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 150             else:
 151                 env[pathvar] = libpath
 152
 153         if mozinfo.info["asan"]:
 154             # Use llvm-symbolizer for ASan if available/required
 155             llvmsym = os.path.join(
 156                 self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 157             )
 158             if os.path.isfile(llvmsym):
 159                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 160                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 161             else:
 162                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 163
 164             # dom/media/webrtc/transport tests statically link in NSS, which
 165             # causes ODR violations. See bug 1215679.
 166             assert "ASAN_OPTIONS" not in env
 167             env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0"
 168
 169         return env
 170
 171     def run_tests(
 172         self,
 173         programs,
 174         xre_path,
 175         symbols_path=None,
 176         utility_path=None,
 177         enable_webrender=False,
 178         interactive=False,
 179     ):
 180         """
 181         Run a set of C++ unit test programs.
 182
 183         Arguments:
 184         * programs: An iterable containing (test path, test timeout factor) tuples
 185         * xre_path: A path to a directory containing a XUL Runtime Environment.
 186         * symbols_path: A path to a directory containing Breakpad-formatted
 187                         symbol files for producing stack traces on crash.
 188         * utility_path: A path to a directory containing utility programs
 189                         (xpcshell et al)
 190
 191         Returns True if all test programs exited with a zero status, False
 192         otherwise.
 193         """
 194         self.xre_path = xre_path
 195         self.log = mozlog.get_default_logger()
 196         if utility_path:
 197             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 198                 utility_path, symbols_path
 199             )
 200         self.log.suite_start(programs, name="cppunittest")
 201         env = self.build_environment(enable_webrender)
 202         pass_count = 0
 203         fail_count = 0
 204         for prog in programs:
 205             test_path = prog[0]
 206             timeout_factor = prog[1]
 207             single_result = self.run_one_test(
 208                 test_path, env, symbols_path, interactive, timeout_factor
 209             )
 210             if single_result:
 211                 pass_count += 1
 212             else:
 213                 fail_count += 1
 214         self.log.suite_end()
 215
 216         # Mozharness-parseable summary formatting.
 217         self.log.info("Result summary:")
 218         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 219         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 220         return fail_count == 0
 221
 222
 223 class CPPUnittestOptions(OptionParser):
 224     def __init__(self):
 225         OptionParser.__init__(self)
 226         self.add_option(
 227             "--xre-path",
 228             action="store",
 229             type="string",
 230             dest="xre_path",
 231             default=None,
 232             help="absolute path to directory containing XRE (probably xulrunner)",
 233         )
 234         self.add_option(
 235             "--symbols-path",
 236             action="store",
 237             type="string",
 238             dest="symbols_path",
 239             default=None,
 240             help="absolute path to directory containing breakpad symbols, or "
 241             "the URL of a zip file containing symbols",
 242         )
 243         self.add_option(
 244             "--manifest-path",
 245             action="store",
 246             type="string",
 247             dest="manifest_path",
 248             default=None,
 249             help="path to test manifest, if different from the path to test binaries",
 250         )
 251         self.add_option(
 252             "--utility-path",
 253             action="store",
 254             type="string",
 255             dest="utility_path",
 256             default=None,
 257             help="path to directory containing utility programs",
 258         )
 259         self.add_option(
 260             "--enable-webrender",
 261             action="store_true",
 262             dest="enable_webrender",
 263             default=False,
 264             help="Enable the WebRender compositor in Gecko",
 265         )
 266
 267
 268 def extract_unittests_from_args(args, environ, manifest_path):
 269     """Extract unittests from args, expanding directories as needed"""
 270     mp = manifestparser.TestManifest(strict=True)
 271     tests = []
 272     binary_path = None
 273
 274     if manifest_path:
 275         mp.read(manifest_path)
 276         binary_path = os.path.abspath(args[0])
 277     else:
 278         for p in args:
 279             if os.path.isdir(p):
 280                 try:
 281                     mp.read(os.path.join(p, "cppunittest.ini"))
 282                 except IOError:
 283                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 284                     tests.extend(
 285                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 286                     )
 287             else:
 288                 tests.append((os.path.abspath(p), 1))
 289
 290     # we skip the existence check here because not all tests are built
 291     # for all platforms (and it will fail on Windows anyway)
 292     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 293     suffix = ".exe" if mozinfo.isWin else ""
 294     if binary_path:
 295         tests.extend(
 296             [
 297                 (
 298                     os.path.join(binary_path, test["relpath"] + suffix),
 299                     int(test.get("requesttimeoutfactor", 1)),
 300                 )
 301                 for test in active_tests
 302             ]
 303         )
 304     else:
 305         tests.extend(
 306             [
 307                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 308                 for test in active_tests
 309             ]
 310         )
 311
 312     # skip and warn for any tests in the manifest that are not found
 313     final_tests = []
 314     log = mozlog.get_default_logger()
 315     for test in tests:
 316         if os.path.isfile(test[0]):
 317             final_tests.append(test)
 318         else:
 319             log.warning("test file not found: %s - skipped" % test[0])
 320
 321     return final_tests
 322
 323
 324 def update_mozinfo():
 325     """walk up directories to find mozinfo.json update the info"""
 326     path = SCRIPT_DIR
 327     dirs = set()
 328     while path != os.path.expanduser("~"):
 329         if path in dirs:
 330             break
 331         dirs.add(path)
 332         path = os.path.split(path)[0]
 333     mozinfo.find_and_update_from_json(*dirs)
 334
 335
 336 def run_test_harness(options, args):
 337     update_mozinfo()
 338     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 339     options.xre_path = os.path.abspath(options.xre_path)
 340     options.utility_path = os.path.abspath(options.utility_path)
 341     tester = CPPUnitTests()
 342     result = tester.run_tests(
 343         progs,
 344         options.xre_path,
 345         options.symbols_path,
 346         options.utility_path,
 347         options.enable_webrender,
 348     )
 349
 350     return result
 351
 352
 353 def main():
 354     parser = CPPUnittestOptions()
 355     mozlog.commandline.add_logging_group(parser)
 356     options, args = parser.parse_args()
 357     if not args:
 358         print(
 359             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 360             file=sys.stderr,
 361         )
 362         sys.exit(1)
 363     if not options.xre_path:
 364         print("""Error: --xre-path is required""", file=sys.stderr)
 365         sys.exit(1)
 366     if options.manifest_path and len(args) > 1:
 367         print(
 368             "Error: multiple arguments not supported with --test-manifest",
 369             file=sys.stderr,
 370         )
 371         sys.exit(1)
 372     log = mozlog.commandline.setup_logging(
 373         "cppunittests", options, {"tbpl": sys.stdout}
 374     )
 375     try:
 376         result = run_test_harness(options, args)
 377     except Exception as e:
 378         log.error(str(e))
 379         result = False
 380
 381     sys.exit(0 if result else 1)
 382
 383
 384 if __name__ == "__main__":
 385     main()