testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(
  33         self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
  34     ):
  35         """
  36         Run a single C++ unit test program.
  37
  38         Arguments:
  39         * prog: The path to the test program to run.
  40         * env: The environment to use for running the program.
  41         * symbols_path: A path to a directory containing Breakpad-formatted
  42                         symbol files for producing stack traces on crash.
  43         * timeout_factor: An optional test-specific timeout multiplier.
  44
  45         Return True if the program exits with a zero status, False otherwise.
  46         """
  47         basename = os.path.basename(prog)
  48         self.log.test_start(basename)
  49         with mozfile.TemporaryDirectory() as tempdir:
  50             if interactive:
  51                 # For tests run locally, via mach, print output directly
  52                 proc = mozprocess.ProcessHandler(
  53                     [prog],
  54                     cwd=tempdir,
  55                     env=env,
  56                     storeOutput=False,
  57                     universal_newlines=True,
  58                 )
  59             else:
  60                 proc = mozprocess.ProcessHandler(
  61                     [prog],
  62                     cwd=tempdir,
  63                     env=env,
  64                     storeOutput=True,
  65                     processOutputLine=lambda _: None,
  66                     universal_newlines=True,
  67                 )
  68             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  69             # instead use a timeout in .wait() and then kill to get a stack.
  70             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  71             proc.run(
  72                 timeout=test_timeout,
  73                 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
  74             )
  75             proc.wait()
  76             if proc.output:
  77                 if self.fix_stack:
  78                     procOutput = [self.fix_stack(l) for l in proc.output]
  79                 else:
  80                     procOutput = proc.output
  81
  82                 output = "\n%s" % "\n".join(procOutput)
  83                 self.log.process_output(proc.pid, output, command=[prog])
  84             if proc.timedOut:
  85                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  86                 self.log.test_end(
  87                     basename, status="TIMEOUT", expected="PASS", message=message
  88                 )
  89                 return False
  90             if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
  91                 self.log.test_end(basename, status="CRASH", expected="PASS")
  92                 return False
  93             result = proc.proc.returncode == 0
  94             if not result:
  95                 self.log.test_end(
  96                     basename,
  97                     status="FAIL",
  98                     expected="PASS",
  99                     message=("test failed with return code %d" % proc.proc.returncode),
 100                 )
 101             else:
 102                 self.log.test_end(basename, status="PASS", expected="PASS")
 103             return result
 104
 105     def build_core_environment(self, env):
 106         """
 107         Add environment variables likely to be used across all platforms, including remote systems.
 108         """
 109         env["MOZ_XRE_DIR"] = self.xre_path
 110         # TODO: switch this to just abort once all C++ unit tests have
 111         # been fixed to enable crash reporting
 112         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 113         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 114         env["MOZ_CRASHREPORTER"] = "1"
 115         return env
 116
 117     def build_environment(self):
 118         """
 119         Create and return a dictionary of all the appropriate env variables and values.
 120         On a remote system, we overload this to set different values and are missing things
 121         like os.environ and PATH.
 122         """
 123         if not os.path.isdir(self.xre_path):
 124             raise Exception("xre_path does not exist: %s", self.xre_path)
 125         env = dict(os.environ)
 126         env = self.build_core_environment(env)
 127         pathvar = ""
 128         libpath = self.xre_path
 129         if mozinfo.os == "linux":
 130             pathvar = "LD_LIBRARY_PATH"
 131         elif mozinfo.os == "mac":
 132             applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
 133             if os.path.exists(applibpath):
 134                 # Set the library load path to Contents/MacOS if we're run from
 135                 # the app bundle.
 136                 libpath = applibpath
 137             pathvar = "DYLD_LIBRARY_PATH"
 138         elif mozinfo.os == "win":
 139             pathvar = "PATH"
 140         if pathvar:
 141             if pathvar in env:
 142                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 143             else:
 144                 env[pathvar] = libpath
 145
 146         if mozinfo.info["asan"]:
 147             # Use llvm-symbolizer for ASan if available/required
 148             llvmsym = os.path.join(
 149                 self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
 150             )
 151             if os.path.isfile(llvmsym):
 152                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 153                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 154             else:
 155                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 156
 157             # dom/media/webrtc/transport tests statically link in NSS, which
 158             # causes ODR violations. See bug 1215679.
 159             assert "ASAN_OPTIONS" not in env
 160             env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0"
 161
 162         return env
 163
 164     def run_tests(
 165         self,
 166         programs,
 167         xre_path,
 168         symbols_path=None,
 169         utility_path=None,
 170         interactive=False,
 171     ):
 172         """
 173         Run a set of C++ unit test programs.
 174
 175         Arguments:
 176         * programs: An iterable containing (test path, test timeout factor) tuples
 177         * xre_path: A path to a directory containing a XUL Runtime Environment.
 178         * symbols_path: A path to a directory containing Breakpad-formatted
 179                         symbol files for producing stack traces on crash.
 180         * utility_path: A path to a directory containing utility programs
 181                         (xpcshell et al)
 182
 183         Returns True if all test programs exited with a zero status, False
 184         otherwise.
 185         """
 186         self.xre_path = xre_path
 187         self.log = mozlog.get_default_logger()
 188         if utility_path:
 189             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 190                 utility_path, symbols_path
 191             )
 192         self.log.suite_start(programs, name="cppunittest")
 193         env = self.build_environment()
 194         pass_count = 0
 195         fail_count = 0
 196         for prog in programs:
 197             test_path = prog[0]
 198             timeout_factor = prog[1]
 199             single_result = self.run_one_test(
 200                 test_path, env, symbols_path, interactive, timeout_factor
 201             )
 202             if single_result:
 203                 pass_count += 1
 204             else:
 205                 fail_count += 1
 206         self.log.suite_end()
 207
 208         # Mozharness-parseable summary formatting.
 209         self.log.info("Result summary:")
 210         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 211         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 212         return fail_count == 0
 213
 214
 215 class CPPUnittestOptions(OptionParser):
 216     def __init__(self):
 217         OptionParser.__init__(self)
 218         self.add_option(
 219             "--xre-path",
 220             action="store",
 221             type="string",
 222             dest="xre_path",
 223             default=None,
 224             help="absolute path to directory containing XRE (probably xulrunner)",
 225         )
 226         self.add_option(
 227             "--symbols-path",
 228             action="store",
 229             type="string",
 230             dest="symbols_path",
 231             default=None,
 232             help="absolute path to directory containing breakpad symbols, or "
 233             "the URL of a zip file containing symbols",
 234         )
 235         self.add_option(
 236             "--manifest-path",
 237             action="store",
 238             type="string",
 239             dest="manifest_path",
 240             default=None,
 241             help="path to test manifest, if different from the path to test binaries",
 242         )
 243         self.add_option(
 244             "--utility-path",
 245             action="store",
 246             type="string",
 247             dest="utility_path",
 248             default=None,
 249             help="path to directory containing utility programs",
 250         )
 251
 252
 253 def extract_unittests_from_args(args, environ, manifest_path):
 254     """Extract unittests from args, expanding directories as needed"""
 255     mp = manifestparser.TestManifest(strict=True)
 256     tests = []
 257     binary_path = None
 258
 259     if manifest_path:
 260         mp.read(manifest_path)
 261         binary_path = os.path.abspath(args[0])
 262     else:
 263         for p in args:
 264             if os.path.isdir(p):
 265                 try:
 266                     mp.read(os.path.join(p, "cppunittest.ini"))
 267                 except IOError:
 268                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 269                     tests.extend(
 270                         (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
 271                     )
 272             else:
 273                 tests.append((os.path.abspath(p), 1))
 274
 275     # we skip the existence check here because not all tests are built
 276     # for all platforms (and it will fail on Windows anyway)
 277     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 278     suffix = ".exe" if mozinfo.isWin else ""
 279     if binary_path:
 280         tests.extend(
 281             [
 282                 (
 283                     os.path.join(binary_path, test["relpath"] + suffix),
 284                     int(test.get("requesttimeoutfactor", 1)),
 285                 )
 286                 for test in active_tests
 287             ]
 288         )
 289     else:
 290         tests.extend(
 291             [
 292                 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
 293                 for test in active_tests
 294             ]
 295         )
 296
 297     # skip and warn for any tests in the manifest that are not found
 298     final_tests = []
 299     log = mozlog.get_default_logger()
 300     for test in tests:
 301         if os.path.isfile(test[0]):
 302             final_tests.append(test)
 303         else:
 304             log.warning("test file not found: %s - skipped" % test[0])
 305
 306     return final_tests
 307
 308
 309 def update_mozinfo():
 310     """walk up directories to find mozinfo.json update the info"""
 311     path = SCRIPT_DIR
 312     dirs = set()
 313     while path != os.path.expanduser("~"):
 314         if path in dirs:
 315             break
 316         dirs.add(path)
 317         path = os.path.split(path)[0]
 318     mozinfo.find_and_update_from_json(*dirs)
 319
 320
 321 def run_test_harness(options, args):
 322     update_mozinfo()
 323     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 324     options.xre_path = os.path.abspath(options.xre_path)
 325     options.utility_path = os.path.abspath(options.utility_path)
 326     tester = CPPUnitTests()
 327     result = tester.run_tests(
 328         progs,
 329         options.xre_path,
 330         options.symbols_path,
 331         options.utility_path,
 332     )
 333
 334     return result
 335
 336
 337 def main():
 338     parser = CPPUnittestOptions()
 339     mozlog.commandline.add_logging_group(parser)
 340     options, args = parser.parse_args()
 341     if not args:
 342         print(
 343             """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
 344             file=sys.stderr,
 345         )
 346         sys.exit(1)
 347     if not options.xre_path:
 348         print("""Error: --xre-path is required""", file=sys.stderr)
 349         sys.exit(1)
 350     if options.manifest_path and len(args) > 1:
 351         print(
 352             "Error: multiple arguments not supported with --test-manifest",
 353             file=sys.stderr,
 354         )
 355         sys.exit(1)
 356     log = mozlog.commandline.setup_logging(
 357         "cppunittests", options, {"tbpl": sys.stdout}
 358     )
 359     try:
 360         result = run_test_harness(options, args)
 361     except Exception as e:
 362         log.error(str(e))
 363         result = False
 364
 365     sys.exit(0 if result else 1)
 366
 367
 368 if __name__ == "__main__":
 369     main()