testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18
  19 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  20
  21 # Export directory js/src for tests that need it.
  22 env['CPP_UNIT_TESTS_DIR_JS_SRC'] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  23
  24
  25 class CPPUnitTests(object):
  26     # Time (seconds) to wait for test process to complete
  27     TEST_PROC_TIMEOUT = 900
  28     # Time (seconds) in which process will be killed if it produces no output.
  29     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  30
  31     def run_one_test(self, prog, env, symbols_path=None, interactive=False,
  32                      timeout_factor=1):
  33         """
  34         Run a single C++ unit test program.
  35
  36         Arguments:
  37         * prog: The path to the test program to run.
  38         * env: The environment to use for running the program.
  39         * symbols_path: A path to a directory containing Breakpad-formatted
  40                         symbol files for producing stack traces on crash.
  41         * timeout_factor: An optional test-specific timeout multiplier.
  42
  43         Return True if the program exits with a zero status, False otherwise.
  44         """
  45         basename = os.path.basename(prog)
  46         self.log.test_start(basename)
  47         with mozfile.TemporaryDirectory() as tempdir:
  48             if interactive:
  49                 # For tests run locally, via mach, print output directly
  50                 proc = mozprocess.ProcessHandler([prog],
  51                                                  cwd=tempdir,
  52                                                  env=env,
  53                                                  storeOutput=False)
  54             else:
  55                 proc = mozprocess.ProcessHandler([prog],
  56                                                  cwd=tempdir,
  57                                                  env=env,
  58                                                  storeOutput=True,
  59                                                  processOutputLine=lambda _: None)
  60             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  61             # instead use a timeout in .wait() and then kill to get a stack.
  62             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  63             proc.run(timeout=test_timeout,
  64                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
  65             proc.wait()
  66             if proc.output:
  67                 output = "\n%s" % "\n".join(proc.output)
  68                 self.log.process_output(proc.pid, output, command=[prog])
  69             if proc.timedOut:
  70                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  71                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
  72                                   message=message)
  73                 return False
  74             if mozcrash.check_for_crashes(tempdir, symbols_path,
  75                                           test_name=basename):
  76                 self.log.test_end(basename, status='CRASH', expected='PASS')
  77                 return False
  78             result = proc.proc.returncode == 0
  79             if not result:
  80                 self.log.test_end(basename, status='FAIL', expected='PASS',
  81                                   message=("test failed with return code %d" %
  82                                            proc.proc.returncode))
  83             else:
  84                 self.log.test_end(basename, status='PASS', expected='PASS')
  85             return result
  86
  87     def build_core_environment(self, env={}):
  88         """
  89         Add environment variables likely to be used across all platforms, including remote systems.
  90         """
  91         env["MOZ_XRE_DIR"] = self.xre_path
  92         # TODO: switch this to just abort once all C++ unit tests have
  93         # been fixed to enable crash reporting
  94         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
  95         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
  96         env["MOZ_CRASHREPORTER"] = "1"
  97         return env
  98
  99     def build_environment(self):
 100         """
 101         Create and return a dictionary of all the appropriate env variables and values.
 102         On a remote system, we overload this to set different values and are missing things
 103         like os.environ and PATH.
 104         """
 105         if not os.path.isdir(self.xre_path):
 106             raise Exception("xre_path does not exist: %s", self.xre_path)
 107         env = dict(os.environ)
 108         env = self.build_core_environment(env)
 109         pathvar = ""
 110         libpath = self.xre_path
 111         if mozinfo.os == "linux":
 112             pathvar = "LD_LIBRARY_PATH"
 113         elif mozinfo.os == "mac":
 114             applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
 115             if os.path.exists(applibpath):
 116                 # Set the library load path to Contents/MacOS if we're run from
 117                 # the app bundle.
 118                 libpath = applibpath
 119             pathvar = "DYLD_LIBRARY_PATH"
 120         elif mozinfo.os == "win":
 121             pathvar = "PATH"
 122         if pathvar:
 123             if pathvar in env:
 124                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 125             else:
 126                 env[pathvar] = libpath
 127
 128         if mozinfo.info["asan"]:
 129             # Use llvm-symbolizer for ASan if available/required
 130             llvmsym = os.path.join(
 131                 self.xre_path,
 132                 "llvm-symbolizer" + mozinfo.info["bin_suffix"].encode('ascii'))
 133             if os.path.isfile(llvmsym):
 134                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 135                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 136             else:
 137                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 138
 139             # media/mtransport tests statically link in NSS, which
 140             # causes ODR violations. See bug 1215679.
 141             assert 'ASAN_OPTIONS' not in env
 142             env['ASAN_OPTIONS'] = 'detect_leaks=0:detect_odr_violation=0'
 143
 144         return env
 145
 146     def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
 147         """
 148         Run a set of C++ unit test programs.
 149
 150         Arguments:
 151         * programs: An iterable containing (test path, test timeout factor) tuples
 152         * xre_path: A path to a directory containing a XUL Runtime Environment.
 153         * symbols_path: A path to a directory containing Breakpad-formatted
 154                         symbol files for producing stack traces on crash.
 155
 156         Returns True if all test programs exited with a zero status, False
 157         otherwise.
 158         """
 159         self.xre_path = xre_path
 160         self.log = mozlog.get_default_logger()
 161         self.log.suite_start(programs, name='cppunittest')
 162         env = self.build_environment()
 163         pass_count = 0
 164         fail_count = 0
 165         for prog in programs:
 166             test_path = prog[0]
 167             timeout_factor = prog[1]
 168             single_result = self.run_one_test(test_path, env, symbols_path,
 169                                               interactive, timeout_factor)
 170             if single_result:
 171                 pass_count += 1
 172             else:
 173                 fail_count += 1
 174         self.log.suite_end()
 175
 176         # Mozharness-parseable summary formatting.
 177         self.log.info("Result summary:")
 178         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 179         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 180         return fail_count == 0
 181
 182
 183 class CPPUnittestOptions(OptionParser):
 184     def __init__(self):
 185         OptionParser.__init__(self)
 186         self.add_option("--xre-path",
 187                         action="store", type="string", dest="xre_path",
 188                         default=None,
 189                         help="absolute path to directory containing XRE (probably xulrunner)")
 190         self.add_option("--symbols-path",
 191                         action="store", type="string", dest="symbols_path",
 192                         default=None,
 193                         help="absolute path to directory containing breakpad symbols, or "
 194                         "the URL of a zip file containing symbols")
 195         self.add_option("--manifest-path",
 196                         action="store", type="string", dest="manifest_path",
 197                         default=None,
 198                         help="path to test manifest, if different from the path to test binaries")
 199
 200
 201 def extract_unittests_from_args(args, environ, manifest_path):
 202     """Extract unittests from args, expanding directories as needed"""
 203     mp = manifestparser.TestManifest(strict=True)
 204     tests = []
 205     binary_path = None
 206
 207     if manifest_path:
 208         mp.read(manifest_path)
 209         binary_path = os.path.abspath(args[0])
 210     else:
 211         for p in args:
 212             if os.path.isdir(p):
 213                 try:
 214                     mp.read(os.path.join(p, 'cppunittest.ini'))
 215                 except IOError:
 216                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 217                     tests.extend((f, 1) for f in files
 218                                  if os.access(f, os.R_OK | os.X_OK))
 219             else:
 220                 tests.append((os.path.abspath(p), 1))
 221
 222     # we skip the existence check here because not all tests are built
 223     # for all platforms (and it will fail on Windows anyway)
 224     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 225     suffix = '.exe' if mozinfo.isWin else ''
 226     if binary_path:
 227         tests.extend([
 228             (os.path.join(binary_path, test['relpath'] + suffix),
 229              int(test.get('requesttimeoutfactor', 1)))
 230             for test in active_tests])
 231     else:
 232         tests.extend([
 233             (test['path'] + suffix,
 234              int(test.get('requesttimeoutfactor', 1)))
 235             for test in active_tests
 236         ])
 237
 238     # skip non-existing tests
 239     tests = [test for test in tests if os.path.isfile(test[0])]
 240
 241     return tests
 242
 243
 244 def update_mozinfo():
 245     """walk up directories to find mozinfo.json update the info"""
 246     path = SCRIPT_DIR
 247     dirs = set()
 248     while path != os.path.expanduser('~'):
 249         if path in dirs:
 250             break
 251         dirs.add(path)
 252         path = os.path.split(path)[0]
 253     mozinfo.find_and_update_from_json(*dirs)
 254
 255
 256 def run_test_harness(options, args):
 257     update_mozinfo()
 258     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 259     options.xre_path = os.path.abspath(options.xre_path)
 260     tester = CPPUnitTests()
 261     result = tester.run_tests(progs, options.xre_path, options.symbols_path)
 262
 263     return result
 264
 265
 266 def main():
 267     parser = CPPUnittestOptions()
 268     mozlog.commandline.add_logging_group(parser)
 269     options, args = parser.parse_args()
 270     if not args:
 271         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
 272         sys.exit(1)
 273     if not options.xre_path:
 274         print >>sys.stderr, """Error: --xre-path is required"""
 275         sys.exit(1)
 276     if options.manifest_path and len(args) > 1:
 277         print >>sys.stderr, "Error: multiple arguments not supported with --test-manifest"
 278         sys.exit(1)
 279     log = mozlog.commandline.setup_logging("cppunittests", options,
 280                                            {"tbpl": sys.stdout})
 281     try:
 282         result = run_test_harness(options, args)
 283     except Exception as e:
 284         log.error(str(e))
 285         result = False
 286
 287     sys.exit(0 if result else 1)
 288
 289
 290 if __name__ == '__main__':
 291     main()