testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import with_statement
   8 import sys, os, tempfile, shutil
   9 from optparse import OptionParser
  10 import mozprocess
  11 import mozinfo
  12 import mozcrash
  13 import mozfile
  14 from contextlib import contextmanager
  15 from mozlog import structured
  16 from subprocess import PIPE
  17
  18 class CPPUnitTests(object):
  19     # Time (seconds) to wait for test process to complete
  20     TEST_PROC_TIMEOUT = 900
  21     # Time (seconds) in which process will be killed if it produces no output.
  22     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  23
  24     def run_one_test(self, prog, env, symbols_path=None, interactive=False):
  25         """
  26         Run a single C++ unit test program.
  27
  28         Arguments:
  29         * prog: The path to the test program to run.
  30         * env: The environment to use for running the program.
  31         * symbols_path: A path to a directory containing Breakpad-formatted
  32                         symbol files for producing stack traces on crash.
  33
  34         Return True if the program exits with a zero status, False otherwise.
  35         """
  36         basename = os.path.basename(prog)
  37         self.log.test_start(basename)
  38         with mozfile.TemporaryDirectory() as tempdir:
  39             if interactive:
  40                 # For tests run locally, via mach, print output directly
  41                 proc = mozprocess.ProcessHandler([prog],
  42                                                  cwd=tempdir,
  43                                                  env=env,
  44                                                  storeOutput=False)
  45             else:
  46                 proc = mozprocess.ProcessHandler([prog],
  47                                                  cwd=tempdir,
  48                                                  env=env,
  49                                                  storeOutput=True,
  50                                                  processOutputLine=lambda _: None)
  51             #TODO: After bug 811320 is fixed, don't let .run() kill the process,
  52             # instead use a timeout in .wait() and then kill to get a stack.
  53             proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT,
  54                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
  55             proc.wait()
  56             if proc.output:
  57                 output = "\n%s" % "\n".join(proc.output)
  58                 self.log.process_output(proc.pid, output, command=[prog])
  59             if proc.timedOut:
  60                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  61                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
  62                                   message=message)
  63                 return False
  64             if mozcrash.check_for_crashes(tempdir, symbols_path,
  65                                           test_name=basename):
  66                 self.log.test_end(basename, status='CRASH', expected='PASS')
  67                 return False
  68             result = proc.proc.returncode == 0
  69             if not result:
  70                 self.log.test_end(basename, status='FAIL', expected='PASS',
  71                                   message=("test failed with return code %d" %
  72                                            proc.proc.returncode))
  73             else:
  74                 self.log.test_end(basename, status='PASS', expected='PASS')
  75             return result
  76
  77     def build_core_environment(self, env = {}):
  78         """
  79         Add environment variables likely to be used across all platforms, including remote systems.
  80         """
  81         env["MOZILLA_FIVE_HOME"] = self.xre_path
  82         env["MOZ_XRE_DIR"] = self.xre_path
  83         #TODO: switch this to just abort once all C++ unit tests have
  84         # been fixed to enable crash reporting
  85         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
  86         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
  87         env["MOZ_CRASHREPORTER"] = "1"
  88         return env
  89
  90     def build_environment(self):
  91         """
  92         Create and return a dictionary of all the appropriate env variables and values.
  93         On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
  94         """
  95         if not os.path.isdir(self.xre_path):
  96             raise Exception("xre_path does not exist: %s", self.xre_path)
  97         env = dict(os.environ)
  98         env = self.build_core_environment(env)
  99         pathvar = ""
 100         libpath = self.xre_path
 101         if mozinfo.os == "linux":
 102             pathvar = "LD_LIBRARY_PATH"
 103         elif mozinfo.os == "mac":
 104             applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
 105             if os.path.exists(applibpath):
 106                 # Set the library load path to Contents/MacOS if we're run from
 107                 # the app bundle.
 108                 libpath = applibpath
 109             pathvar = "DYLD_LIBRARY_PATH"
 110         elif mozinfo.os == "win":
 111             pathvar = "PATH"
 112         if pathvar:
 113             if pathvar in env:
 114                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 115             else:
 116                 env[pathvar] = libpath
 117
 118         # Use llvm-symbolizer for ASan if available/required
 119         llvmsym = os.path.join(self.xre_path, "llvm-symbolizer")
 120         if os.path.isfile(llvmsym):
 121             env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 122             self.log.info("ASan using symbolizer at %s" % llvmsym)
 123         else:
 124             self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 125
 126         return env
 127
 128     def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
 129         """
 130         Run a set of C++ unit test programs.
 131
 132         Arguments:
 133         * programs: An iterable containing paths to test programs.
 134         * xre_path: A path to a directory containing a XUL Runtime Environment.
 135         * symbols_path: A path to a directory containing Breakpad-formatted
 136                         symbol files for producing stack traces on crash.
 137
 138         Returns True if all test programs exited with a zero status, False
 139         otherwise.
 140         """
 141         self.xre_path = xre_path
 142         self.log = structured.structuredlog.get_default_logger()
 143         self.log.suite_start(programs)
 144         env = self.build_environment()
 145         pass_count = 0
 146         fail_count = 0
 147         for prog in programs:
 148             single_result = self.run_one_test(prog, env, symbols_path, interactive)
 149             if single_result:
 150                 pass_count += 1
 151             else:
 152                 fail_count += 1
 153         self.log.suite_end()
 154
 155         # Mozharness-parseable summary formatting.
 156         self.log.info("Result summary:")
 157         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 158         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 159         return fail_count == 0
 160
 161 class CPPUnittestOptions(OptionParser):
 162     def __init__(self):
 163         OptionParser.__init__(self)
 164         self.add_option("--xre-path",
 165                         action = "store", type = "string", dest = "xre_path",
 166                         default = None,
 167                         help = "absolute path to directory containing XRE (probably xulrunner)")
 168         self.add_option("--symbols-path",
 169                         action = "store", type = "string", dest = "symbols_path",
 170                         default = None,
 171                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
 172         self.add_option("--skip-manifest",
 173                         action = "store", type = "string", dest = "manifest_file",
 174                         default = None,
 175                         help = "absolute path to a manifest file")
 176
 177 def extract_unittests_from_args(args, manifest_file):
 178     """Extract unittests from args, expanding directories as needed"""
 179     progs = []
 180
 181     # Known files commonly packaged with the cppunittests that are not tests
 182     skipped_progs = set(['.mkdir.done', 'remotecppunittests.py', 'runcppunittests.py', 'runcppunittests.pyc'])
 183
 184     if manifest_file:
 185         skipped_progs.add(os.path.basename(manifest_file))
 186         with open(manifest_file) as f:
 187             for line in f:
 188                 # strip out comment, if any
 189                 prog = line.split('#')[0]
 190                 if prog:
 191                     skipped_progs.add(prog.strip())
 192
 193     for p in args:
 194         if os.path.isdir(p):
 195             progs.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p) if not x in skipped_progs])
 196         elif p not in skipped_progs:
 197             progs.append(os.path.abspath(p))
 198
 199     return progs
 200
 201 def main():
 202     parser = CPPUnittestOptions()
 203     structured.commandline.add_logging_group(parser)
 204     options, args = parser.parse_args()
 205     if not args:
 206         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
 207         sys.exit(1)
 208     if not options.xre_path:
 209         print >>sys.stderr, """Error: --xre-path is required"""
 210         sys.exit(1)
 211
 212     log = structured.commandline.setup_logging("cppunittests",
 213                                                options,
 214                                                {"tbpl": sys.stdout})
 215
 216     progs = extract_unittests_from_args(args, options.manifest_file)
 217     options.xre_path = os.path.abspath(options.xre_path)
 218     if mozinfo.isMac:
 219         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
 220     tester = CPPUnitTests()
 221
 222     try:
 223         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
 224     except Exception as e:
 225         log.error(str(e))
 226         result = False
 227
 228     sys.exit(0 if result else 1)
 229
 230 if __name__ == '__main__':
 231     main()