testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env['CPP_UNIT_TESTS_DIR_JS_SRC'] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(self, prog, env, symbols_path=None, interactive=False,
  33                      timeout_factor=1):
  34         """
  35         Run a single C++ unit test program.
  36
  37         Arguments:
  38         * prog: The path to the test program to run.
  39         * env: The environment to use for running the program.
  40         * symbols_path: A path to a directory containing Breakpad-formatted
  41                         symbol files for producing stack traces on crash.
  42         * timeout_factor: An optional test-specific timeout multiplier.
  43
  44         Return True if the program exits with a zero status, False otherwise.
  45         """
  46         basename = os.path.basename(prog)
  47         self.log.test_start(basename)
  48         with mozfile.TemporaryDirectory() as tempdir:
  49             if interactive:
  50                 # For tests run locally, via mach, print output directly
  51                 proc = mozprocess.ProcessHandler([prog],
  52                                                  cwd=tempdir,
  53                                                  env=env,
  54                                                  storeOutput=False)
  55             else:
  56                 proc = mozprocess.ProcessHandler([prog],
  57                                                  cwd=tempdir,
  58                                                  env=env,
  59                                                  storeOutput=True,
  60                                                  processOutputLine=lambda _: None)
  61             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  62             # instead use a timeout in .wait() and then kill to get a stack.
  63             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  64             proc.run(timeout=test_timeout,
  65                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
  66             proc.wait()
  67             if proc.output:
  68                 if self.fix_stack:
  69                     procOutput = [self.fix_stack(l) for l in proc.output]
  70                 else:
  71                     procOutput = proc.output
  72
  73                 output = "\n%s" % "\n".join(procOutput)
  74                 self.log.process_output(proc.pid, output, command=[prog])
  75             if proc.timedOut:
  76                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  77                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
  78                                   message=message)
  79                 return False
  80             if mozcrash.check_for_crashes(tempdir, symbols_path,
  81                                           test_name=basename):
  82                 self.log.test_end(basename, status='CRASH', expected='PASS')
  83                 return False
  84             result = proc.proc.returncode == 0
  85             if not result:
  86                 self.log.test_end(basename, status='FAIL', expected='PASS',
  87                                   message=("test failed with return code %d" %
  88                                            proc.proc.returncode))
  89             else:
  90                 self.log.test_end(basename, status='PASS', expected='PASS')
  91             return result
  92
  93     def build_core_environment(self, env, enable_webrender):
  94         """
  95         Add environment variables likely to be used across all platforms, including remote systems.
  96         """
  97         env["MOZ_XRE_DIR"] = self.xre_path
  98         # TODO: switch this to just abort once all C++ unit tests have
  99         # been fixed to enable crash reporting
 100         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 101         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 102         env["MOZ_CRASHREPORTER"] = "1"
 103
 104         if enable_webrender:
 105             env['MOZ_WEBRENDER'] = '1'
 106             env['MOZ_ACCELERATED'] = '1'
 107         else:
 108             env['MOZ_WEBRENDER'] = '0'
 109
 110         return env
 111
 112     def build_environment(self, enable_webrender=False):
 113         """
 114         Create and return a dictionary of all the appropriate env variables and values.
 115         On a remote system, we overload this to set different values and are missing things
 116         like os.environ and PATH.
 117         """
 118         if not os.path.isdir(self.xre_path):
 119             raise Exception("xre_path does not exist: %s", self.xre_path)
 120         env = dict(os.environ)
 121         env = self.build_core_environment(env, enable_webrender)
 122         pathvar = ""
 123         libpath = self.xre_path
 124         if mozinfo.os == "linux":
 125             pathvar = "LD_LIBRARY_PATH"
 126         elif mozinfo.os == "mac":
 127             applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
 128             if os.path.exists(applibpath):
 129                 # Set the library load path to Contents/MacOS if we're run from
 130                 # the app bundle.
 131                 libpath = applibpath
 132             pathvar = "DYLD_LIBRARY_PATH"
 133         elif mozinfo.os == "win":
 134             pathvar = "PATH"
 135         if pathvar:
 136             if pathvar in env:
 137                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 138             else:
 139                 env[pathvar] = libpath
 140
 141         if mozinfo.info["asan"]:
 142             # Use llvm-symbolizer for ASan if available/required
 143             llvmsym = os.path.join(
 144                 self.xre_path,
 145                 "llvm-symbolizer" + mozinfo.info["bin_suffix"].encode('ascii'))
 146             if os.path.isfile(llvmsym):
 147                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 148                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 149             else:
 150                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 151
 152             # media/mtransport tests statically link in NSS, which
 153             # causes ODR violations. See bug 1215679.
 154             assert 'ASAN_OPTIONS' not in env
 155             env['ASAN_OPTIONS'] = 'detect_leaks=0:detect_odr_violation=0'
 156
 157         return env
 158
 159     def run_tests(self, programs, xre_path, symbols_path=None,
 160                   utility_path=None, enable_webrender=False, interactive=False):
 161         """
 162         Run a set of C++ unit test programs.
 163
 164         Arguments:
 165         * programs: An iterable containing (test path, test timeout factor) tuples
 166         * xre_path: A path to a directory containing a XUL Runtime Environment.
 167         * symbols_path: A path to a directory containing Breakpad-formatted
 168                         symbol files for producing stack traces on crash.
 169         * utility_path: A path to a directory containing utility programs
 170                         (xpcshell et al)
 171
 172         Returns True if all test programs exited with a zero status, False
 173         otherwise.
 174         """
 175         self.xre_path = xre_path
 176         self.log = mozlog.get_default_logger()
 177         if utility_path:
 178             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 179                 utility_path, symbols_path)
 180         self.log.suite_start(programs, name='cppunittest')
 181         env = self.build_environment(enable_webrender)
 182         pass_count = 0
 183         fail_count = 0
 184         for prog in programs:
 185             test_path = prog[0]
 186             timeout_factor = prog[1]
 187             single_result = self.run_one_test(test_path, env, symbols_path,
 188                                               interactive, timeout_factor)
 189             if single_result:
 190                 pass_count += 1
 191             else:
 192                 fail_count += 1
 193         self.log.suite_end()
 194
 195         # Mozharness-parseable summary formatting.
 196         self.log.info("Result summary:")
 197         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 198         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 199         return fail_count == 0
 200
 201
 202 class CPPUnittestOptions(OptionParser):
 203     def __init__(self):
 204         OptionParser.__init__(self)
 205         self.add_option("--xre-path",
 206                         action="store", type="string", dest="xre_path",
 207                         default=None,
 208                         help="absolute path to directory containing XRE (probably xulrunner)")
 209         self.add_option("--symbols-path",
 210                         action="store", type="string", dest="symbols_path",
 211                         default=None,
 212                         help="absolute path to directory containing breakpad symbols, or "
 213                         "the URL of a zip file containing symbols")
 214         self.add_option("--manifest-path",
 215                         action="store", type="string", dest="manifest_path",
 216                         default=None,
 217                         help="path to test manifest, if different from the path to test binaries")
 218         self.add_option("--utility-path",
 219                         action="store", type="string", dest="utility_path",
 220                         default=None,
 221                         help="path to directory containing utility programs")
 222         self.add_option("--enable-webrender",
 223                         action="store_true", dest="enable_webrender",
 224                         default=False,
 225                         help="Enable the WebRender compositor in Gecko")
 226
 227
 228 def extract_unittests_from_args(args, environ, manifest_path):
 229     """Extract unittests from args, expanding directories as needed"""
 230     mp = manifestparser.TestManifest(strict=True)
 231     tests = []
 232     binary_path = None
 233
 234     if manifest_path:
 235         mp.read(manifest_path)
 236         binary_path = os.path.abspath(args[0])
 237     else:
 238         for p in args:
 239             if os.path.isdir(p):
 240                 try:
 241                     mp.read(os.path.join(p, 'cppunittest.ini'))
 242                 except IOError:
 243                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 244                     tests.extend((f, 1) for f in files
 245                                  if os.access(f, os.R_OK | os.X_OK))
 246             else:
 247                 tests.append((os.path.abspath(p), 1))
 248
 249     # we skip the existence check here because not all tests are built
 250     # for all platforms (and it will fail on Windows anyway)
 251     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 252     suffix = '.exe' if mozinfo.isWin else ''
 253     if binary_path:
 254         tests.extend([
 255             (os.path.join(binary_path, test['relpath'] + suffix),
 256              int(test.get('requesttimeoutfactor', 1)))
 257             for test in active_tests])
 258     else:
 259         tests.extend([
 260             (test['path'] + suffix,
 261              int(test.get('requesttimeoutfactor', 1)))
 262             for test in active_tests
 263         ])
 264
 265     # skip non-existing tests
 266     tests = [test for test in tests if os.path.isfile(test[0])]
 267
 268     return tests
 269
 270
 271 def update_mozinfo():
 272     """walk up directories to find mozinfo.json update the info"""
 273     path = SCRIPT_DIR
 274     dirs = set()
 275     while path != os.path.expanduser('~'):
 276         if path in dirs:
 277             break
 278         dirs.add(path)
 279         path = os.path.split(path)[0]
 280     mozinfo.find_and_update_from_json(*dirs)
 281
 282
 283 def run_test_harness(options, args):
 284     update_mozinfo()
 285     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 286     options.xre_path = os.path.abspath(options.xre_path)
 287     options.utility_path = os.path.abspath(options.utility_path)
 288     tester = CPPUnitTests()
 289     result = tester.run_tests(progs, options.xre_path, options.symbols_path,
 290                               options.utility_path, options.enable_webrender)
 291
 292     return result
 293
 294
 295 def main():
 296     parser = CPPUnittestOptions()
 297     mozlog.commandline.add_logging_group(parser)
 298     options, args = parser.parse_args()
 299     if not args:
 300         print("""Usage: %s <test binary> [<test binary>...]""" % sys.argv[0], file=sys.stderr)
 301         sys.exit(1)
 302     if not options.xre_path:
 303         print("""Error: --xre-path is required""", file=sys.stderr)
 304         sys.exit(1)
 305     if options.manifest_path and len(args) > 1:
 306         print("Error: multiple arguments not supported with --test-manifest", file=sys.stderr)
 307         sys.exit(1)
 308     log = mozlog.commandline.setup_logging("cppunittests", options,
 309                                            {"tbpl": sys.stdout})
 310     try:
 311         result = run_test_harness(options, args)
 312     except Exception as e:
 313         log.error(str(e))
 314         result = False
 315
 316     sys.exit(0 if result else 1)
 317
 318
 319 if __name__ == '__main__':
 320     main()