testing/runcppunittests.py

   1 #!/usr/bin/env python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, with_statement
   8 import sys
   9 import os
  10 from optparse import OptionParser
  11 from os import environ as env
  12 import manifestparser
  13 import mozprocess
  14 import mozinfo
  15 import mozcrash
  16 import mozfile
  17 import mozlog
  18 import mozrunner.utils
  19
  20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  21
  22 # Export directory js/src for tests that need it.
  23 env['CPP_UNIT_TESTS_DIR_JS_SRC'] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
  24
  25
  26 class CPPUnitTests(object):
  27     # Time (seconds) to wait for test process to complete
  28     TEST_PROC_TIMEOUT = 900
  29     # Time (seconds) in which process will be killed if it produces no output.
  30     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
  31
  32     def run_one_test(self, prog, env, symbols_path=None, interactive=False,
  33                      timeout_factor=1):
  34         """
  35         Run a single C++ unit test program.
  36
  37         Arguments:
  38         * prog: The path to the test program to run.
  39         * env: The environment to use for running the program.
  40         * symbols_path: A path to a directory containing Breakpad-formatted
  41                         symbol files for producing stack traces on crash.
  42         * timeout_factor: An optional test-specific timeout multiplier.
  43
  44         Return True if the program exits with a zero status, False otherwise.
  45         """
  46         basename = os.path.basename(prog)
  47         self.log.test_start(basename)
  48         with mozfile.TemporaryDirectory() as tempdir:
  49             if interactive:
  50                 # For tests run locally, via mach, print output directly
  51                 proc = mozprocess.ProcessHandler([prog],
  52                                                  cwd=tempdir,
  53                                                  env=env,
  54                                                  storeOutput=False,
  55                                                  universal_newlines=True)
  56             else:
  57                 proc = mozprocess.ProcessHandler([prog],
  58                                                  cwd=tempdir,
  59                                                  env=env,
  60                                                  storeOutput=True,
  61                                                  processOutputLine=lambda _: None,
  62                                                  universal_newlines=True)
  63             # TODO: After bug 811320 is fixed, don't let .run() kill the process,
  64             # instead use a timeout in .wait() and then kill to get a stack.
  65             test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
  66             proc.run(timeout=test_timeout,
  67                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
  68             proc.wait()
  69             if proc.output:
  70                 if self.fix_stack:
  71                     procOutput = [self.fix_stack(l) for l in proc.output]
  72                 else:
  73                     procOutput = proc.output
  74
  75                 output = "\n%s" % "\n".join(procOutput)
  76                 self.log.process_output(proc.pid, output, command=[prog])
  77             if proc.timedOut:
  78                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
  79                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
  80                                   message=message)
  81                 return False
  82             if mozcrash.check_for_crashes(tempdir, symbols_path,
  83                                           test_name=basename):
  84                 self.log.test_end(basename, status='CRASH', expected='PASS')
  85                 return False
  86             result = proc.proc.returncode == 0
  87             if not result:
  88                 self.log.test_end(basename, status='FAIL', expected='PASS',
  89                                   message=("test failed with return code %d" %
  90                                            proc.proc.returncode))
  91             else:
  92                 self.log.test_end(basename, status='PASS', expected='PASS')
  93             return result
  94
  95     def build_core_environment(self, env, enable_webrender):
  96         """
  97         Add environment variables likely to be used across all platforms, including remote systems.
  98         """
  99         env["MOZ_XRE_DIR"] = self.xre_path
 100         # TODO: switch this to just abort once all C++ unit tests have
 101         # been fixed to enable crash reporting
 102         env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 103         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 104         env["MOZ_CRASHREPORTER"] = "1"
 105
 106         if enable_webrender:
 107             env['MOZ_WEBRENDER'] = '1'
 108             env['MOZ_ACCELERATED'] = '1'
 109         else:
 110             env['MOZ_WEBRENDER'] = '0'
 111
 112         return env
 113
 114     def build_environment(self, enable_webrender=False):
 115         """
 116         Create and return a dictionary of all the appropriate env variables and values.
 117         On a remote system, we overload this to set different values and are missing things
 118         like os.environ and PATH.
 119         """
 120         if not os.path.isdir(self.xre_path):
 121             raise Exception("xre_path does not exist: %s", self.xre_path)
 122         env = dict(os.environ)
 123         env = self.build_core_environment(env, enable_webrender)
 124         pathvar = ""
 125         libpath = self.xre_path
 126         if mozinfo.os == "linux":
 127             pathvar = "LD_LIBRARY_PATH"
 128         elif mozinfo.os == "mac":
 129             applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
 130             if os.path.exists(applibpath):
 131                 # Set the library load path to Contents/MacOS if we're run from
 132                 # the app bundle.
 133                 libpath = applibpath
 134             pathvar = "DYLD_LIBRARY_PATH"
 135         elif mozinfo.os == "win":
 136             pathvar = "PATH"
 137         if pathvar:
 138             if pathvar in env:
 139                 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
 140             else:
 141                 env[pathvar] = libpath
 142
 143         if mozinfo.info["asan"]:
 144             # Use llvm-symbolizer for ASan if available/required
 145             llvmsym = os.path.join(
 146                 self.xre_path,
 147                 "llvm-symbolizer" + mozinfo.info["bin_suffix"].encode('ascii'))
 148             if os.path.isfile(llvmsym):
 149                 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 150                 self.log.info("ASan using symbolizer at %s" % llvmsym)
 151             else:
 152                 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
 153
 154             # dom/media/webrtc/transport tests statically link in NSS, which
 155             # causes ODR violations. See bug 1215679.
 156             assert 'ASAN_OPTIONS' not in env
 157             env['ASAN_OPTIONS'] = 'detect_leaks=0:detect_odr_violation=0'
 158
 159         return env
 160
 161     def run_tests(self, programs, xre_path, symbols_path=None,
 162                   utility_path=None, enable_webrender=False, interactive=False):
 163         """
 164         Run a set of C++ unit test programs.
 165
 166         Arguments:
 167         * programs: An iterable containing (test path, test timeout factor) tuples
 168         * xre_path: A path to a directory containing a XUL Runtime Environment.
 169         * symbols_path: A path to a directory containing Breakpad-formatted
 170                         symbol files for producing stack traces on crash.
 171         * utility_path: A path to a directory containing utility programs
 172                         (xpcshell et al)
 173
 174         Returns True if all test programs exited with a zero status, False
 175         otherwise.
 176         """
 177         self.xre_path = xre_path
 178         self.log = mozlog.get_default_logger()
 179         if utility_path:
 180             self.fix_stack = mozrunner.utils.get_stack_fixer_function(
 181                 utility_path, symbols_path)
 182         self.log.suite_start(programs, name='cppunittest')
 183         env = self.build_environment(enable_webrender)
 184         pass_count = 0
 185         fail_count = 0
 186         for prog in programs:
 187             test_path = prog[0]
 188             timeout_factor = prog[1]
 189             single_result = self.run_one_test(test_path, env, symbols_path,
 190                                               interactive, timeout_factor)
 191             if single_result:
 192                 pass_count += 1
 193             else:
 194                 fail_count += 1
 195         self.log.suite_end()
 196
 197         # Mozharness-parseable summary formatting.
 198         self.log.info("Result summary:")
 199         self.log.info("cppunittests INFO | Passed: %d" % pass_count)
 200         self.log.info("cppunittests INFO | Failed: %d" % fail_count)
 201         return fail_count == 0
 202
 203
 204 class CPPUnittestOptions(OptionParser):
 205     def __init__(self):
 206         OptionParser.__init__(self)
 207         self.add_option("--xre-path",
 208                         action="store", type="string", dest="xre_path",
 209                         default=None,
 210                         help="absolute path to directory containing XRE (probably xulrunner)")
 211         self.add_option("--symbols-path",
 212                         action="store", type="string", dest="symbols_path",
 213                         default=None,
 214                         help="absolute path to directory containing breakpad symbols, or "
 215                         "the URL of a zip file containing symbols")
 216         self.add_option("--manifest-path",
 217                         action="store", type="string", dest="manifest_path",
 218                         default=None,
 219                         help="path to test manifest, if different from the path to test binaries")
 220         self.add_option("--utility-path",
 221                         action="store", type="string", dest="utility_path",
 222                         default=None,
 223                         help="path to directory containing utility programs")
 224         self.add_option("--enable-webrender",
 225                         action="store_true", dest="enable_webrender",
 226                         default=False,
 227                         help="Enable the WebRender compositor in Gecko")
 228
 229
 230 def extract_unittests_from_args(args, environ, manifest_path):
 231     """Extract unittests from args, expanding directories as needed"""
 232     mp = manifestparser.TestManifest(strict=True)
 233     tests = []
 234     binary_path = None
 235
 236     if manifest_path:
 237         mp.read(manifest_path)
 238         binary_path = os.path.abspath(args[0])
 239     else:
 240         for p in args:
 241             if os.path.isdir(p):
 242                 try:
 243                     mp.read(os.path.join(p, 'cppunittest.ini'))
 244                 except IOError:
 245                     files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
 246                     tests.extend((f, 1) for f in files
 247                                  if os.access(f, os.R_OK | os.X_OK))
 248             else:
 249                 tests.append((os.path.abspath(p), 1))
 250
 251     # we skip the existence check here because not all tests are built
 252     # for all platforms (and it will fail on Windows anyway)
 253     active_tests = mp.active_tests(exists=False, disabled=False, **environ)
 254     suffix = '.exe' if mozinfo.isWin else ''
 255     if binary_path:
 256         tests.extend([
 257             (os.path.join(binary_path, test['relpath'] + suffix),
 258              int(test.get('requesttimeoutfactor', 1)))
 259             for test in active_tests])
 260     else:
 261         tests.extend([
 262             (test['path'] + suffix,
 263              int(test.get('requesttimeoutfactor', 1)))
 264             for test in active_tests
 265         ])
 266
 267     # skip and warn for any tests in the manifest that are not found
 268     final_tests = []
 269     log = mozlog.get_default_logger()
 270     for test in tests:
 271         if os.path.isfile(test[0]):
 272             final_tests.append(test)
 273         else:
 274             log.warning("test file not found: %s - skipped" % test[0])
 275
 276     return final_tests
 277
 278
 279 def update_mozinfo():
 280     """walk up directories to find mozinfo.json update the info"""
 281     path = SCRIPT_DIR
 282     dirs = set()
 283     while path != os.path.expanduser('~'):
 284         if path in dirs:
 285             break
 286         dirs.add(path)
 287         path = os.path.split(path)[0]
 288     mozinfo.find_and_update_from_json(*dirs)
 289
 290
 291 def run_test_harness(options, args):
 292     update_mozinfo()
 293     progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
 294     options.xre_path = os.path.abspath(options.xre_path)
 295     options.utility_path = os.path.abspath(options.utility_path)
 296     tester = CPPUnitTests()
 297     result = tester.run_tests(progs, options.xre_path, options.symbols_path,
 298                               options.utility_path, options.enable_webrender)
 299
 300     return result
 301
 302
 303 def main():
 304     parser = CPPUnittestOptions()
 305     mozlog.commandline.add_logging_group(parser)
 306     options, args = parser.parse_args()
 307     if not args:
 308         print("""Usage: %s <test binary> [<test binary>...]""" % sys.argv[0], file=sys.stderr)
 309         sys.exit(1)
 310     if not options.xre_path:
 311         print("""Error: --xre-path is required""", file=sys.stderr)
 312         sys.exit(1)
 313     if options.manifest_path and len(args) > 1:
 314         print("Error: multiple arguments not supported with --test-manifest", file=sys.stderr)
 315         sys.exit(1)
 316     log = mozlog.commandline.setup_logging("cppunittests", options,
 317                                            {"tbpl": sys.stdout})
 318     try:
 319         result = run_test_harness(options, args)
 320     except Exception as e:
 321         log.error(str(e))
 322         result = False
 323
 324     sys.exit(0 if result else 1)
 325
 326
 327 if __name__ == '__main__':
 328     main()