Bumping manifests a=b2g-bump
[gecko.git] / testing / runcppunittests.py
blob3e83f325b77745da21cf1249fa63f3c17bc05531
1 #!/usr/bin/env python
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 from __future__ import with_statement
8 import sys, os, tempfile, shutil
9 from optparse import OptionParser
10 import mozprocess
11 import mozinfo
12 import mozcrash
13 import mozfile
14 from contextlib import contextmanager
15 from mozlog import structured
16 from subprocess import PIPE
18 class CPPUnitTests(object):
19 # Time (seconds) to wait for test process to complete
20 TEST_PROC_TIMEOUT = 900
21 # Time (seconds) in which process will be killed if it produces no output.
22 TEST_PROC_NO_OUTPUT_TIMEOUT = 300
24 def run_one_test(self, prog, env, symbols_path=None, interactive=False):
25 """
26 Run a single C++ unit test program.
28 Arguments:
29 * prog: The path to the test program to run.
30 * env: The environment to use for running the program.
31 * symbols_path: A path to a directory containing Breakpad-formatted
32 symbol files for producing stack traces on crash.
34 Return True if the program exits with a zero status, False otherwise.
35 """
36 basename = os.path.basename(prog)
37 self.log.test_start(basename)
38 with mozfile.TemporaryDirectory() as tempdir:
39 if interactive:
40 # For tests run locally, via mach, print output directly
41 proc = mozprocess.ProcessHandler([prog],
42 cwd=tempdir,
43 env=env,
44 storeOutput=False)
45 else:
46 proc = mozprocess.ProcessHandler([prog],
47 cwd=tempdir,
48 env=env,
49 storeOutput=True,
50 processOutputLine=lambda _: None)
51 #TODO: After bug 811320 is fixed, don't let .run() kill the process,
52 # instead use a timeout in .wait() and then kill to get a stack.
53 proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT,
54 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
55 proc.wait()
56 if proc.output:
57 output = "\n%s" % "\n".join(proc.output)
58 self.log.process_output(proc.pid, output, command=[prog])
59 if proc.timedOut:
60 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
61 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
62 message=message)
63 return False
64 if mozcrash.check_for_crashes(tempdir, symbols_path,
65 test_name=basename):
66 self.log.test_end(basename, status='CRASH', expected='PASS')
67 return False
68 result = proc.proc.returncode == 0
69 if not result:
70 self.log.test_end(basename, status='FAIL', expected='PASS',
71 message=("test failed with return code %d" %
72 proc.proc.returncode))
73 else:
74 self.log.test_end(basename, status='PASS', expected='PASS')
75 return result
77 def build_core_environment(self, env = {}):
78 """
79 Add environment variables likely to be used across all platforms, including remote systems.
80 """
81 env["MOZILLA_FIVE_HOME"] = self.xre_path
82 env["MOZ_XRE_DIR"] = self.xre_path
83 #TODO: switch this to just abort once all C++ unit tests have
84 # been fixed to enable crash reporting
85 env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
86 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
87 env["MOZ_CRASHREPORTER"] = "1"
88 return env
90 def build_environment(self):
91 """
92 Create and return a dictionary of all the appropriate env variables and values.
93 On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
94 """
95 if not os.path.isdir(self.xre_path):
96 raise Exception("xre_path does not exist: %s", self.xre_path)
97 env = dict(os.environ)
98 env = self.build_core_environment(env)
99 pathvar = ""
100 libpath = self.xre_path
101 if mozinfo.os == "linux":
102 pathvar = "LD_LIBRARY_PATH"
103 elif mozinfo.os == "mac":
104 applibpath = os.path.join(os.path.dirname(libpath), 'MacOS')
105 if os.path.exists(applibpath):
106 # Set the library load path to Contents/MacOS if we're run from
107 # the app bundle.
108 libpath = applibpath
109 pathvar = "DYLD_LIBRARY_PATH"
110 elif mozinfo.os == "win":
111 pathvar = "PATH"
112 if pathvar:
113 if pathvar in env:
114 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
115 else:
116 env[pathvar] = libpath
118 # Use llvm-symbolizer for ASan if available/required
119 llvmsym = os.path.join(self.xre_path, "llvm-symbolizer")
120 if os.path.isfile(llvmsym):
121 env["ASAN_SYMBOLIZER_PATH"] = llvmsym
122 self.log.info("ASan using symbolizer at %s" % llvmsym)
123 else:
124 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
126 return env
128 def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
130 Run a set of C++ unit test programs.
132 Arguments:
133 * programs: An iterable containing paths to test programs.
134 * xre_path: A path to a directory containing a XUL Runtime Environment.
135 * symbols_path: A path to a directory containing Breakpad-formatted
136 symbol files for producing stack traces on crash.
138 Returns True if all test programs exited with a zero status, False
139 otherwise.
141 self.xre_path = xre_path
142 self.log = structured.structuredlog.get_default_logger()
143 self.log.suite_start(programs)
144 env = self.build_environment()
145 pass_count = 0
146 fail_count = 0
147 for prog in programs:
148 single_result = self.run_one_test(prog, env, symbols_path, interactive)
149 if single_result:
150 pass_count += 1
151 else:
152 fail_count += 1
153 self.log.suite_end()
155 # Mozharness-parseable summary formatting.
156 self.log.info("Result summary:")
157 self.log.info("cppunittests INFO | Passed: %d" % pass_count)
158 self.log.info("cppunittests INFO | Failed: %d" % fail_count)
159 return fail_count == 0
161 class CPPUnittestOptions(OptionParser):
162 def __init__(self):
163 OptionParser.__init__(self)
164 self.add_option("--xre-path",
165 action = "store", type = "string", dest = "xre_path",
166 default = None,
167 help = "absolute path to directory containing XRE (probably xulrunner)")
168 self.add_option("--symbols-path",
169 action = "store", type = "string", dest = "symbols_path",
170 default = None,
171 help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
172 self.add_option("--skip-manifest",
173 action = "store", type = "string", dest = "manifest_file",
174 default = None,
175 help = "absolute path to a manifest file")
177 def extract_unittests_from_args(args, manifest_file):
178 """Extract unittests from args, expanding directories as needed"""
179 progs = []
181 # Known files commonly packaged with the cppunittests that are not tests
182 skipped_progs = set(['.mkdir.done', 'remotecppunittests.py', 'runcppunittests.py', 'runcppunittests.pyc'])
184 if manifest_file:
185 skipped_progs.add(os.path.basename(manifest_file))
186 with open(manifest_file) as f:
187 for line in f:
188 # strip out comment, if any
189 prog = line.split('#')[0]
190 if prog:
191 skipped_progs.add(prog.strip())
193 for p in args:
194 if os.path.isdir(p):
195 progs.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p) if not x in skipped_progs])
196 elif p not in skipped_progs:
197 progs.append(os.path.abspath(p))
199 return progs
201 def main():
202 parser = CPPUnittestOptions()
203 structured.commandline.add_logging_group(parser)
204 options, args = parser.parse_args()
205 if not args:
206 print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
207 sys.exit(1)
208 if not options.xre_path:
209 print >>sys.stderr, """Error: --xre-path is required"""
210 sys.exit(1)
212 log = structured.commandline.setup_logging("cppunittests",
213 options,
214 {"tbpl": sys.stdout})
216 progs = extract_unittests_from_args(args, options.manifest_file)
217 options.xre_path = os.path.abspath(options.xre_path)
218 if mozinfo.isMac:
219 options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
220 tester = CPPUnitTests()
222 try:
223 result = tester.run_tests(progs, options.xre_path, options.symbols_path)
224 except Exception as e:
225 log.error(str(e))
226 result = False
228 sys.exit(0 if result else 1)
230 if __name__ == '__main__':
231 main()