[analyzer] Fix a crash until we can handle temporary struct objects properly.
[clang.git] / utils / FuzzTest
blob2aa5989a80d450f85df4c3f9228bb73386e642d4
1 #!/usr/bin/env python
3 """
4 This is a generic fuzz testing tool, see --help for more information.
5 """
7 import os
8 import sys
9 import random
10 import subprocess
11 import itertools
13 class TestGenerator:
14 def __init__(self, inputs, delete, insert, replace,
15 insert_strings, pick_input):
16 self.inputs = [(s, open(s).read()) for s in inputs]
18 self.delete = bool(delete)
19 self.insert = bool(insert)
20 self.replace = bool(replace)
21 self.pick_input = bool(pick_input)
22 self.insert_strings = list(insert_strings)
24 self.num_positions = sum([len(d) for _,d in self.inputs])
25 self.num_insert_strings = len(insert_strings)
26 self.num_tests = ((delete + (insert + replace)*self.num_insert_strings)
27 * self.num_positions)
28 self.num_tests += 1
30 if self.pick_input:
31 self.num_tests *= self.num_positions
33 def position_to_source_index(self, position):
34 for i,(s,d) in enumerate(self.inputs):
35 n = len(d)
36 if position < n:
37 return (i,position)
38 position -= n
39 raise ValueError,'Invalid position.'
41 def get_test(self, index):
42 assert 0 <= index < self.num_tests
44 picked_position = None
45 if self.pick_input:
46 index,picked_position = divmod(index, self.num_positions)
47 picked_position = self.position_to_source_index(picked_position)
49 if index == 0:
50 return ('nothing', None, None, picked_position)
52 index -= 1
53 index,position = divmod(index, self.num_positions)
54 position = self.position_to_source_index(position)
55 if self.delete:
56 if index == 0:
57 return ('delete', position, None, picked_position)
58 index -= 1
60 index,insert_index = divmod(index, self.num_insert_strings)
61 insert_str = self.insert_strings[insert_index]
62 if self.insert:
63 if index == 0:
64 return ('insert', position, insert_str, picked_position)
65 index -= 1
67 assert self.replace
68 assert index == 0
69 return ('replace', position, insert_str, picked_position)
71 class TestApplication:
72 def __init__(self, tg, test):
73 self.tg = tg
74 self.test = test
76 def apply(self):
77 if self.test[0] == 'nothing':
78 pass
79 else:
80 i,j = self.test[1]
81 name,data = self.tg.inputs[i]
82 if self.test[0] == 'delete':
83 data = data[:j] + data[j+1:]
84 elif self.test[0] == 'insert':
85 data = data[:j] + self.test[2] + data[j:]
86 elif self.test[0] == 'replace':
87 data = data[:j] + self.test[2] + data[j+1:]
88 else:
89 raise ValueError,'Invalid test %r' % self.test
90 open(name,'wb').write(data)
92 def revert(self):
93 if self.test[0] != 'nothing':
94 i,j = self.test[1]
95 name,data = self.tg.inputs[i]
96 open(name,'wb').write(data)
98 def quote(str):
99 return '"' + str + '"'
101 def run_one_test(test_application, index, input_files, args):
102 test = test_application.test
104 # Interpolate arguments.
105 options = { 'index' : index,
106 'inputs' : ' '.join(quote(f) for f in input_files) }
108 # Add picked input interpolation arguments, if used.
109 if test[3] is not None:
110 pos = test[3][1]
111 options['picked_input'] = input_files[test[3][0]]
112 options['picked_input_pos'] = pos
113 # Compute the line and column.
114 file_data = test_application.tg.inputs[test[3][0]][1]
115 line = column = 1
116 for i in range(pos):
117 c = file_data[i]
118 if c == '\n':
119 line += 1
120 column = 1
121 else:
122 column += 1
123 options['picked_input_line'] = line
124 options['picked_input_col'] = column
126 test_args = [a % options for a in args]
127 if opts.verbose:
128 print '%s: note: executing %r' % (sys.argv[0], test_args)
130 stdout = None
131 stderr = None
132 if opts.log_dir:
133 stdout_log_path = os.path.join(opts.log_dir, '%s.out' % index)
134 stderr_log_path = os.path.join(opts.log_dir, '%s.err' % index)
135 stdout = open(stdout_log_path, 'wb')
136 stderr = open(stderr_log_path, 'wb')
137 else:
138 sys.stdout.flush()
139 p = subprocess.Popen(test_args, stdout=stdout, stderr=stderr)
140 p.communicate()
141 exit_code = p.wait()
143 test_result = (exit_code == opts.expected_exit_code or
144 exit_code in opts.extra_exit_codes)
146 if stdout is not None:
147 stdout.close()
148 stderr.close()
150 # Remove the logs for passes, unless logging all results.
151 if not opts.log_all and test_result:
152 os.remove(stdout_log_path)
153 os.remove(stderr_log_path)
155 if not test_result:
156 print 'FAIL: %d' % index
157 elif not opts.succinct:
158 print 'PASS: %d' % index
160 def main():
161 global opts
162 from optparse import OptionParser, OptionGroup
163 parser = OptionParser("""%prog [options] ... test command args ...
165 %prog is a tool for fuzzing inputs and testing them.
167 The most basic usage is something like:
169 $ %prog --file foo.txt ./test.sh
171 which will run a default list of fuzzing strategies on the input. For each
172 fuzzed input, it will overwrite the input files (in place), run the test script,
173 then restore the files back to their original contents.
175 NOTE: You should make sure you have a backup copy of your inputs, in case
176 something goes wrong!!!
178 You can cause the fuzzing to not restore the original files with
179 '--no-revert'. Generally this is used with '--test <index>' to run one failing
180 test and then leave the fuzzed inputs in place to examine the failure.
182 For each fuzzed input, %prog will run the test command given on the command
183 line. Each argument in the command is subject to string interpolation before
184 being executed. The syntax is "%(VARIABLE)FORMAT" where FORMAT is a standard
185 printf format, and VARIBLE is one of:
187 'index' - the test index being run
188 'inputs' - the full list of test inputs
189 'picked_input' - (with --pick-input) the selected input file
190 'picked_input_pos' - (with --pick-input) the selected input position
191 'picked_input_line' - (with --pick-input) the selected input line
192 'picked_input_col' - (with --pick-input) the selected input column
194 By default, the script will run forever continually picking new tests to
195 run. You can limit the number of tests that are run with '--max-tests <number>',
196 and you can run a particular test with '--test <index>'.
197 """)
198 parser.add_option("-v", "--verbose", help="Show more output",
199 action='store_true', dest="verbose", default=False)
200 parser.add_option("-s", "--succinct", help="Reduce amount of output",
201 action="store_true", dest="succinct", default=False)
203 group = OptionGroup(parser, "Test Execution")
204 group.add_option("", "--expected-exit-code", help="Set expected exit code",
205 type=int, dest="expected_exit_code",
206 default=0)
207 group.add_option("", "--extra-exit-code",
208 help="Set additional expected exit code",
209 type=int, action="append", dest="extra_exit_codes",
210 default=[])
211 group.add_option("", "--log-dir",
212 help="Capture test logs to an output directory",
213 type=str, dest="log_dir",
214 default=None)
215 group.add_option("", "--log-all",
216 help="Log all outputs (not just failures)",
217 action="store_true", dest="log_all", default=False)
218 parser.add_option_group(group)
220 group = OptionGroup(parser, "Input Files")
221 group.add_option("", "--file", metavar="PATH",
222 help="Add an input file to fuzz",
223 type=str, action="append", dest="input_files", default=[])
224 group.add_option("", "--filelist", metavar="LIST",
225 help="Add a list of inputs files to fuzz (one per line)",
226 type=int, action="append", dest="filelists", default=[])
227 parser.add_option_group(group)
229 group = OptionGroup(parser, "Fuzz Options")
230 group.add_option("", "--replacement-chars", dest="replacement_chars",
231 help="Characters to insert/replace",
232 default="0{}[]<>\;@#$^%& ")
233 group.add_option("", "--replacement-string", dest="replacement_strings",
234 action="append", help="Add a replacement string to use",
235 default=[])
236 group.add_option("", "--replacement-list", dest="replacement_lists",
237 help="Add a list of replacement strings (one per line)",
238 action="append", default=[])
239 group.add_option("", "--no-delete", help="Don't delete characters",
240 action='store_false', dest="enable_delete", default=True)
241 group.add_option("", "--no-insert", help="Don't insert strings",
242 action='store_false', dest="enable_insert", default=True)
243 group.add_option("", "--no-replace", help="Don't replace strings",
244 action='store_false', dest="enable_replace", default=True)
245 group.add_option("", "--no-revert", help="Don't revert changes",
246 action='store_false', dest="revert", default=True)
247 parser.add_option_group(group)
249 group = OptionGroup(parser, "Test Selection")
250 group.add_option("", "--test", help="Run a particular test",
251 type=int, dest="test", default=None, metavar="INDEX")
252 group.add_option("", "--max-tests", help="Maximum number of tests",
253 type=int, dest="max_tests", default=10, metavar="COUNT")
254 group.add_option("", "--pick-input",
255 help="Randomly select an input byte as well as fuzzing",
256 action='store_true', dest="pick_input", default=False)
257 parser.add_option_group(group)
259 parser.disable_interspersed_args()
261 (opts, args) = parser.parse_args()
263 if not args:
264 parser.error("Invalid number of arguments")
266 # Collect the list of inputs.
267 input_files = list(opts.input_files)
268 for filelist in opts.filelists:
269 f = open(filelist)
270 try:
271 for ln in f:
272 ln = ln.strip()
273 if ln:
274 input_files.append(ln)
275 finally:
276 f.close()
277 input_files.sort()
279 if not input_files:
280 parser.error("No input files!")
282 print '%s: note: fuzzing %d files.' % (sys.argv[0], len(input_files))
284 # Make sure the log directory exists if used.
285 if opts.log_dir:
286 if not os.path.exists(opts.log_dir):
287 try:
288 os.mkdir(opts.log_dir)
289 except OSError:
290 print "%s: error: log directory couldn't be created!" % (
291 sys.argv[0],)
292 raise SystemExit,1
294 # Get the list if insert/replacement strings.
295 replacements = list(opts.replacement_chars)
296 replacements.extend(opts.replacement_strings)
297 for replacement_list in opts.replacement_lists:
298 f = open(replacement_list)
299 try:
300 for ln in f:
301 ln = ln[:-1]
302 if ln:
303 replacements.append(ln)
304 finally:
305 f.close()
307 # Unique and order the replacement list.
308 replacements = list(set(replacements))
309 replacements.sort()
311 # Create the test generator.
312 tg = TestGenerator(input_files, opts.enable_delete, opts.enable_insert,
313 opts.enable_replace, replacements, opts.pick_input)
315 print '%s: note: %d input bytes.' % (sys.argv[0], tg.num_positions)
316 print '%s: note: %d total tests.' % (sys.argv[0], tg.num_tests)
317 if opts.test is not None:
318 it = [opts.test]
319 elif opts.max_tests is not None:
320 it = itertools.imap(random.randrange,
321 itertools.repeat(tg.num_tests, opts.max_tests))
322 else:
323 it = itertools.imap(random.randrange, itertools.repeat(tg.num_tests))
324 for test in it:
325 t = tg.get_test(test)
327 if opts.verbose:
328 print '%s: note: running test %d: %r' % (sys.argv[0], test, t)
329 ta = TestApplication(tg, t)
330 try:
331 ta.apply()
332 run_one_test(ta, test, input_files, args)
333 finally:
334 if opts.revert:
335 ta.revert()
337 sys.stdout.flush()
339 if __name__ == '__main__':
340 main()