Daily bump.
[official-gcc.git] / contrib / dg-extract-results.py
blob30aa68771d491451f72d6dbd18f6d5ac339451b5
1 #!/usr/bin/python
3 # Copyright (C) 2014 Free Software Foundation, Inc.
5 # This script is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3, or (at your option)
8 # any later version.
10 import sys
11 import getopt
12 import re
13 import io
14 from datetime import datetime
15 from operator import attrgetter
17 # True if unrecognised lines should cause a fatal error. Might want to turn
18 # this on by default later.
19 strict = False
21 # True if the order of .log segments should match the .sum file, false if
22 # they should keep the original order.
23 sort_logs = True
25 # A version of open() that is safe against whatever binary output
26 # might be added to the log.
27 def safe_open (filename):
28 if sys.version_info >= (3, 0):
29 return open (filename, 'r', errors = 'surrogateescape')
30 return open (filename, 'r')
32 # Force stdout to handle escape sequences from a safe_open file.
33 if sys.version_info >= (3, 0):
34 sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
35 errors = 'surrogateescape')
37 class Named:
38 def __init__ (self, name):
39 self.name = name
41 class ToolRun (Named):
42 def __init__ (self, name):
43 Named.__init__ (self, name)
44 # The variations run for this tool, mapped by --target_board name.
45 self.variations = dict()
47 # Return the VariationRun for variation NAME.
48 def get_variation (self, name):
49 if name not in self.variations:
50 self.variations[name] = VariationRun (name)
51 return self.variations[name]
53 class VariationRun (Named):
54 def __init__ (self, name):
55 Named.__init__ (self, name)
56 # A segment of text before the harness runs start, describing which
57 # baseboard files were loaded for the target.
58 self.header = None
59 # The harnesses run for this variation, mapped by filename.
60 self.harnesses = dict()
61 # A list giving the number of times each type of result has
62 # been seen.
63 self.counts = []
65 # Return the HarnessRun for harness NAME.
66 def get_harness (self, name):
67 if name not in self.harnesses:
68 self.harnesses[name] = HarnessRun (name)
69 return self.harnesses[name]
71 class HarnessRun (Named):
72 def __init__ (self, name):
73 Named.__init__ (self, name)
74 # Segments of text that make up the harness run, mapped by a test-based
75 # key that can be used to order them.
76 self.segments = dict()
77 # Segments of text that make up the harness run but which have
78 # no recognized test results. These are typically harnesses that
79 # are completely skipped for the target.
80 self.empty = []
81 # A list of results. Each entry is a pair in which the first element
82 # is a unique sorting key and in which the second is the full
83 # PASS/FAIL line.
84 self.results = []
86 # Add a segment of text to the harness run. If the segment includes
87 # test results, KEY is an example of one of them, and can be used to
88 # combine the individual segments in order. If the segment has no
89 # test results (e.g. because the harness doesn't do anything for the
90 # current configuration) then KEY is None instead. In that case
91 # just collect the segments in the order that we see them.
92 def add_segment (self, key, segment):
93 if key:
94 assert key not in self.segments
95 self.segments[key] = segment
96 else:
97 self.empty.append (segment)
99 class Segment:
100 def __init__ (self, filename, start):
101 self.filename = filename
102 self.start = start
103 self.lines = 0
105 class Prog:
106 def __init__ (self):
107 # The variations specified on the command line.
108 self.variations = []
109 # The variations seen in the input files.
110 self.known_variations = set()
111 # The tools specified on the command line.
112 self.tools = []
113 # Whether to create .sum rather than .log output.
114 self.do_sum = True
115 # Regexps used while parsing.
116 self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
117 self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
118 self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
119 r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
120 r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)')
121 self.completed_re = re.compile (r'.* completed at (.*)')
122 # Pieces of text to write at the head of the output.
123 # start_line is a pair in which the first element is a datetime
124 # and in which the second is the associated 'Test Run By' line.
125 self.start_line = None
126 self.native_line = ''
127 self.target_line = ''
128 self.host_line = ''
129 self.acats_premable = ''
130 # Pieces of text to write at the end of the output.
131 # end_line is like start_line but for the 'runtest completed' line.
132 self.acats_failures = []
133 self.version_output = ''
134 self.end_line = None
135 # Known summary types.
136 self.count_names = [
137 '# of DejaGnu errors\t\t',
138 '# of expected passes\t\t',
139 '# of unexpected failures\t',
140 '# of unexpected successes\t',
141 '# of expected failures\t\t',
142 '# of unknown successes\t\t',
143 '# of known failures\t\t',
144 '# of untested testcases\t\t',
145 '# of unresolved testcases\t',
146 '# of unsupported tests\t\t',
147 '# of paths in test names\t',
148 '# of duplicate test names\t'
150 self.runs = dict()
152 def usage (self):
153 name = sys.argv[0]
154 sys.stderr.write ('Usage: ' + name
155 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
157 tool The tool (e.g. g++, libffi) for which to create a
158 new test summary file. If not specified then output
159 is created for all tools.
160 variant-list One or more test variant names. If the list is
161 not specified then one is constructed from all
162 variants in the files for <tool>.
163 sum-file A test summary file with the format of those
164 created by runtest from DejaGnu.
165 If -L is used, merge *.log files instead of *.sum. In this
166 mode the exact order of lines may not be preserved, just different
167 Running *.exp chunks should be in correct order.
168 ''')
169 sys.exit (1)
171 def fatal (self, what, string):
172 if not what:
173 what = sys.argv[0]
174 sys.stderr.write (what + ': ' + string + '\n')
175 sys.exit (1)
177 # Parse the command-line arguments.
178 def parse_cmdline (self):
179 try:
180 (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
181 if len (self.files) == 0:
182 self.usage()
183 for (option, value) in options:
184 if option == '-l':
185 self.variations.append (value)
186 elif option == '-t':
187 self.tools.append (value)
188 else:
189 self.do_sum = False
190 except getopt.GetoptError as e:
191 self.fatal (None, e.msg)
193 # Try to parse time string TIME, returning an arbitrary time on failure.
194 # Getting this right is just a nice-to-have so failures should be silent.
195 def parse_time (self, time):
196 try:
197 return datetime.strptime (time, '%c')
198 except ValueError:
199 return datetime.now()
201 # Parse an integer and abort on failure.
202 def parse_int (self, filename, value):
203 try:
204 return int (value)
205 except ValueError:
206 self.fatal (filename, 'expected an integer, got: ' + value)
208 # Return a list that represents no test results.
209 def zero_counts (self):
210 return [0 for x in self.count_names]
212 # Return the ToolRun for tool NAME.
213 def get_tool (self, name):
214 if name not in self.runs:
215 self.runs[name] = ToolRun (name)
216 return self.runs[name]
218 # Add the result counts in list FROMC to TOC.
219 def accumulate_counts (self, toc, fromc):
220 for i in range (len (self.count_names)):
221 toc[i] += fromc[i]
223 # Parse the list of variations after 'Schedule of variations:'.
224 # Return the number seen.
225 def parse_variations (self, filename, file):
226 num_variations = 0
227 while True:
228 line = file.readline()
229 if line == '':
230 self.fatal (filename, 'could not parse variation list')
231 if line == '\n':
232 break
233 self.known_variations.add (line.strip())
234 num_variations += 1
235 return num_variations
237 # Parse from the first line after 'Running target ...' to the end
238 # of the run's summary.
239 def parse_run (self, filename, file, tool, variation, num_variations):
240 header = None
241 harness = None
242 segment = None
243 final_using = 0
244 has_warning = 0
246 # If this is the first run for this variation, add any text before
247 # the first harness to the header.
248 if not variation.header:
249 segment = Segment (filename, file.tell())
250 variation.header = segment
252 # Parse the rest of the summary (the '# of ' lines).
253 if len (variation.counts) == 0:
254 variation.counts = self.zero_counts()
256 # Parse up until the first line of the summary.
257 if num_variations == 1:
258 end = '\t\t=== ' + tool.name + ' Summary ===\n'
259 else:
260 end = ('\t\t=== ' + tool.name + ' Summary for '
261 + variation.name + ' ===\n')
262 while True:
263 line = file.readline()
264 if line == '':
265 self.fatal (filename, 'no recognised summary line')
266 if line == end:
267 break
269 # Look for the start of a new harness.
270 if line.startswith ('Running ') and line.endswith (' ...\n'):
271 # Close off the current harness segment, if any.
272 if harness:
273 segment.lines -= final_using
274 harness.add_segment (first_key, segment)
275 name = line[len ('Running '):-len(' ...\n')]
276 harness = variation.get_harness (name)
277 segment = Segment (filename, file.tell())
278 first_key = None
279 final_using = 0
280 continue
282 # Record test results. Associate the first test result with
283 # the harness segment, so that if a run for a particular harness
284 # has been split up, we can reassemble the individual segments
285 # in a sensible order.
287 # dejagnu sometimes issues warnings about the testing environment
288 # before running any tests. Treat them as part of the header
289 # rather than as a test result.
290 match = self.result_re.match (line)
291 if match and (harness or not line.startswith ('WARNING:')):
292 if not harness:
293 self.fatal (filename, 'saw test result before harness name')
294 name = match.group (2)
295 # Ugly hack to get the right order for gfortran.
296 if name.startswith ('gfortran.dg/g77/'):
297 name = 'h' + name
298 # If we have a time out warning, make sure it appears
299 # before the following testcase diagnostic: we insert
300 # the testname before 'program' so that sort faces a
301 # list of testnames.
302 if line.startswith ('WARNING: program timed out'):
303 has_warning = 1
304 else:
305 if has_warning == 1:
306 key = (name, len (harness.results))
307 myline = 'WARNING: %s program timed out.\n' % name
308 harness.results.append ((key, myline))
309 has_warning = 0
310 key = (name, len (harness.results))
311 harness.results.append ((key, line))
312 if not first_key and sort_logs:
313 first_key = key
314 if line.startswith ('ERROR: (DejaGnu)'):
315 for i in range (len (self.count_names)):
316 if 'DejaGnu errors' in self.count_names[i]:
317 variation.counts[i] += 1
318 break
320 # 'Using ...' lines are only interesting in a header. Splitting
321 # the test up into parallel runs leads to more 'Using ...' lines
322 # than there would be in a single log.
323 if line.startswith ('Using '):
324 final_using += 1
325 else:
326 final_using = 0
328 # Add other text to the current segment, if any.
329 if segment:
330 segment.lines += 1
332 # Close off the final harness segment, if any.
333 if harness:
334 segment.lines -= final_using
335 harness.add_segment (first_key, segment)
337 while True:
338 before = file.tell()
339 line = file.readline()
340 if line == '':
341 break
342 if line == '\n':
343 continue
344 if not line.startswith ('# '):
345 file.seek (before)
346 break
347 found = False
348 for i in range (len (self.count_names)):
349 if line.startswith (self.count_names[i]):
350 count = line[len (self.count_names[i]):-1].strip()
351 variation.counts[i] += self.parse_int (filename, count)
352 found = True
353 break
354 if not found:
355 self.fatal (filename, 'unknown test result: ' + line[:-1])
357 # Parse an acats run, which uses a different format from dejagnu.
358 # We have just skipped over '=== acats configuration ==='.
359 def parse_acats_run (self, filename, file):
360 # Parse the preamble, which describes the configuration and logs
361 # the creation of support files.
362 record = (self.acats_premable == '')
363 if record:
364 self.acats_premable = '\t\t=== acats configuration ===\n'
365 while True:
366 line = file.readline()
367 if line == '':
368 self.fatal (filename, 'could not parse acats preamble')
369 if line == '\t\t=== acats tests ===\n':
370 break
371 if record:
372 self.acats_premable += line
374 # Parse the test results themselves, using a dummy variation name.
375 tool = self.get_tool ('acats')
376 variation = tool.get_variation ('none')
377 self.parse_run (filename, file, tool, variation, 1)
379 # Parse the failure list.
380 while True:
381 before = file.tell()
382 line = file.readline()
383 if line.startswith ('*** FAILURES: '):
384 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
385 continue
386 file.seek (before)
387 break
389 # Parse the final summary at the end of a log in order to capture
390 # the version output that follows it.
391 def parse_final_summary (self, filename, file):
392 record = (self.version_output == '')
393 while True:
394 line = file.readline()
395 if line == '':
396 break
397 if line.startswith ('# of '):
398 continue
399 if record:
400 self.version_output += line
401 if line == '\n':
402 break
404 # Parse a .log or .sum file.
405 def parse_file (self, filename, file):
406 tool = None
407 target = None
408 num_variations = 1
409 while True:
410 line = file.readline()
411 if line == '':
412 return
414 # Parse the list of variations, which comes before the test
415 # runs themselves.
416 if line.startswith ('Schedule of variations:'):
417 num_variations = self.parse_variations (filename, file)
418 continue
420 # Parse a testsuite run for one tool/variation combination.
421 if line.startswith ('Running target '):
422 name = line[len ('Running target '):-1]
423 if not tool:
424 self.fatal (filename, 'could not parse tool name')
425 if name not in self.known_variations:
426 self.fatal (filename, 'unknown target: ' + name)
427 self.parse_run (filename, file, tool,
428 tool.get_variation (name),
429 num_variations)
430 # If there is only one variation then there is no separate
431 # summary for it. Record any following version output.
432 if num_variations == 1:
433 self.parse_final_summary (filename, file)
434 continue
436 # Parse the start line. In the case where several files are being
437 # parsed, pick the one with the earliest time.
438 match = self.test_run_re.match (line)
439 if match:
440 time = self.parse_time (match.group (2))
441 if not self.start_line or self.start_line[0] > time:
442 self.start_line = (time, line)
443 continue
445 # Parse the form used for native testing.
446 if line.startswith ('Native configuration is '):
447 self.native_line = line
448 continue
450 # Parse the target triplet.
451 if line.startswith ('Target is '):
452 self.target_line = line
453 continue
455 # Parse the host triplet.
456 if line.startswith ('Host is '):
457 self.host_line = line
458 continue
460 # Parse the acats premable.
461 if line == '\t\t=== acats configuration ===\n':
462 self.parse_acats_run (filename, file)
463 continue
465 # Parse the tool name.
466 match = self.tool_re.match (line)
467 if match:
468 tool = self.get_tool (match.group (1))
469 continue
471 # Skip over the final summary (which we instead create from
472 # individual runs) and parse the version output.
473 if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
474 if file.readline() != '\n':
475 self.fatal (filename, 'expected blank line after summary')
476 self.parse_final_summary (filename, file)
477 continue
479 # Parse the completion line. In the case where several files
480 # are being parsed, pick the one with the latest time.
481 match = self.completed_re.match (line)
482 if match:
483 time = self.parse_time (match.group (1))
484 if not self.end_line or self.end_line[0] < time:
485 self.end_line = (time, line)
486 continue
488 # Sanity check to make sure that important text doesn't get
489 # dropped accidentally.
490 if strict and line.strip() != '':
491 self.fatal (filename, 'unrecognised line: ' + line[:-1])
493 # Output a segment of text.
494 def output_segment (self, segment):
495 with safe_open (segment.filename) as file:
496 file.seek (segment.start)
497 for i in range (segment.lines):
498 sys.stdout.write (file.readline())
500 # Output a summary giving the number of times each type of result has
501 # been seen.
502 def output_summary (self, tool, counts):
503 for i in range (len (self.count_names)):
504 name = self.count_names[i]
505 # dejagnu only prints result types that were seen at least once,
506 # but acats always prints a number of unexpected failures.
507 if (counts[i] > 0
508 or (tool.name == 'acats'
509 and name.startswith ('# of unexpected failures'))):
510 sys.stdout.write ('%s%d\n' % (name, counts[i]))
512 # Output unified .log or .sum information for a particular variation,
513 # with a summary at the end.
514 def output_variation (self, tool, variation):
515 self.output_segment (variation.header)
516 for harness in sorted (variation.harnesses.values(),
517 key = attrgetter ('name')):
518 sys.stdout.write ('Running ' + harness.name + ' ...\n')
519 if self.do_sum:
520 harness.results.sort()
521 for (key, line) in harness.results:
522 sys.stdout.write (line)
523 else:
524 # Rearrange the log segments into test order (but without
525 # rearranging text within those segments).
526 for key in sorted (harness.segments.keys()):
527 self.output_segment (harness.segments[key])
528 for segment in harness.empty:
529 self.output_segment (segment)
530 if len (self.variations) > 1:
531 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
532 + variation.name + ' ===\n\n')
533 self.output_summary (tool, variation.counts)
535 # Output unified .log or .sum information for a particular tool,
536 # with a summary at the end.
537 def output_tool (self, tool):
538 counts = self.zero_counts()
539 if tool.name == 'acats':
540 # acats doesn't use variations, so just output everything.
541 # It also has a different approach to whitespace.
542 sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
543 for variation in tool.variations.values():
544 self.output_variation (tool, variation)
545 self.accumulate_counts (counts, variation.counts)
546 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
547 else:
548 # Output the results in the usual dejagnu runtest format.
549 sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
550 'Schedule of variations:\n')
551 for name in self.variations:
552 if name in tool.variations:
553 sys.stdout.write (' ' + name + '\n')
554 sys.stdout.write ('\n')
555 for name in self.variations:
556 if name in tool.variations:
557 variation = tool.variations[name]
558 sys.stdout.write ('Running target '
559 + variation.name + '\n')
560 self.output_variation (tool, variation)
561 self.accumulate_counts (counts, variation.counts)
562 sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
563 self.output_summary (tool, counts)
565 def main (self):
566 self.parse_cmdline()
567 try:
568 # Parse the input files.
569 for filename in self.files:
570 with safe_open (filename) as file:
571 self.parse_file (filename, file)
573 # Decide what to output.
574 if len (self.variations) == 0:
575 self.variations = sorted (self.known_variations)
576 else:
577 for name in self.variations:
578 if name not in self.known_variations:
579 self.fatal (None, 'no results for ' + name)
580 if len (self.tools) == 0:
581 self.tools = sorted (self.runs.keys())
583 # Output the header.
584 if self.start_line:
585 sys.stdout.write (self.start_line[1])
586 sys.stdout.write (self.native_line)
587 sys.stdout.write (self.target_line)
588 sys.stdout.write (self.host_line)
589 sys.stdout.write (self.acats_premable)
591 # Output the main body.
592 for name in self.tools:
593 if name not in self.runs:
594 self.fatal (None, 'no results for ' + name)
595 self.output_tool (self.runs[name])
597 # Output the footer.
598 if len (self.acats_failures) > 0:
599 sys.stdout.write ('*** FAILURES: '
600 + ' '.join (self.acats_failures) + '\n')
601 sys.stdout.write (self.version_output)
602 if self.end_line:
603 sys.stdout.write (self.end_line[1])
604 except IOError as e:
605 self.fatal (e.filename, e.strerror)
607 Prog().main()