Automatic date update in version.in
[binutils-gdb.git] / contrib / dg-extract-results.py
blob0fe3c5f15199dae2f9fd704d7f6efae42e81cf29
1 #!/usr/bin/python
3 # Copyright (C) 2014-2024 Free Software Foundation, Inc.
5 # This script is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3, or (at your option)
8 # any later version.
10 import sys
11 import getopt
12 import re
13 import io
14 from datetime import datetime
15 from operator import attrgetter
17 # True if unrecognised lines should cause a fatal error. Might want to turn
18 # this on by default later.
19 strict = False
21 # True if the order of .log segments should match the .sum file, false if
22 # they should keep the original order.
23 sort_logs = True
25 # A version of open() that is safe against whatever binary output
26 # might be added to the log.
27 def safe_open (filename):
28 if sys.version_info >= (3, 0):
29 return open (filename, 'r', errors = 'surrogateescape')
30 return open (filename, 'r')
32 # Force stdout to handle escape sequences from a safe_open file.
33 if sys.version_info >= (3, 0):
34 sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
35 errors = 'surrogateescape')
37 class Named:
38 def __init__ (self, name):
39 self.name = name
41 class ToolRun (Named):
42 def __init__ (self, name):
43 Named.__init__ (self, name)
44 # The variations run for this tool, mapped by --target_board name.
45 self.variations = dict()
47 # Return the VariationRun for variation NAME.
48 def get_variation (self, name):
49 if name not in self.variations:
50 self.variations[name] = VariationRun (name)
51 return self.variations[name]
53 class VariationRun (Named):
54 def __init__ (self, name):
55 Named.__init__ (self, name)
56 # A segment of text before the harness runs start, describing which
57 # baseboard files were loaded for the target.
58 self.header = None
59 # The harnesses run for this variation, mapped by filename.
60 self.harnesses = dict()
61 # A list giving the number of times each type of result has
62 # been seen.
63 self.counts = []
65 # Return the HarnessRun for harness NAME.
66 def get_harness (self, name):
67 if name not in self.harnesses:
68 self.harnesses[name] = HarnessRun (name)
69 return self.harnesses[name]
71 class HarnessRun (Named):
72 def __init__ (self, name):
73 Named.__init__ (self, name)
74 # Segments of text that make up the harness run, mapped by a test-based
75 # key that can be used to order them.
76 self.segments = dict()
77 # Segments of text that make up the harness run but which have
78 # no recognized test results. These are typically harnesses that
79 # are completely skipped for the target.
80 self.empty = []
81 # A list of results. Each entry is a pair in which the first element
82 # is a unique sorting key and in which the second is the full
83 # PASS/FAIL line.
84 self.results = []
86 # Add a segment of text to the harness run. If the segment includes
87 # test results, KEY is an example of one of them, and can be used to
88 # combine the individual segments in order. If the segment has no
89 # test results (e.g. because the harness doesn't do anything for the
90 # current configuration) then KEY is None instead. In that case
91 # just collect the segments in the order that we see them.
92 def add_segment (self, key, segment):
93 if key:
94 assert key not in self.segments
95 self.segments[key] = segment
96 else:
97 self.empty.append (segment)
99 class Segment:
100 def __init__ (self, filename, start):
101 self.filename = filename
102 self.start = start
103 self.lines = 0
105 class Prog:
106 def __init__ (self):
107 # The variations specified on the command line.
108 self.variations = []
109 # The variations seen in the input files.
110 self.known_variations = set()
111 # The tools specified on the command line.
112 self.tools = []
113 # Whether to create .sum rather than .log output.
114 self.do_sum = True
115 # Regexps used while parsing.
116 self.test_run_re = re.compile (r'^Test run by (\S+) on (.*)$',
117 re.IGNORECASE)
118 self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
119 self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
120 r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
121 r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)')
122 self.completed_re = re.compile (r'.* completed at (.*)')
123 # Pieces of text to write at the head of the output.
124 # start_line is a pair in which the first element is a datetime
125 # and in which the second is the associated 'Test Run By' line.
126 self.start_line = None
127 self.native_line = ''
128 self.target_line = ''
129 self.host_line = ''
130 self.acats_premable = ''
131 # Pieces of text to write at the end of the output.
132 # end_line is like start_line but for the 'runtest completed' line.
133 self.acats_failures = []
134 self.version_output = ''
135 self.end_line = None
136 # Known summary types.
137 self.count_names = [
138 '# of DejaGnu errors\t\t',
139 '# of expected passes\t\t',
140 '# of unexpected failures\t',
141 '# of unexpected successes\t',
142 '# of expected failures\t\t',
143 '# of unknown successes\t\t',
144 '# of known failures\t\t',
145 '# of untested testcases\t\t',
146 '# of unresolved testcases\t',
147 '# of unsupported tests\t\t',
148 '# of paths in test names\t',
149 '# of duplicate test names\t'
151 self.runs = dict()
153 def usage (self):
154 name = sys.argv[0]
155 sys.stderr.write ('Usage: ' + name
156 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
158 tool The tool (e.g. g++, libffi) for which to create a
159 new test summary file. If not specified then output
160 is created for all tools.
161 variant-list One or more test variant names. If the list is
162 not specified then one is constructed from all
163 variants in the files for <tool>.
164 sum-file A test summary file with the format of those
165 created by runtest from DejaGnu.
166 If -L is used, merge *.log files instead of *.sum. In this
167 mode the exact order of lines may not be preserved, just different
168 Running *.exp chunks should be in correct order.
169 ''')
170 sys.exit (1)
172 def fatal (self, what, string):
173 if not what:
174 what = sys.argv[0]
175 sys.stderr.write (what + ': ' + string + '\n')
176 sys.exit (1)
178 # Parse the command-line arguments.
179 def parse_cmdline (self):
180 try:
181 (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
182 if len (self.files) == 0:
183 self.usage()
184 for (option, value) in options:
185 if option == '-l':
186 self.variations.append (value)
187 elif option == '-t':
188 self.tools.append (value)
189 else:
190 self.do_sum = False
191 except getopt.GetoptError as e:
192 self.fatal (None, e.msg)
194 # Try to parse time string TIME, returning an arbitrary time on failure.
195 # Getting this right is just a nice-to-have so failures should be silent.
196 def parse_time (self, time):
197 try:
198 return datetime.strptime (time, '%c')
199 except ValueError:
200 return datetime.now()
202 # Parse an integer and abort on failure.
203 def parse_int (self, filename, value):
204 try:
205 return int (value)
206 except ValueError:
207 self.fatal (filename, 'expected an integer, got: ' + value)
209 # Return a list that represents no test results.
210 def zero_counts (self):
211 return [0 for x in self.count_names]
213 # Return the ToolRun for tool NAME.
214 def get_tool (self, name):
215 if name not in self.runs:
216 self.runs[name] = ToolRun (name)
217 return self.runs[name]
219 # Add the result counts in list FROMC to TOC.
220 def accumulate_counts (self, toc, fromc):
221 for i in range (len (self.count_names)):
222 toc[i] += fromc[i]
224 # Parse the list of variations after 'Schedule of variations:'.
225 # Return the number seen.
226 def parse_variations (self, filename, file):
227 num_variations = 0
228 while True:
229 line = file.readline()
230 if line == '':
231 self.fatal (filename, 'could not parse variation list')
232 if line == '\n':
233 break
234 self.known_variations.add (line.strip())
235 num_variations += 1
236 return num_variations
238 # Parse from the first line after 'Running target ...' to the end
239 # of the run's summary.
240 def parse_run (self, filename, file, tool, variation, num_variations):
241 header = None
242 harness = None
243 segment = None
244 final_using = 0
245 has_warning = 0
247 # If this is the first run for this variation, add any text before
248 # the first harness to the header.
249 if not variation.header:
250 segment = Segment (filename, file.tell())
251 variation.header = segment
253 # Parse the rest of the summary (the '# of ' lines).
254 if len (variation.counts) == 0:
255 variation.counts = self.zero_counts()
257 # Parse up until the first line of the summary.
258 if num_variations == 1:
259 end = '\t\t=== ' + tool.name + ' Summary ===\n'
260 else:
261 end = ('\t\t=== ' + tool.name + ' Summary for '
262 + variation.name + ' ===\n')
263 while True:
264 line = file.readline()
265 if line == '':
266 self.fatal (filename, 'no recognised summary line')
267 if line == end:
268 break
270 # Look for the start of a new harness.
271 if line.startswith ('Running ') and line.endswith (' ...\n'):
272 # Close off the current harness segment, if any.
273 if harness:
274 segment.lines -= final_using
275 harness.add_segment (first_key, segment)
276 name = line[len ('Running '):-len(' ...\n')]
277 harness = variation.get_harness (name)
278 segment = Segment (filename, file.tell())
279 first_key = None
280 final_using = 0
281 continue
283 # Record test results. Associate the first test result with
284 # the harness segment, so that if a run for a particular harness
285 # has been split up, we can reassemble the individual segments
286 # in a sensible order.
288 # dejagnu sometimes issues warnings about the testing environment
289 # before running any tests. Treat them as part of the header
290 # rather than as a test result.
291 match = self.result_re.match (line)
292 if match and (harness or not line.startswith ('WARNING:')):
293 if not harness:
294 self.fatal (filename, 'saw test result before harness name')
295 name = match.group (2)
296 # Ugly hack to get the right order for gfortran.
297 if name.startswith ('gfortran.dg/g77/'):
298 name = 'h' + name
299 # If we have a time out warning, make sure it appears
300 # before the following testcase diagnostic: we insert
301 # the testname before 'program' so that sort faces a
302 # list of testnames.
303 if line.startswith ('WARNING: program timed out'):
304 has_warning = 1
305 else:
306 if has_warning == 1:
307 key = (name, len (harness.results))
308 myline = 'WARNING: %s program timed out.\n' % name
309 harness.results.append ((key, myline))
310 has_warning = 0
311 key = (name, len (harness.results))
312 harness.results.append ((key, line))
313 if not first_key and sort_logs:
314 first_key = key
315 if line.startswith ('ERROR: (DejaGnu)'):
316 for i in range (len (self.count_names)):
317 if 'DejaGnu errors' in self.count_names[i]:
318 variation.counts[i] += 1
319 break
321 # 'Using ...' lines are only interesting in a header. Splitting
322 # the test up into parallel runs leads to more 'Using ...' lines
323 # than there would be in a single log.
324 if line.startswith ('Using '):
325 final_using += 1
326 else:
327 final_using = 0
329 # Add other text to the current segment, if any.
330 if segment:
331 segment.lines += 1
333 # Close off the final harness segment, if any.
334 if harness:
335 segment.lines -= final_using
336 harness.add_segment (first_key, segment)
338 while True:
339 before = file.tell()
340 line = file.readline()
341 if line == '':
342 break
343 if line == '\n':
344 continue
345 if not line.startswith ('# '):
346 file.seek (before)
347 break
348 found = False
349 for i in range (len (self.count_names)):
350 if line.startswith (self.count_names[i]):
351 count = line[len (self.count_names[i]):-1].strip()
352 variation.counts[i] += self.parse_int (filename, count)
353 found = True
354 break
355 if not found:
356 self.fatal (filename, 'unknown test result: ' + line[:-1])
358 # Parse an acats run, which uses a different format from dejagnu.
359 # We have just skipped over '=== acats configuration ==='.
360 def parse_acats_run (self, filename, file):
361 # Parse the preamble, which describes the configuration and logs
362 # the creation of support files.
363 record = (self.acats_premable == '')
364 if record:
365 self.acats_premable = '\t\t=== acats configuration ===\n'
366 while True:
367 line = file.readline()
368 if line == '':
369 self.fatal (filename, 'could not parse acats preamble')
370 if line == '\t\t=== acats tests ===\n':
371 break
372 if record:
373 self.acats_premable += line
375 # Parse the test results themselves, using a dummy variation name.
376 tool = self.get_tool ('acats')
377 variation = tool.get_variation ('none')
378 self.parse_run (filename, file, tool, variation, 1)
380 # Parse the failure list.
381 while True:
382 before = file.tell()
383 line = file.readline()
384 if line.startswith ('*** FAILURES: '):
385 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
386 continue
387 file.seek (before)
388 break
390 # Parse the final summary at the end of a log in order to capture
391 # the version output that follows it.
392 def parse_final_summary (self, filename, file):
393 record = (self.version_output == '')
394 while True:
395 line = file.readline()
396 if line == '':
397 break
398 if line.startswith ('# of '):
399 continue
400 if record:
401 self.version_output += line
402 if line == '\n':
403 break
405 # Parse a .log or .sum file.
406 def parse_file (self, filename, file):
407 tool = None
408 target = None
409 num_variations = 1
410 while True:
411 line = file.readline()
412 if line == '':
413 return
415 # Parse the list of variations, which comes before the test
416 # runs themselves.
417 if line.startswith ('Schedule of variations:'):
418 num_variations = self.parse_variations (filename, file)
419 continue
421 # Parse a testsuite run for one tool/variation combination.
422 if line.startswith ('Running target '):
423 name = line[len ('Running target '):-1]
424 if not tool:
425 self.fatal (filename, 'could not parse tool name')
426 if name not in self.known_variations:
427 self.fatal (filename, 'unknown target: ' + name)
428 self.parse_run (filename, file, tool,
429 tool.get_variation (name),
430 num_variations)
431 # If there is only one variation then there is no separate
432 # summary for it. Record any following version output.
433 if num_variations == 1:
434 self.parse_final_summary (filename, file)
435 continue
437 # Parse the start line. In the case where several files are being
438 # parsed, pick the one with the earliest time.
439 match = self.test_run_re.match (line)
440 if match:
441 time = self.parse_time (match.group (2))
442 if not self.start_line or self.start_line[0] > time:
443 self.start_line = (time, line)
444 continue
446 # Parse the form used for native testing.
447 if line.startswith ('Native configuration is '):
448 self.native_line = line
449 continue
451 # Parse the target triplet.
452 if line.startswith ('Target is '):
453 self.target_line = line
454 continue
456 # Parse the host triplet.
457 if line.startswith ('Host is '):
458 self.host_line = line
459 continue
461 # Parse the acats premable.
462 if line == '\t\t=== acats configuration ===\n':
463 self.parse_acats_run (filename, file)
464 continue
466 # Parse the tool name.
467 match = self.tool_re.match (line)
468 if match:
469 tool = self.get_tool (match.group (1))
470 continue
472 # Skip over the final summary (which we instead create from
473 # individual runs) and parse the version output.
474 if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
475 if file.readline() != '\n':
476 self.fatal (filename, 'expected blank line after summary')
477 self.parse_final_summary (filename, file)
478 continue
480 # Parse the completion line. In the case where several files
481 # are being parsed, pick the one with the latest time.
482 match = self.completed_re.match (line)
483 if match:
484 time = self.parse_time (match.group (1))
485 if not self.end_line or self.end_line[0] < time:
486 self.end_line = (time, line)
487 continue
489 # Sanity check to make sure that important text doesn't get
490 # dropped accidentally.
491 if strict and line.strip() != '':
492 self.fatal (filename, 'unrecognised line: ' + line[:-1])
494 # Output a segment of text.
495 def output_segment (self, segment):
496 with safe_open (segment.filename) as file:
497 file.seek (segment.start)
498 for i in range (segment.lines):
499 sys.stdout.write (file.readline())
501 # Output a summary giving the number of times each type of result has
502 # been seen.
503 def output_summary (self, tool, counts):
504 for i in range (len (self.count_names)):
505 name = self.count_names[i]
506 # dejagnu only prints result types that were seen at least once,
507 # but acats always prints a number of unexpected failures.
508 if (counts[i] > 0
509 or (tool.name == 'acats'
510 and name.startswith ('# of unexpected failures'))):
511 sys.stdout.write ('%s%d\n' % (name, counts[i]))
513 # Output unified .log or .sum information for a particular variation,
514 # with a summary at the end.
515 def output_variation (self, tool, variation):
516 self.output_segment (variation.header)
517 for harness in sorted (variation.harnesses.values(),
518 key = attrgetter ('name')):
519 sys.stdout.write ('Running ' + harness.name + ' ...\n')
520 if self.do_sum:
521 harness.results.sort()
522 for (key, line) in harness.results:
523 sys.stdout.write (line)
524 else:
525 # Rearrange the log segments into test order (but without
526 # rearranging text within those segments).
527 for key in sorted (harness.segments.keys()):
528 self.output_segment (harness.segments[key])
529 for segment in harness.empty:
530 self.output_segment (segment)
531 if len (self.variations) > 1:
532 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
533 + variation.name + ' ===\n\n')
534 self.output_summary (tool, variation.counts)
536 # Output unified .log or .sum information for a particular tool,
537 # with a summary at the end.
538 def output_tool (self, tool):
539 counts = self.zero_counts()
540 if tool.name == 'acats':
541 # acats doesn't use variations, so just output everything.
542 # It also has a different approach to whitespace.
543 sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
544 for variation in tool.variations.values():
545 self.output_variation (tool, variation)
546 self.accumulate_counts (counts, variation.counts)
547 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
548 else:
549 # Output the results in the usual dejagnu runtest format.
550 sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
551 'Schedule of variations:\n')
552 for name in self.variations:
553 if name in tool.variations:
554 sys.stdout.write (' ' + name + '\n')
555 sys.stdout.write ('\n')
556 for name in self.variations:
557 if name in tool.variations:
558 variation = tool.variations[name]
559 sys.stdout.write ('Running target '
560 + variation.name + '\n')
561 self.output_variation (tool, variation)
562 self.accumulate_counts (counts, variation.counts)
563 sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
564 self.output_summary (tool, counts)
566 def main (self):
567 self.parse_cmdline()
568 try:
569 # Parse the input files.
570 for filename in self.files:
571 with safe_open (filename) as file:
572 self.parse_file (filename, file)
574 # Decide what to output.
575 if len (self.variations) == 0:
576 self.variations = sorted (self.known_variations)
577 else:
578 for name in self.variations:
579 if name not in self.known_variations:
580 self.fatal (None, 'no results for ' + name)
581 if len (self.tools) == 0:
582 self.tools = sorted (self.runs.keys())
584 # Output the header.
585 if self.start_line:
586 sys.stdout.write (self.start_line[1])
587 sys.stdout.write (self.native_line)
588 sys.stdout.write (self.target_line)
589 sys.stdout.write (self.host_line)
590 sys.stdout.write (self.acats_premable)
592 # Output the main body.
593 for name in self.tools:
594 if name not in self.runs:
595 self.fatal (None, 'no results for ' + name)
596 self.output_tool (self.runs[name])
598 # Output the footer.
599 if len (self.acats_failures) > 0:
600 sys.stdout.write ('*** FAILURES: '
601 + ' '.join (self.acats_failures) + '\n')
602 sys.stdout.write (self.version_output)
603 if self.end_line:
604 sys.stdout.write (self.end_line[1])
605 except IOError as e:
606 self.fatal (e.filename, e.strerror)
608 Prog().main()