contrib/dg-extract-results.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (C) 2014 Free Software Foundation, Inc.
   4 #
   5 # This script is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3, or (at your option)
   8 # any later version.
   9
  10 import sys
  11 import getopt
  12 import re
  13 import io
  14 from datetime import datetime
  15 from operator import attrgetter
  16
  17 # True if unrecognised lines should cause a fatal error.  Might want to turn
  18 # this on by default later.
  19 strict = False
  20
  21 # True if the order of .log segments should match the .sum file, false if
  22 # they should keep the original order.
  23 sort_logs = True
  24
  25 # A version of open() that is safe against whatever binary output
  26 # might be added to the log.
  27 def safe_open (filename):
  28     if sys.version_info >= (3, 0):
  29         return open (filename, 'r', errors = 'surrogateescape')
  30     return open (filename, 'r')
  31
  32 # Force stdout to handle escape sequences from a safe_open file.
  33 if sys.version_info >= (3, 0):
  34     sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
  35                                    errors = 'surrogateescape')
  36
  37 class Named:
  38     def __init__ (self, name):
  39         self.name = name
  40
  41 class ToolRun (Named):
  42     def __init__ (self, name):
  43         Named.__init__ (self, name)
  44         # The variations run for this tool, mapped by --target_board name.
  45         self.variations = dict()
  46
  47     # Return the VariationRun for variation NAME.
  48     def get_variation (self, name):
  49         if name not in self.variations:
  50             self.variations[name] = VariationRun (name)
  51         return self.variations[name]
  52
  53 class VariationRun (Named):
  54     def __init__ (self, name):
  55         Named.__init__ (self, name)
  56         # A segment of text before the harness runs start, describing which
  57         # baseboard files were loaded for the target.
  58         self.header = None
  59         # The harnesses run for this variation, mapped by filename.
  60         self.harnesses = dict()
  61         # A list giving the number of times each type of result has
  62         # been seen.
  63         self.counts = []
  64
  65     # Return the HarnessRun for harness NAME.
  66     def get_harness (self, name):
  67         if name not in self.harnesses:
  68             self.harnesses[name] = HarnessRun (name)
  69         return self.harnesses[name]
  70
  71 class HarnessRun (Named):
  72     def __init__ (self, name):
  73         Named.__init__ (self, name)
  74         # Segments of text that make up the harness run, mapped by a test-based
  75         # key that can be used to order them.
  76         self.segments = dict()
  77         # Segments of text that make up the harness run but which have
  78         # no recognized test results.  These are typically harnesses that
  79         # are completely skipped for the target.
  80         self.empty = []
  81         # A list of results.  Each entry is a pair in which the first element
  82         # is a unique sorting key and in which the second is the full
  83         # PASS/FAIL line.
  84         self.results = []
  85
  86     # Add a segment of text to the harness run.  If the segment includes
  87     # test results, KEY is an example of one of them, and can be used to
  88     # combine the individual segments in order.  If the segment has no
  89     # test results (e.g. because the harness doesn't do anything for the
  90     # current configuration) then KEY is None instead.  In that case
  91     # just collect the segments in the order that we see them.
  92     def add_segment (self, key, segment):
  93         if key:
  94             assert key not in self.segments
  95             self.segments[key] = segment
  96         else:
  97             self.empty.append (segment)
  98
  99 class Segment:
 100     def __init__ (self, filename, start):
 101         self.filename = filename
 102         self.start = start
 103         self.lines = 0
 104
 105 class Prog:
 106     def __init__ (self):
 107         # The variations specified on the command line.
 108         self.variations = []
 109         # The variations seen in the input files.
 110         self.known_variations = set()
 111         # The tools specified on the command line.
 112         self.tools = []
 113         # Whether to create .sum rather than .log output.
 114         self.do_sum = True
 115         # Regexps used while parsing.
 116         self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
 117         self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
 118         self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
 119                                      r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
 120                                      r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)')
 121         self.completed_re = re.compile (r'.* completed at (.*)')
 122         # Pieces of text to write at the head of the output.
 123         # start_line is a pair in which the first element is a datetime
 124         # and in which the second is the associated 'Test Run By' line.
 125         self.start_line = None
 126         self.native_line = ''
 127         self.target_line = ''
 128         self.host_line = ''
 129         self.acats_premable = ''
 130         # Pieces of text to write at the end of the output.
 131         # end_line is like start_line but for the 'runtest completed' line.
 132         self.acats_failures = []
 133         self.version_output = ''
 134         self.end_line = None
 135         # Known summary types.
 136         self.count_names = [
 137             '# of DejaGnu errors\t\t',
 138             '# of expected passes\t\t',
 139             '# of unexpected failures\t',
 140             '# of unexpected successes\t',
 141             '# of expected failures\t\t',
 142             '# of unknown successes\t\t',
 143             '# of known failures\t\t',
 144             '# of untested testcases\t\t',
 145             '# of unresolved testcases\t',
 146             '# of unsupported tests\t\t',
 147             '# of paths in test names\t',
 148             '# of duplicate test names\t'
 149         ]
 150         self.runs = dict()
 151
 152     def usage (self):
 153         name = sys.argv[0]
 154         sys.stderr.write ('Usage: ' + name
 155                           + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
 156
 157     tool           The tool (e.g. g++, libffi) for which to create a
 158                    new test summary file.  If not specified then output
 159                    is created for all tools.
 160     variant-list   One or more test variant names.  If the list is
 161                    not specified then one is constructed from all
 162                    variants in the files for <tool>.
 163     sum-file       A test summary file with the format of those
 164                    created by runtest from DejaGnu.
 165     If -L is used, merge *.log files instead of *.sum.  In this
 166     mode the exact order of lines may not be preserved, just different
 167     Running *.exp chunks should be in correct order.
 168 ''')
 169         sys.exit (1)
 170
 171     def fatal (self, what, string):
 172         if not what:
 173             what = sys.argv[0]
 174         sys.stderr.write (what + ': ' + string + '\n')
 175         sys.exit (1)
 176
 177     # Parse the command-line arguments.
 178     def parse_cmdline (self):
 179         try:
 180             (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
 181             if len (self.files) == 0:
 182                 self.usage()
 183             for (option, value) in options:
 184                 if option == '-l':
 185                     self.variations.append (value)
 186                 elif option == '-t':
 187                     self.tools.append (value)
 188                 else:
 189                     self.do_sum = False
 190         except getopt.GetoptError as e:
 191             self.fatal (None, e.msg)
 192
 193     # Try to parse time string TIME, returning an arbitrary time on failure.
 194     # Getting this right is just a nice-to-have so failures should be silent.
 195     def parse_time (self, time):
 196         try:
 197             return datetime.strptime (time, '%c')
 198         except ValueError:
 199             return datetime.now()
 200
 201     # Parse an integer and abort on failure.
 202     def parse_int (self, filename, value):
 203         try:
 204             return int (value)
 205         except ValueError:
 206             self.fatal (filename, 'expected an integer, got: ' + value)
 207
 208     # Return a list that represents no test results.
 209     def zero_counts (self):
 210         return [0 for x in self.count_names]
 211
 212     # Return the ToolRun for tool NAME.
 213     def get_tool (self, name):
 214         if name not in self.runs:
 215             self.runs[name] = ToolRun (name)
 216         return self.runs[name]
 217
 218     # Add the result counts in list FROMC to TOC.
 219     def accumulate_counts (self, toc, fromc):
 220         for i in range (len (self.count_names)):
 221             toc[i] += fromc[i]
 222
 223     # Parse the list of variations after 'Schedule of variations:'.
 224     # Return the number seen.
 225     def parse_variations (self, filename, file):
 226         num_variations = 0
 227         while True:
 228             line = file.readline()
 229             if line == '':
 230                 self.fatal (filename, 'could not parse variation list')
 231             if line == '\n':
 232                 break
 233             self.known_variations.add (line.strip())
 234             num_variations += 1
 235         return num_variations
 236
 237     # Parse from the first line after 'Running target ...' to the end
 238     # of the run's summary.
 239     def parse_run (self, filename, file, tool, variation, num_variations):
 240         header = None
 241         harness = None
 242         segment = None
 243         final_using = 0
 244         has_warning = 0
 245
 246         # If this is the first run for this variation, add any text before
 247         # the first harness to the header.
 248         if not variation.header:
 249             segment = Segment (filename, file.tell())
 250             variation.header = segment
 251
 252         # Parse the rest of the summary (the '# of ' lines).
 253         if len (variation.counts) == 0:
 254             variation.counts = self.zero_counts()
 255
 256         # Parse up until the first line of the summary.
 257         if num_variations == 1:
 258             end = '\t\t=== ' + tool.name + ' Summary ===\n'
 259         else:
 260             end = ('\t\t=== ' + tool.name + ' Summary for '
 261                    + variation.name + ' ===\n')
 262         while True:
 263             line = file.readline()
 264             if line == '':
 265                 self.fatal (filename, 'no recognised summary line')
 266             if line == end:
 267                 break
 268
 269             # Look for the start of a new harness.
 270             if line.startswith ('Running ') and line.endswith (' ...\n'):
 271                 # Close off the current harness segment, if any.
 272                 if harness:
 273                     segment.lines -= final_using
 274                     harness.add_segment (first_key, segment)
 275                 name = line[len ('Running '):-len(' ...\n')]
 276                 harness = variation.get_harness (name)
 277                 segment = Segment (filename, file.tell())
 278                 first_key = None
 279                 final_using = 0
 280                 continue
 281
 282             # Record test results.  Associate the first test result with
 283             # the harness segment, so that if a run for a particular harness
 284             # has been split up, we can reassemble the individual segments
 285             # in a sensible order.
 286             #
 287             # dejagnu sometimes issues warnings about the testing environment
 288             # before running any tests.  Treat them as part of the header
 289             # rather than as a test result.
 290             match = self.result_re.match (line)
 291             if match and (harness or not line.startswith ('WARNING:')):
 292                 if not harness:
 293                     self.fatal (filename, 'saw test result before harness name')
 294                 name = match.group (2)
 295                 # Ugly hack to get the right order for gfortran.
 296                 if name.startswith ('gfortran.dg/g77/'):
 297                     name = 'h' + name
 298                 # If we have a time out warning, make sure it appears
 299                 # before the following testcase diagnostic: we insert
 300                 # the testname before 'program' so that sort faces a
 301                 # list of testnames.
 302                 if line.startswith ('WARNING: program timed out'):
 303                   has_warning = 1
 304                 else:
 305                   if has_warning == 1:
 306                       key = (name, len (harness.results))
 307                       myline = 'WARNING: %s program timed out.\n' % name
 308                       harness.results.append ((key, myline))
 309                       has_warning = 0
 310                   key = (name, len (harness.results))
 311                   harness.results.append ((key, line))
 312                   if not first_key and sort_logs:
 313                       first_key = key
 314                 if line.startswith ('ERROR: (DejaGnu)'):
 315                     for i in range (len (self.count_names)):
 316                         if 'DejaGnu errors' in self.count_names[i]:
 317                             variation.counts[i] += 1
 318                             break
 319
 320             # 'Using ...' lines are only interesting in a header.  Splitting
 321             # the test up into parallel runs leads to more 'Using ...' lines
 322             # than there would be in a single log.
 323             if line.startswith ('Using '):
 324                 final_using += 1
 325             else:
 326                 final_using = 0
 327
 328             # Add other text to the current segment, if any.
 329             if segment:
 330                 segment.lines += 1
 331
 332         # Close off the final harness segment, if any.
 333         if harness:
 334             segment.lines -= final_using
 335             harness.add_segment (first_key, segment)
 336
 337         while True:
 338             before = file.tell()
 339             line = file.readline()
 340             if line == '':
 341                 break
 342             if line == '\n':
 343                 continue
 344             if not line.startswith ('# '):
 345                 file.seek (before)
 346                 break
 347             found = False
 348             for i in range (len (self.count_names)):
 349                 if line.startswith (self.count_names[i]):
 350                     count = line[len (self.count_names[i]):-1].strip()
 351                     variation.counts[i] += self.parse_int (filename, count)
 352                     found = True
 353                     break
 354             if not found:
 355                 self.fatal (filename, 'unknown test result: ' + line[:-1])
 356
 357     # Parse an acats run, which uses a different format from dejagnu.
 358     # We have just skipped over '=== acats configuration ==='.
 359     def parse_acats_run (self, filename, file):
 360         # Parse the preamble, which describes the configuration and logs
 361         # the creation of support files.
 362         record = (self.acats_premable == '')
 363         if record:
 364             self.acats_premable = '\t\t=== acats configuration ===\n'
 365         while True:
 366             line = file.readline()
 367             if line == '':
 368                 self.fatal (filename, 'could not parse acats preamble')
 369             if line == '\t\t=== acats tests ===\n':
 370                 break
 371             if record:
 372                 self.acats_premable += line
 373
 374         # Parse the test results themselves, using a dummy variation name.
 375         tool = self.get_tool ('acats')
 376         variation = tool.get_variation ('none')
 377         self.parse_run (filename, file, tool, variation, 1)
 378
 379         # Parse the failure list.
 380         while True:
 381             before = file.tell()
 382             line = file.readline()
 383             if line.startswith ('*** FAILURES: '):
 384                 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
 385                 continue
 386             file.seek (before)
 387             break
 388
 389     # Parse the final summary at the end of a log in order to capture
 390     # the version output that follows it.
 391     def parse_final_summary (self, filename, file):
 392         record = (self.version_output == '')
 393         while True:
 394             line = file.readline()
 395             if line == '':
 396                 break
 397             if line.startswith ('# of '):
 398                 continue
 399             if record:
 400                 self.version_output += line
 401             if line == '\n':
 402                 break
 403
 404     # Parse a .log or .sum file.
 405     def parse_file (self, filename, file):
 406         tool = None
 407         target = None
 408         num_variations = 1
 409         while True:
 410             line = file.readline()
 411             if line == '':
 412                 return
 413
 414             # Parse the list of variations, which comes before the test
 415             # runs themselves.
 416             if line.startswith ('Schedule of variations:'):
 417                 num_variations = self.parse_variations (filename, file)
 418                 continue
 419
 420             # Parse a testsuite run for one tool/variation combination.
 421             if line.startswith ('Running target '):
 422                 name = line[len ('Running target '):-1]
 423                 if not tool:
 424                     self.fatal (filename, 'could not parse tool name')
 425                 if name not in self.known_variations:
 426                     self.fatal (filename, 'unknown target: ' + name)
 427                 self.parse_run (filename, file, tool,
 428                                 tool.get_variation (name),
 429                                 num_variations)
 430                 # If there is only one variation then there is no separate
 431                 # summary for it.  Record any following version output.
 432                 if num_variations == 1:
 433                     self.parse_final_summary (filename, file)
 434                 continue
 435
 436             # Parse the start line.  In the case where several files are being
 437             # parsed, pick the one with the earliest time.
 438             match = self.test_run_re.match (line)
 439             if match:
 440                 time = self.parse_time (match.group (2))
 441                 if not self.start_line or self.start_line[0] > time:
 442                     self.start_line = (time, line)
 443                 continue
 444
 445             # Parse the form used for native testing.
 446             if line.startswith ('Native configuration is '):
 447                 self.native_line = line
 448                 continue
 449
 450             # Parse the target triplet.
 451             if line.startswith ('Target is '):
 452                 self.target_line = line
 453                 continue
 454
 455             # Parse the host triplet.
 456             if line.startswith ('Host   is '):
 457                 self.host_line = line
 458                 continue
 459
 460             # Parse the acats premable.
 461             if line == '\t\t=== acats configuration ===\n':
 462                 self.parse_acats_run (filename, file)
 463                 continue
 464
 465             # Parse the tool name.
 466             match = self.tool_re.match (line)
 467             if match:
 468                 tool = self.get_tool (match.group (1))
 469                 continue
 470
 471             # Skip over the final summary (which we instead create from
 472             # individual runs) and parse the version output.
 473             if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
 474                 if file.readline() != '\n':
 475                     self.fatal (filename, 'expected blank line after summary')
 476                 self.parse_final_summary (filename, file)
 477                 continue
 478
 479             # Parse the completion line.  In the case where several files
 480             # are being parsed, pick the one with the latest time.
 481             match = self.completed_re.match (line)
 482             if match:
 483                 time = self.parse_time (match.group (1))
 484                 if not self.end_line or self.end_line[0] < time:
 485                     self.end_line = (time, line)
 486                 continue
 487
 488             # Sanity check to make sure that important text doesn't get
 489             # dropped accidentally.
 490             if strict and line.strip() != '':
 491                 self.fatal (filename, 'unrecognised line: ' + line[:-1])
 492
 493     # Output a segment of text.
 494     def output_segment (self, segment):
 495         with safe_open (segment.filename) as file:
 496             file.seek (segment.start)
 497             for i in range (segment.lines):
 498                 sys.stdout.write (file.readline())
 499
 500     # Output a summary giving the number of times each type of result has
 501     # been seen.
 502     def output_summary (self, tool, counts):
 503         for i in range (len (self.count_names)):
 504             name = self.count_names[i]
 505             # dejagnu only prints result types that were seen at least once,
 506             # but acats always prints a number of unexpected failures.
 507             if (counts[i] > 0
 508                 or (tool.name == 'acats'
 509                     and name.startswith ('# of unexpected failures'))):
 510                 sys.stdout.write ('%s%d\n' % (name, counts[i]))
 511
 512     # Output unified .log or .sum information for a particular variation,
 513     # with a summary at the end.
 514     def output_variation (self, tool, variation):
 515         self.output_segment (variation.header)
 516         for harness in sorted (variation.harnesses.values(),
 517                                key = attrgetter ('name')):
 518             sys.stdout.write ('Running ' + harness.name + ' ...\n')
 519             if self.do_sum:
 520                 harness.results.sort()
 521                 for (key, line) in harness.results:
 522                     sys.stdout.write (line)
 523             else:
 524                 # Rearrange the log segments into test order (but without
 525                 # rearranging text within those segments).
 526                 for key in sorted (harness.segments.keys()):
 527                     self.output_segment (harness.segments[key])
 528                 for segment in harness.empty:
 529                     self.output_segment (segment)
 530         if len (self.variations) > 1:
 531             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
 532                               + variation.name + ' ===\n\n')
 533             self.output_summary (tool, variation.counts)
 534
 535     # Output unified .log or .sum information for a particular tool,
 536     # with a summary at the end.
 537     def output_tool (self, tool):
 538         counts = self.zero_counts()
 539         if tool.name == 'acats':
 540             # acats doesn't use variations, so just output everything.
 541             # It also has a different approach to whitespace.
 542             sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
 543             for variation in tool.variations.values():
 544                 self.output_variation (tool, variation)
 545                 self.accumulate_counts (counts, variation.counts)
 546             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
 547         else:
 548             # Output the results in the usual dejagnu runtest format.
 549             sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
 550                               'Schedule of variations:\n')
 551             for name in self.variations:
 552                 if name in tool.variations:
 553                     sys.stdout.write ('    ' + name + '\n')
 554             sys.stdout.write ('\n')
 555             for name in self.variations:
 556                 if name in tool.variations:
 557                     variation = tool.variations[name]
 558                     sys.stdout.write ('Running target '
 559                                       + variation.name + '\n')
 560                     self.output_variation (tool, variation)
 561                     self.accumulate_counts (counts, variation.counts)
 562             sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
 563         self.output_summary (tool, counts)
 564
 565     def main (self):
 566         self.parse_cmdline()
 567         try:
 568             # Parse the input files.
 569             for filename in self.files:
 570                 with safe_open (filename) as file:
 571                     self.parse_file (filename, file)
 572
 573             # Decide what to output.
 574             if len (self.variations) == 0:
 575                 self.variations = sorted (self.known_variations)
 576             else:
 577                 for name in self.variations:
 578                     if name not in self.known_variations:
 579                         self.fatal (None, 'no results for ' + name)
 580             if len (self.tools) == 0:
 581                 self.tools = sorted (self.runs.keys())
 582
 583             # Output the header.
 584             if self.start_line:
 585                 sys.stdout.write (self.start_line[1])
 586             sys.stdout.write (self.native_line)
 587             sys.stdout.write (self.target_line)
 588             sys.stdout.write (self.host_line)
 589             sys.stdout.write (self.acats_premable)
 590
 591             # Output the main body.
 592             for name in self.tools:
 593                 if name not in self.runs:
 594                     self.fatal (None, 'no results for ' + name)
 595                 self.output_tool (self.runs[name])
 596
 597             # Output the footer.
 598             if len (self.acats_failures) > 0:
 599                 sys.stdout.write ('*** FAILURES: '
 600                                   + ' '.join (self.acats_failures) + '\n')
 601             sys.stdout.write (self.version_output)
 602             if self.end_line:
 603                 sys.stdout.write (self.end_line[1])
 604         except IOError as e:
 605             self.fatal (e.filename, e.strerror)
 606
 607 Prog().main()