contrib/dg-extract-results.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (C) 2014 Free Software Foundation, Inc.
   4 #
   5 # This script is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3, or (at your option)
   8 # any later version.
   9
  10 import sys
  11 import getopt
  12 import re
  13 from datetime import datetime
  14
  15 # True if unrecognised lines should cause a fatal error.  Might want to turn
  16 # this on by default later.
  17 strict = False
  18
  19 # True if the order of .log segments should match the .sum file, false if
  20 # they should keep the original order.
  21 sort_logs = True
  22
  23 class Named:
  24     def __init__ (self, name):
  25         self.name = name
  26
  27     def __cmp__ (self, other):
  28         return cmp (self.name, other.name)
  29
  30 class ToolRun (Named):
  31     def __init__ (self, name):
  32         Named.__init__ (self, name)
  33         # The variations run for this tool, mapped by --target_board name.
  34         self.variations = dict()
  35
  36     # Return the VariationRun for variation NAME.
  37     def get_variation (self, name):
  38         if name not in self.variations:
  39             self.variations[name] = VariationRun (name)
  40         return self.variations[name]
  41
  42 class VariationRun (Named):
  43     def __init__ (self, name):
  44         Named.__init__ (self, name)
  45         # A segment of text before the harness runs start, describing which
  46         # baseboard files were loaded for the target.
  47         self.header = None
  48         # The harnesses run for this variation, mapped by filename.
  49         self.harnesses = dict()
  50         # A list giving the number of times each type of result has
  51         # been seen.
  52         self.counts = []
  53
  54     # Return the HarnessRun for harness NAME.
  55     def get_harness (self, name):
  56         if name not in self.harnesses:
  57             self.harnesses[name] = HarnessRun (name)
  58         return self.harnesses[name]
  59
  60 class HarnessRun (Named):
  61     def __init__ (self, name):
  62         Named.__init__ (self, name)
  63         # Segments of text that make up the harness run, mapped by a test-based
  64         # key that can be used to order them.
  65         self.segments = dict()
  66         # Segments of text that make up the harness run but which have
  67         # no recognized test results.  These are typically harnesses that
  68         # are completely skipped for the target.
  69         self.empty = []
  70         # A list of results.  Each entry is a pair in which the first element
  71         # is a unique sorting key and in which the second is the full
  72         # PASS/FAIL line.
  73         self.results = []
  74
  75     # Add a segment of text to the harness run.  If the segment includes
  76     # test results, KEY is an example of one of them, and can be used to
  77     # combine the individual segments in order.  If the segment has no
  78     # test results (e.g. because the harness doesn't do anything for the
  79     # current configuration) then KEY is None instead.  In that case
  80     # just collect the segments in the order that we see them.
  81     def add_segment (self, key, segment):
  82         if key:
  83             assert key not in self.segments
  84             self.segments[key] = segment
  85         else:
  86             self.empty.append (segment)
  87
  88 class Segment:
  89     def __init__ (self, filename, start):
  90         self.filename = filename
  91         self.start = start
  92         self.lines = 0
  93
  94 class Prog:
  95     def __init__ (self):
  96         # The variations specified on the command line.
  97         self.variations = []
  98         # The variations seen in the input files.
  99         self.known_variations = set()
 100         # The tools specified on the command line.
 101         self.tools = []
 102         # Whether to create .sum rather than .log output.
 103         self.do_sum = True
 104         # Regexps used while parsing.
 105         self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
 106         self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
 107         self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
 108                                      r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
 109                                      r'|KFAIL):\s*(\S+)')
 110         self.completed_re = re.compile (r'.* completed at (.*)')
 111         # Pieces of text to write at the head of the output.
 112         # start_line is a pair in which the first element is a datetime
 113         # and in which the second is the associated 'Test Run By' line.
 114         self.start_line = None
 115         self.native_line = ''
 116         self.target_line = ''
 117         self.host_line = ''
 118         self.acats_premable = ''
 119         # Pieces of text to write at the end of the output.
 120         # end_line is like start_line but for the 'runtest completed' line.
 121         self.acats_failures = []
 122         self.version_output = ''
 123         self.end_line = None
 124         # Known summary types.
 125         self.count_names = [
 126             '# of expected passes\t\t',
 127             '# of unexpected failures\t',
 128             '# of unexpected successes\t',
 129             '# of expected failures\t\t',
 130             '# of unknown successes\t\t',
 131             '# of known failures\t\t',
 132             '# of untested testcases\t\t',
 133             '# of unresolved testcases\t',
 134             '# of unsupported tests\t\t'
 135         ]
 136         self.runs = dict()
 137
 138     def usage (self):
 139         name = sys.argv[0]
 140         sys.stderr.write ('Usage: ' + name
 141                           + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
 142
 143     tool           The tool (e.g. g++, libffi) for which to create a
 144                    new test summary file.  If not specified then output
 145                    is created for all tools.
 146     variant-list   One or more test variant names.  If the list is
 147                    not specified then one is constructed from all
 148                    variants in the files for <tool>.
 149     sum-file       A test summary file with the format of those
 150                    created by runtest from DejaGnu.
 151     If -L is used, merge *.log files instead of *.sum.  In this
 152     mode the exact order of lines may not be preserved, just different
 153     Running *.exp chunks should be in correct order.
 154 ''')
 155         sys.exit (1)
 156
 157     def fatal (self, what, string):
 158         if not what:
 159             what = sys.argv[0]
 160         sys.stderr.write (what + ': ' + string + '\n')
 161         sys.exit (1)
 162
 163     # Parse the command-line arguments.
 164     def parse_cmdline (self):
 165         try:
 166             (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
 167             if len (self.files) == 0:
 168                 self.usage()
 169             for (option, value) in options:
 170                 if option == '-l':
 171                     self.variations.append (value)
 172                 elif option == '-t':
 173                     self.tools.append (value)
 174                 else:
 175                     self.do_sum = False
 176         except getopt.GetoptError as e:
 177             self.fatal (None, e.msg)
 178
 179     # Try to parse time string TIME, returning an arbitrary time on failure.
 180     # Getting this right is just a nice-to-have so failures should be silent.
 181     def parse_time (self, time):
 182         try:
 183             return datetime.strptime (time, '%c')
 184         except ValueError:
 185             return datetime.now()
 186
 187     # Parse an integer and abort on failure.
 188     def parse_int (self, filename, value):
 189         try:
 190             return int (value)
 191         except ValueError:
 192             self.fatal (filename, 'expected an integer, got: ' + value)
 193
 194     # Return a list that represents no test results.
 195     def zero_counts (self):
 196         return [0 for x in self.count_names]
 197
 198     # Return the ToolRun for tool NAME.
 199     def get_tool (self, name):
 200         if name not in self.runs:
 201             self.runs[name] = ToolRun (name)
 202         return self.runs[name]
 203
 204     # Add the result counts in list FROMC to TOC.
 205     def accumulate_counts (self, toc, fromc):
 206         for i in range (len (self.count_names)):
 207             toc[i] += fromc[i]
 208
 209     # Parse the list of variations after 'Schedule of variations:'.
 210     # Return the number seen.
 211     def parse_variations (self, filename, file):
 212         num_variations = 0
 213         while True:
 214             line = file.readline()
 215             if line == '':
 216                 self.fatal (filename, 'could not parse variation list')
 217             if line == '\n':
 218                 break
 219             self.known_variations.add (line.strip())
 220             num_variations += 1
 221         return num_variations
 222
 223     # Parse from the first line after 'Running target ...' to the end
 224     # of the run's summary.
 225     def parse_run (self, filename, file, tool, variation, num_variations):
 226         header = None
 227         harness = None
 228         segment = None
 229         final_using = 0
 230
 231         # If this is the first run for this variation, add any text before
 232         # the first harness to the header.
 233         if not variation.header:
 234             segment = Segment (filename, file.tell())
 235             variation.header = segment
 236
 237         # Parse up until the first line of the summary.
 238         if num_variations == 1:
 239             end = '\t\t=== ' + tool.name + ' Summary ===\n'
 240         else:
 241             end = ('\t\t=== ' + tool.name + ' Summary for '
 242                    + variation.name + ' ===\n')
 243         while True:
 244             line = file.readline()
 245             if line == '':
 246                 self.fatal (filename, 'no recognised summary line')
 247             if line == end:
 248                 break
 249
 250             # Look for the start of a new harness.
 251             if line.startswith ('Running ') and line.endswith (' ...\n'):
 252                 # Close off the current harness segment, if any.
 253                 if harness:
 254                     segment.lines -= final_using
 255                     harness.add_segment (first_key, segment)
 256                 name = line[len ('Running '):-len(' ...\n')]
 257                 harness = variation.get_harness (name)
 258                 segment = Segment (filename, file.tell())
 259                 first_key = None
 260                 final_using = 0
 261                 continue
 262
 263             # Record test results.  Associate the first test result with
 264             # the harness segment, so that if a run for a particular harness
 265             # has been split up, we can reassemble the individual segments
 266             # in a sensible order.
 267             match = self.result_re.match (line)
 268             if match:
 269                 if not harness:
 270                     self.fatal (filename, 'saw test result before harness name')
 271                 name = match.group (2)
 272                 # Ugly hack to get the right order for gfortran.
 273                 if name.startswith ('gfortran.dg/g77/'):
 274                     name = 'h' + name
 275                 key = (name, len (harness.results))
 276                 harness.results.append ((key, line))
 277                 if not first_key and sort_logs:
 278                     first_key = key
 279
 280             # 'Using ...' lines are only interesting in a header.  Splitting
 281             # the test up into parallel runs leads to more 'Using ...' lines
 282             # than there would be in a single log.
 283             if line.startswith ('Using '):
 284                 final_using += 1
 285             else:
 286                 final_using = 0
 287
 288             # Add other text to the current segment, if any.
 289             if segment:
 290                 segment.lines += 1
 291
 292         # Close off the final harness segment, if any.
 293         if harness:
 294             segment.lines -= final_using
 295             harness.add_segment (first_key, segment)
 296
 297         # Parse the rest of the summary (the '# of ' lines).
 298         if len (variation.counts) == 0:
 299             variation.counts = self.zero_counts()
 300         while True:
 301             before = file.tell()
 302             line = file.readline()
 303             if line == '':
 304                 break
 305             if line == '\n':
 306                 continue
 307             if not line.startswith ('# '):
 308                 file.seek (before)
 309                 break
 310             found = False
 311             for i in range (len (self.count_names)):
 312                 if line.startswith (self.count_names[i]):
 313                     count = line[len (self.count_names[i]):-1].strip()
 314                     variation.counts[i] += self.parse_int (filename, count)
 315                     found = True
 316                     break
 317             if not found:
 318                 self.fatal (filename, 'unknown test result: ' + line[:-1])
 319
 320     # Parse an acats run, which uses a different format from dejagnu.
 321     # We have just skipped over '=== acats configuration ==='.
 322     def parse_acats_run (self, filename, file):
 323         # Parse the preamble, which describes the configuration and logs
 324         # the creation of support files.
 325         record = (self.acats_premable == '')
 326         if record:
 327             self.acats_premable = '\t\t=== acats configuration ===\n'
 328         while True:
 329             line = file.readline()
 330             if line == '':
 331                 self.fatal (filename, 'could not parse acats preamble')
 332             if line == '\t\t=== acats tests ===\n':
 333                 break
 334             if record:
 335                 self.acats_premable += line
 336
 337         # Parse the test results themselves, using a dummy variation name.
 338         tool = self.get_tool ('acats')
 339         variation = tool.get_variation ('none')
 340         self.parse_run (filename, file, tool, variation, 1)
 341
 342         # Parse the failure list.
 343         while True:
 344             before = file.tell()
 345             line = file.readline()
 346             if line.startswith ('*** FAILURES: '):
 347                 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
 348                 continue
 349             file.seek (before)
 350             break
 351
 352     # Parse the final summary at the end of a log in order to capture
 353     # the version output that follows it.
 354     def parse_final_summary (self, filename, file):
 355         record = (self.version_output == '')
 356         while True:
 357             line = file.readline()
 358             if line == '':
 359                 break
 360             if line.startswith ('# of '):
 361                 continue
 362             if record:
 363                 self.version_output += line
 364             if line == '\n':
 365                 break
 366
 367     # Parse a .log or .sum file.
 368     def parse_file (self, filename, file):
 369         tool = None
 370         target = None
 371         num_variations = 1
 372         while True:
 373             line = file.readline()
 374             if line == '':
 375                 return
 376
 377             # Parse the list of variations, which comes before the test
 378             # runs themselves.
 379             if line.startswith ('Schedule of variations:'):
 380                 num_variations = self.parse_variations (filename, file)
 381                 continue
 382
 383             # Parse a testsuite run for one tool/variation combination.
 384             if line.startswith ('Running target '):
 385                 name = line[len ('Running target '):-1]
 386                 if not tool:
 387                     self.fatal (filename, 'could not parse tool name')
 388                 if name not in self.known_variations:
 389                     self.fatal (filename, 'unknown target: ' + name)
 390                 self.parse_run (filename, file, tool,
 391                                 tool.get_variation (name),
 392                                 num_variations)
 393                 # If there is only one variation then there is no separate
 394                 # summary for it.  Record any following version output.
 395                 if num_variations == 1:
 396                     self.parse_final_summary (filename, file)
 397                 continue
 398
 399             # Parse the start line.  In the case where several files are being
 400             # parsed, pick the one with the earliest time.
 401             match = self.test_run_re.match (line)
 402             if match:
 403                 time = self.parse_time (match.group (2))
 404                 if not self.start_line or self.start_line[0] > time:
 405                     self.start_line = (time, line)
 406                 continue
 407
 408             # Parse the form used for native testing.
 409             if line.startswith ('Native configuration is '):
 410                 self.native_line = line
 411                 continue
 412
 413             # Parse the target triplet.
 414             if line.startswith ('Target is '):
 415                 self.target_line = line
 416                 continue
 417
 418             # Parse the host triplet.
 419             if line.startswith ('Host   is '):
 420                 self.host_line = line
 421                 continue
 422
 423             # Parse the acats premable.
 424             if line == '\t\t=== acats configuration ===\n':
 425                 self.parse_acats_run (filename, file)
 426                 continue
 427
 428             # Parse the tool name.
 429             match = self.tool_re.match (line)
 430             if match:
 431                 tool = self.get_tool (match.group (1))
 432                 continue
 433
 434             # Skip over the final summary (which we instead create from
 435             # individual runs) and parse the version output.
 436             if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
 437                 if file.readline() != '\n':
 438                     self.fatal (filename, 'expected blank line after summary')
 439                 self.parse_final_summary (filename, file)
 440                 continue
 441
 442             # Parse the completion line.  In the case where several files
 443             # are being parsed, pick the one with the latest time.
 444             match = self.completed_re.match (line)
 445             if match:
 446                 time = self.parse_time (match.group (1))
 447                 if not self.end_line or self.end_line[0] < time:
 448                     self.end_line = (time, line)
 449                 continue
 450
 451             # Sanity check to make sure that important text doesn't get
 452             # dropped accidentally.
 453             if strict and line.strip() != '':
 454                 self.fatal (filename, 'unrecognised line: ' + line[:-1])
 455
 456     # Output a segment of text.
 457     def output_segment (self, segment):
 458         with open (segment.filename, 'r') as file:
 459             file.seek (segment.start)
 460             for i in range (segment.lines):
 461                 sys.stdout.write (file.readline())
 462
 463     # Output a summary giving the number of times each type of result has
 464     # been seen.
 465     def output_summary (self, tool, counts):
 466         for i in range (len (self.count_names)):
 467             name = self.count_names[i]
 468             # dejagnu only prints result types that were seen at least once,
 469             # but acats always prints a number of unexpected failures.
 470             if (counts[i] > 0
 471                 or (tool.name == 'acats'
 472                     and name.startswith ('# of unexpected failures'))):
 473                 sys.stdout.write ('%s%d\n' % (name, counts[i]))
 474
 475     # Output unified .log or .sum information for a particular variation,
 476     # with a summary at the end.
 477     def output_variation (self, tool, variation):
 478         self.output_segment (variation.header)
 479         for harness in sorted (variation.harnesses.values()):
 480             sys.stdout.write ('Running ' + harness.name + ' ...\n')
 481             if self.do_sum:
 482                 # Keep the original test result order if there was only
 483                 # one segment for this harness.  This is needed for
 484                 # unsorted.exp, which has unusual test names.  Otherwise
 485                 # sort the tests by test filename.  If there are several
 486                 # subtests for the same test filename (such as 'compilation',
 487                 # 'test for excess errors', etc.) then keep the subtests
 488                 # in the original order.
 489                 if len (harness.segments) > 1:
 490                     harness.results.sort()
 491                 for (key, line) in harness.results:
 492                     sys.stdout.write (line)
 493             else:
 494                 # Rearrange the log segments into test order (but without
 495                 # rearranging text within those segments).
 496                 for key in sorted (harness.segments.keys()):
 497                     self.output_segment (harness.segments[key])
 498                 for segment in harness.empty:
 499                     self.output_segment (segment)
 500         if len (self.variations) > 1:
 501             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
 502                               + variation.name + ' ===\n\n')
 503             self.output_summary (tool, variation.counts)
 504
 505     # Output unified .log or .sum information for a particular tool,
 506     # with a summary at the end.
 507     def output_tool (self, tool):
 508         counts = self.zero_counts()
 509         if tool.name == 'acats':
 510             # acats doesn't use variations, so just output everything.
 511             # It also has a different approach to whitespace.
 512             sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
 513             for variation in tool.variations.values():
 514                 self.output_variation (tool, variation)
 515                 self.accumulate_counts (counts, variation.counts)
 516             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
 517         else:
 518             # Output the results in the usual dejagnu runtest format.
 519             sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
 520                               'Schedule of variations:\n')
 521             for name in self.variations:
 522                 if name in tool.variations:
 523                     sys.stdout.write ('    ' + name + '\n')
 524             sys.stdout.write ('\n')
 525             for name in self.variations:
 526                 if name in tool.variations:
 527                     variation = tool.variations[name]
 528                     sys.stdout.write ('Running target '
 529                                       + variation.name + '\n')
 530                     self.output_variation (tool, variation)
 531                     self.accumulate_counts (counts, variation.counts)
 532             sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
 533         self.output_summary (tool, counts)
 534
 535     def main (self):
 536         self.parse_cmdline()
 537         try:
 538             # Parse the input files.
 539             for filename in self.files:
 540                 with open (filename, 'r') as file:
 541                     self.parse_file (filename, file)
 542
 543             # Decide what to output.
 544             if len (self.variations) == 0:
 545                 self.variations = sorted (self.known_variations)
 546             else:
 547                 for name in self.variations:
 548                     if name not in self.known_variations:
 549                         self.fatal (None, 'no results for ' + name)
 550             if len (self.tools) == 0:
 551                 self.tools = sorted (self.runs.keys())
 552
 553             # Output the header.
 554             if self.start_line:
 555                 sys.stdout.write (self.start_line[1])
 556             sys.stdout.write (self.native_line)
 557             sys.stdout.write (self.target_line)
 558             sys.stdout.write (self.host_line)
 559             sys.stdout.write (self.acats_premable)
 560
 561             # Output the main body.
 562             for name in self.tools:
 563                 if name not in self.runs:
 564                     self.fatal (None, 'no results for ' + name)
 565                 self.output_tool (self.runs[name])
 566
 567             # Output the footer.
 568             if len (self.acats_failures) > 0:
 569                 sys.stdout.write ('*** FAILURES: '
 570                                   + ' '.join (self.acats_failures) + '\n')
 571             sys.stdout.write (self.version_output)
 572             if self.end_line:
 573                 sys.stdout.write (self.end_line[1])
 574         except IOError as e:
 575             self.fatal (e.filename, e.strerror)
 576
 577 Prog().main()