contrib/testsuite-management/validate_failures.py

   1 #!/usr/bin/python
   2
   3 # Script to compare testsuite failures against a list of known-to-fail
   4 # tests.
   5 #
   6 # NOTE: This script is used in installations that are running Python 2.4.
   7 #       Please stick to syntax features available in 2.4 and earlier
   8 #       versions.
   9
  10 # Contributed by Diego Novillo <dnovillo@google.com>
  11 #
  12 # Copyright (C) 2011-2013 Free Software Foundation, Inc.
  13 #
  14 # This file is part of GCC.
  15 #
  16 # GCC is free software; you can redistribute it and/or modify
  17 # it under the terms of the GNU General Public License as published by
  18 # the Free Software Foundation; either version 3, or (at your option)
  19 # any later version.
  20 #
  21 # GCC is distributed in the hope that it will be useful,
  22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  24 # GNU General Public License for more details.
  25 #
  26 # You should have received a copy of the GNU General Public License
  27 # along with GCC; see the file COPYING.  If not, write to
  28 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
  29 # Boston, MA 02110-1301, USA.
  30
  31 """This script provides a coarser XFAILing mechanism that requires no
  32 detailed DejaGNU markings.  This is useful in a variety of scenarios:
  33
  34 - Development branches with many known failures waiting to be fixed.
  35 - Release branches with known failures that are not considered
  36   important for the particular release criteria used in that branch.
  37
  38 The script must be executed from the toplevel build directory.  When
  39 executed it will:
  40
  41 1- Determine the target built: TARGET
  42 2- Determine the source directory: SRCDIR
  43 3- Look for a failure manifest file in
  44    <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail
  45 4- Collect all the <tool>.sum files from the build tree.
  46 5- Produce a report stating:
  47    a- Failures expected in the manifest but not present in the build.
  48    b- Failures in the build not expected in the manifest.
  49 6- If all the build failures are expected in the manifest, it exits
  50    with exit code 0.  Otherwise, it exits with error code 1.
  51
  52 Manifest files contain expected DejaGNU results that are otherwise
  53 treated as failures.
  54 They may also contain additional text:
  55
  56 # This is a comment.  - self explanatory
  57 @include file         - the file is a path relative to the includer
  58 @remove result text   - result text is removed from the expected set
  59 """
  60
  61 import datetime
  62 import optparse
  63 import os
  64 import re
  65 import sys
  66
  67 # Handled test results.
  68 _VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
  69 _VALID_TEST_RESULTS_REX = re.compile("%s" % "|".join(_VALID_TEST_RESULTS))
  70
  71 # Subdirectory of srcdir in which to find the manifest file.
  72 _MANIFEST_SUBDIR = 'contrib/testsuite-management'
  73
  74 # Pattern for naming manifest files.
  75 # The first argument should be the toplevel GCC(/GNU tool) source directory.
  76 # The second argument is the manifest subdir.
  77 # The third argument is the manifest target, which defaults to the target
  78 # triplet used during the build.
  79 _MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail'
  80
  81 # The options passed to the program.
  82 _OPTIONS = None
  83
  84 def Error(msg):
  85   print >>sys.stderr, 'error: %s' % msg
  86   sys.exit(1)
  87
  88
  89 class TestResult(object):
  90   """Describes a single DejaGNU test result as emitted in .sum files.
  91
  92   We are only interested in representing unsuccessful tests.  So, only
  93   a subset of all the tests are loaded.
  94
  95   The summary line used to build the test result should have this format:
  96
  97   attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors)
  98   ^^^^^^^^   ^^^^^  ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^
  99   optional   state  name              description
 100   attributes
 101
 102   Attributes:
 103     attrlist: A comma separated list of attributes.
 104       Valid values:
 105         flaky            Indicates that this test may not always fail.  These
 106                          tests are reported, but their presence does not affect
 107                          the results.
 108
 109         expire=YYYYMMDD  After this date, this test will produce an error
 110                          whether it is in the manifest or not.
 111
 112     state: One of UNRESOLVED, XPASS or FAIL.
 113     name: File name for the test.
 114     description: String describing the test (flags used, dejagnu message, etc)
 115     ordinal: Monotonically increasing integer.
 116              It is used to keep results for one .exp file sorted
 117              by the order the tests were run.
 118   """
 119
 120   def __init__(self, summary_line, ordinal=-1):
 121     try:
 122       self.attrs = ''
 123       if '|' in summary_line:
 124         (self.attrs, summary_line) = summary_line.split('|', 1)
 125       try:
 126         (self.state,
 127          self.name,
 128          self.description) = re.match(r' *([A-Z]+):\s*(\S+)\s+(.*)',
 129                                       summary_line).groups()
 130       except:
 131         print 'Failed to parse summary line: "%s"' % summary_line
 132         raise
 133       self.attrs = self.attrs.strip()
 134       self.state = self.state.strip()
 135       self.description = self.description.strip()
 136       self.ordinal = ordinal
 137     except ValueError:
 138       Error('Cannot parse summary line "%s"' % summary_line)
 139
 140     if self.state not in _VALID_TEST_RESULTS:
 141       Error('Invalid test result %s in "%s" (parsed as "%s")' % (
 142             self.state, summary_line, self))
 143
 144   def __lt__(self, other):
 145     return (self.name < other.name or
 146             (self.name == other.name and self.ordinal < other.ordinal))
 147
 148   def __hash__(self):
 149     return hash(self.state) ^ hash(self.name) ^ hash(self.description)
 150
 151   def __eq__(self, other):
 152     return (self.state == other.state and
 153             self.name == other.name and
 154             self.description == other.description)
 155
 156   def __ne__(self, other):
 157     return not (self == other)
 158
 159   def __str__(self):
 160     attrs = ''
 161     if self.attrs:
 162       attrs = '%s | ' % self.attrs
 163     return '%s%s: %s %s' % (attrs, self.state, self.name, self.description)
 164
 165   def ExpirationDate(self):
 166     # Return a datetime.date object with the expiration date for this
 167     # test result.  Return None, if no expiration has been set.
 168     if re.search(r'expire=', self.attrs):
 169       expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs)
 170       if not expiration:
 171         Error('Invalid expire= format in "%s".  Must be of the form '
 172               '"expire=YYYYMMDD"' % self)
 173       return datetime.date(int(expiration.group(1)),
 174                            int(expiration.group(2)),
 175                            int(expiration.group(3)))
 176     return None
 177
 178   def HasExpired(self):
 179     # Return True if the expiration date of this result has passed.
 180     expiration_date = self.ExpirationDate()
 181     if expiration_date:
 182       now = datetime.date.today()
 183       return now > expiration_date
 184
 185
 186 def GetMakefileValue(makefile_name, value_name):
 187   if os.path.exists(makefile_name):
 188     makefile = open(makefile_name)
 189     for line in makefile:
 190       if line.startswith(value_name):
 191         (_, value) = line.split('=', 1)
 192         value = value.strip()
 193         makefile.close()
 194         return value
 195     makefile.close()
 196   return None
 197
 198
 199 def ValidBuildDirectory(builddir, target):
 200   if (not os.path.exists(builddir) or
 201       not os.path.exists('%s/Makefile' % builddir) or
 202       (not os.path.exists('%s/build-%s' % (builddir, target)) and
 203        not os.path.exists('%s/%s' % (builddir, target)))):
 204     return False
 205   return True
 206
 207
 208 def IsComment(line):
 209   """Return True if line is a comment."""
 210   return line.startswith('#')
 211
 212
 213 def IsInterestingResult(line):
 214   """Return True if line is one of the summary lines we care about."""
 215   if '|' in line:
 216     (_, line) = line.split('|', 1)
 217     line = line.strip()
 218   return bool(_VALID_TEST_RESULTS_REX.match(line))
 219
 220
 221 def IsInclude(line):
 222   """Return True if line is an include of another file."""
 223   return line.startswith("@include ")
 224
 225
 226 def GetIncludeFile(line, includer):
 227   """Extract the name of the include file from line."""
 228   includer_dir = os.path.dirname(includer)
 229   include_file = line[len("@include "):]
 230   return os.path.join(includer_dir, include_file.strip())
 231
 232
 233 def IsNegativeResult(line):
 234   """Return True if line should be removed from the expected results."""
 235   return line.startswith("@remove ")
 236
 237
 238 def GetNegativeResult(line):
 239   """Extract the name of the negative result from line."""
 240   line = line[len("@remove "):]
 241   return line.strip()
 242
 243
 244 def ParseManifestWorker(result_set, manifest_path):
 245   """Read manifest_path, adding the contents to result_set."""
 246   if _OPTIONS.verbosity >= 1:
 247     print 'Parsing manifest file %s.' % manifest_path
 248   manifest_file = open(manifest_path)
 249   for line in manifest_file:
 250     line = line.strip()
 251     if line == "":
 252       pass
 253     elif IsComment(line):
 254       pass
 255     elif IsNegativeResult(line):
 256       result_set.remove(TestResult(GetNegativeResult(line)))
 257     elif IsInclude(line):
 258       ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path))
 259     elif IsInterestingResult(line):
 260       result_set.add(TestResult(line))
 261     else:
 262       Error('Unrecognized line in manifest file: %s' % line)
 263   manifest_file.close()
 264
 265
 266 def ParseManifest(manifest_path):
 267   """Create a set of TestResult instances from the given manifest file."""
 268   result_set = set()
 269   ParseManifestWorker(result_set, manifest_path)
 270   return result_set
 271
 272
 273 def ParseSummary(sum_fname):
 274   """Create a set of TestResult instances from the given summary file."""
 275   result_set = set()
 276   # ordinal is used when sorting the results so that tests within each
 277   # .exp file are kept sorted.
 278   ordinal=0
 279   sum_file = open(sum_fname)
 280   for line in sum_file:
 281     if IsInterestingResult(line):
 282       result = TestResult(line, ordinal)
 283       ordinal += 1
 284       if result.HasExpired():
 285         # Tests that have expired are not added to the set of expected
 286         # results. If they are still present in the set of actual results,
 287         # they will cause an error to be reported.
 288         print 'WARNING: Expected failure "%s" has expired.' % line.strip()
 289         continue
 290       result_set.add(result)
 291   sum_file.close()
 292   return result_set
 293
 294
 295 def GetManifest(manifest_path):
 296   """Build a set of expected failures from the manifest file.
 297
 298   Each entry in the manifest file should have the format understood
 299   by the TestResult constructor.
 300
 301   If no manifest file exists for this target, it returns an empty set.
 302   """
 303   if os.path.exists(manifest_path):
 304     return ParseManifest(manifest_path)
 305   else:
 306     return set()
 307
 308
 309 def CollectSumFiles(builddir):
 310   sum_files = []
 311   for root, dirs, files in os.walk(builddir):
 312     for ignored in ('.svn', '.git'):
 313       if ignored in dirs:
 314         dirs.remove(ignored)
 315     for fname in files:
 316       if fname.endswith('.sum'):
 317         sum_files.append(os.path.join(root, fname))
 318   return sum_files
 319
 320
 321 def GetResults(sum_files):
 322   """Collect all the test results from the given .sum files."""
 323   build_results = set()
 324   for sum_fname in sum_files:
 325     print '\t%s' % sum_fname
 326     build_results |= ParseSummary(sum_fname)
 327   return build_results
 328
 329
 330 def CompareResults(manifest, actual):
 331   """Compare sets of results and return two lists:
 332      - List of results present in ACTUAL but missing from MANIFEST.
 333      - List of results present in MANIFEST but missing from ACTUAL.
 334   """
 335   # Collect all the actual results not present in the manifest.
 336   # Results in this set will be reported as errors.
 337   actual_vs_manifest = set()
 338   for actual_result in actual:
 339     if actual_result not in manifest:
 340       actual_vs_manifest.add(actual_result)
 341
 342   # Collect all the tests in the manifest that were not found
 343   # in the actual results.
 344   # Results in this set will be reported as warnings (since
 345   # they are expected failures that are not failing anymore).
 346   manifest_vs_actual = set()
 347   for expected_result in manifest:
 348     # Ignore tests marked flaky.
 349     if 'flaky' in expected_result.attrs:
 350       continue
 351     if expected_result not in actual:
 352       manifest_vs_actual.add(expected_result)
 353
 354   return actual_vs_manifest, manifest_vs_actual
 355
 356
 357 def GetManifestPath(srcdir, target, user_provided_must_exist):
 358   """Return the full path to the manifest file."""
 359   manifest_path = _OPTIONS.manifest
 360   if manifest_path:
 361     if user_provided_must_exist and not os.path.exists(manifest_path):
 362       Error('Manifest does not exist: %s' % manifest_path)
 363     return manifest_path
 364   else:
 365     assert srdir and target
 366     return _MANIFEST_PATH_PATTERN % (srcdir, _MANIFEST_SUBDIR, target)
 367
 368
 369 def GetBuildData():
 370   srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =')
 371   target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=')
 372   if not ValidBuildDirectory(_OPTIONS.build_dir, target):
 373     # If we have been given a set of results to use, we may
 374     # not be inside a valid GCC build directory.  In that case,
 375     # the user must provide both a manifest file and a set
 376     # of results to check against it.
 377     if not _OPTIONS.results or not _OPTIONS.manifest:
 378       Error('%s is not a valid GCC top level build directory. '
 379             'You must use --manifest and --results to do the validation.' %
 380             _OPTIONS.build_dir)
 381     else:
 382       return None, None
 383   print 'Source directory: %s' % srcdir
 384   print 'Build target:     %s' % target
 385   return srcdir, target
 386
 387
 388 def PrintSummary(msg, summary):
 389   print '\n\n%s' % msg
 390   for result in sorted(summary):
 391     print result
 392
 393
 394 def GetSumFiles(results, build_dir):
 395   if not results:
 396     print 'Getting actual results from build directory %s' % build_dir
 397     sum_files = CollectSumFiles(build_dir)
 398   else:
 399     print 'Getting actual results from user-provided results'
 400     sum_files = results.split()
 401   return sum_files
 402
 403
 404 def PerformComparison(expected, actual, ignore_missing_failures):
 405   actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)
 406
 407   tests_ok = True
 408   if len(actual_vs_expected) > 0:
 409     PrintSummary('Unexpected results in this build (new failures)',
 410                  actual_vs_expected)
 411     tests_ok = False
 412
 413   if not ignore_missing_failures and len(expected_vs_actual) > 0:
 414     PrintSummary('Expected results not present in this build (fixed tests)'
 415                  '\n\nNOTE: This is not a failure.  It just means that these '
 416                  'tests were expected\nto fail, but they worked in this '
 417                  'configuration.\n', expected_vs_actual)
 418
 419   if tests_ok:
 420     print '\nSUCCESS: No unexpected failures.'
 421
 422   return tests_ok
 423
 424
 425 def CheckExpectedResults():
 426   srcdir, target = GetBuildData()
 427   manifest_path = GetManifestPath(srcdir, target, True)
 428   print 'Manifest:         %s' % manifest_path
 429   manifest = GetManifest(manifest_path)
 430   sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
 431   actual = GetResults(sum_files)
 432
 433   if _OPTIONS.verbosity >= 1:
 434     PrintSummary('Tests expected to fail', manifest)
 435     PrintSummary('\nActual test results', actual)
 436
 437   return PerformComparison(manifest, actual, _OPTIONS.ignore_missing_failures)
 438
 439
 440 def ProduceManifest():
 441   (srcdir, target) = GetBuildData()
 442   manifest_path = GetManifestPath(srcdir, target, False)
 443   print 'Manifest:         %s' % manifest_path
 444   if os.path.exists(manifest_path) and not _OPTIONS.force:
 445     Error('Manifest file %s already exists.\nUse --force to overwrite.' %
 446           manifest_path)
 447
 448   sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
 449   actual = GetResults(sum_files)
 450   manifest_file = open(manifest_path, 'w')
 451   for result in sorted(actual):
 452     print result
 453     manifest_file.write('%s\n' % result)
 454   manifest_file.close()
 455
 456   return True
 457
 458
 459 def CompareBuilds():
 460   (srcdir, target) = GetBuildData()
 461
 462   sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
 463   actual = GetResults(sum_files)
 464
 465   clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build)
 466   clean = GetResults(clean_sum_files)
 467
 468   return PerformComparison(clean, actual, _OPTIONS.ignore_missing_failures)
 469
 470
 471 def Main(argv):
 472   parser = optparse.OptionParser(usage=__doc__)
 473
 474   # Keep the following list sorted by option name.
 475   parser.add_option('--build_dir', action='store', type='string',
 476                     dest='build_dir', default='.',
 477                     help='Build directory to check (default = .)')
 478   parser.add_option('--clean_build', action='store', type='string',
 479                     dest='clean_build', default=None,
 480                     help='Compare test results from this build against '
 481                     'those of another (clean) build.  Use this option '
 482                     'when comparing the test results of your patch versus '
 483                     'the test results of a clean build without your patch. '
 484                     'You must provide the path to the top directory of your '
 485                     'clean build.')
 486   parser.add_option('--force', action='store_true', dest='force',
 487                     default=False, help='When used with --produce_manifest, '
 488                     'it will overwrite an existing manifest file '
 489                     '(default = False)')
 490   parser.add_option('--ignore_missing_failures', action='store_true',
 491                     dest='ignore_missing_failures', default=False,
 492                     help='When a failure is expected in the manifest but '
 493                     'it is not found in the actual results, the script '
 494                     'produces a note alerting to this fact. This means '
 495                     'that the expected failure has been fixed, or '
 496                     'it did not run, or it may simply be flaky '
 497                     '(default = False)')
 498   parser.add_option('--manifest', action='store', type='string',
 499                     dest='manifest', default=None,
 500                     help='Name of the manifest file to use (default = '
 501                     'taken from '
 502                     'contrib/testsuite-managment/<target_alias>.xfail)')
 503   parser.add_option('--produce_manifest', action='store_true',
 504                     dest='produce_manifest', default=False,
 505                     help='Produce the manifest for the current '
 506                     'build (default = False)')
 507   parser.add_option('--results', action='store', type='string',
 508                     dest='results', default=None, help='Space-separated list '
 509                     'of .sum files with the testing results to check. The '
 510                     'only content needed from these files are the lines '
 511                     'starting with FAIL, XPASS or UNRESOLVED (default = '
 512                     '.sum files collected from the build directory).')
 513   parser.add_option('--verbosity', action='store', dest='verbosity',
 514                     type='int', default=0, help='Verbosity level (default = 0)')
 515   global _OPTIONS
 516   (_OPTIONS, _) = parser.parse_args(argv[1:])
 517
 518   if _OPTIONS.produce_manifest:
 519     retval = ProduceManifest()
 520   elif _OPTIONS.clean_build:
 521     retval = CompareBuilds()
 522   else:
 523     retval = CheckExpectedResults()
 524
 525   if retval:
 526     return 0
 527   else:
 528     return 1
 529
 530
 531 if __name__ == '__main__':
 532   retval = Main(sys.argv)
 533   sys.exit(retval)