src/prototest.py

   1 ''' prototest regression tester for Proto
   2 Copyright (C) 2005-2008, Jonathan Bachrach, Jacob Beal, and contributors
   3 listed in the AUTHORS file in the MIT Proto distribution's top directory.
   4
   5 This file is part of MIT Proto, and is distributed under the terms of
   6 the GNU General Public License, with a linking exception, as described
   7 in the file LICENSE in the MIT Proto distribution's top directory.
   8 '''
   9
  10 '''
  11 Prototest helps analyze dump files to perform regression testing on
  12 the proto language core. It reads config file(s) containing "tests"
  13 each with one or more "assertions", and verifies the assertions. The
  14 results are stored in a log file.  Look at proto_readme for a long
  15 help including tutorial on how to create config files, or run
  16 prototest with -h parameter to see how to use prototest.
  17
  18 USAGE:
  19 python prototest.py <TestName> [one or more switches]
  20 '''
  21
  22 import linecache, optparse, os, random, subprocess, sys
  23 prototest_version = "Prototest 0.75"
  24 #Global Variables
  25 verbosity = 1
  26 dump_dir = ""
  27 recursive_dir_scan = False
  28
  29 #Exceptions
  30 class MissingDumpFileError(Exception):
  31     '''Thrown when a test cannot find its dumpfile'''
  32     pass
  33
  34 class InvalidConfigPathError(Exception):
  35     '''
  36     Thrown when user passes a file that is not a testfile
  37     or if the directory doesn't contain any other test files
  38     '''
  39     pass
  40
  41 #The Hierarchy is:
  42 #Test Suite -> Test File -> Test -> Assertion (-> means one or more)
  43
  44 class Assertion():
  45     '''
  46     Assertions compare expected value to actual values in the dump file.
  47     They contain the comparison function, along with the line and col number
  48     to read the actual value.
  49     '''
  50
  51     def __init__(self, line, column, expected_val, comparison_fn, desc, is_numeric = True):
  52         '''
  53         Initialized using line & column number to find the actual value, the
  54         expected value, the function to compare the two values, and a string
  55         describing the type of assertion.
  56         '''
  57         #Cast to ints & floats because parser passes strings
  58         self.line = int(line)
  59         self.col = int(column) if column.isdigit() else "_"
  60         self.is_numeric  = is_numeric
  61         self.expected_val = float(expected_val) if is_numeric else expected_val
  62         self.comparison_fn = comparison_fn
  63         self.desc = str(desc)
  64         self.dumpfile = "" #is known only after the tests are run
  65
  66         #Failed means value didn't match expected.
  67         #Crashed means couldn't read value (etc.)
  68         self.failed = False #Default starting value
  69         self.actual_val = 0 if is_numeric else expected_val
  70         self.crash = False
  71         self.crash_reason = ""
  72
  73     def run(self, dumpfile):
  74         '''
  75         Reads the actual value from the given dumpfile, and compares it to the
  76         expected value.
  77         '''
  78         self.dumpfile = dumpfile
  79         #Read value
  80         try:
  81             #Use linecache to provide random-access to lines within the file
  82             #Add 1 to line num because linecache numbers from 1...
  83             lineval = linecache.getline(dumpfile, self.line + 1)
  84
  85             if self.col == "_": #Test whole lines
  86                 self.actual_val = lineval.strip()
  87             else:
  88                 actual_val = lineval.split()[self.col]
  89                 #Since linecache silently returns "" instead of raising exceptions:
  90                 if actual_val == '' or actual_val == '\n':
  91                     raise IndexError
  92                 self.actual_val = float(actual_val) if self.is_numeric else actual_val
  93         except IndexError:
  94             self.set_exception("\nIndex out of bounds (NOTE: Line/Column Numbering is done from 0) ")
  95         except ValueError:
  96             self.set_exception("Non-numeric value %s found (this assertion is numeric)" % actual_val)
  97         else:
  98             self.failed = not self.comparison_fn(self.actual_val, self.expected_val)
  99
 100         return self.get_result()
 101
 102     def set_exception(self, error_desc):
 103         ''' Called to set exception when trying to run assertion function '''
 104         self.crash = True
 105         self.crash_reason = "FAIL\t" + error_desc + \
 106                             "Trying to access line %d, column %s in %s" % (self.line, self.col, self.dumpfile)
 107         self.failed = True
 108
 109     def get_result(self):
 110         ''' Returns a pretty-output version of the result. '''
 111
 112         if self.crash:
 113             return self.crash_reason
 114         elif self.failed:
 115             out = "FAIL\t"
 116         else:
 117             out = "pass\t"
 118         out += "Value at Line %s Column %s (%s) %s %s " % \
 119                (self.line, self.col, self.actual_val, self.desc, self.expected_val)
 120         return out
 121
 122
 123 class Test():
 124     '''
 125     A test is a collection of assertions made on a dump file generated by
 126     proto.
 127     '''
 128
 129     def __init__(self, protoarg, dumpfile_prefix):
 130         '''
 131         A test is initialized by providing the arguments to run proto with,
 132         a prefix for the dump file (randomly generated by parser), and if set by
 133         user, a dump directory to find the dump file. The other attributes are
 134         filled in dynamically.
 135         '''
 136         self.protoarg = protoarg
 137         self.dumpfile_prefix = dumpfile_prefix
 138         #Assertions are added dynamically instead of at instantiation
 139         self.asserts = []
 140         self.fail_asserts = []
 141         self.reset()
 142         self.failed = False #Default Starting Value
 143         self.crash = False #Crashed = proto crashed / non-zero return code
 144         self.crash_reason = "" #String to describe an exception.
 145         self.proto_output = ("","") #stdout,stderr
 146
 147     def add(self, assertion):
 148         '''Adds assertion to the test.'''
 149         self.asserts.append(assertion)
 150
 151     def reset(self):
 152         '''Reset to before test situations'''
 153         self.fail_asserts = []
 154         self.failed = False
 155         self.crash = False
 156         self.crash_reason = ""
 157         self.proto_output = ("","")
 158
 159
 160     def find_dump(self):
 161         '''Returns path to the actual dump file with the specified prefix'''
 162         #Set dump directory (if user hasn't set one already)
 163         global dump_dir
 164
 165         if not dump_dir:
 166             current_dir = os.path.abspath(sys.path[0])
 167             dump_dir = os.path.join(current_dir,'dumps')
 168         #Filter to find candidates for the dump file
 169         matches = [x for x in os.listdir(dump_dir) \
 170                    if x.startswith(self.dumpfile_prefix) and x.endswith('.log')]
 171         if len(matches) < 1:
 172             raise MissingDumpFileError("Searched for %s%s" % (self.dumpfile_prefix, ".log"))
 173
 174         logname = matches[0] #Match is first item in the list
 175         #Prefix the dump directory, and return
 176         return os.path.join(dump_dir, logname)
 177
 178     def test_crashed(self, crash_reason):
 179         '''
 180         Convenience method to be called when a test crashes.
 181         Marks all assertions as failed, and sets the crash reason.
 182         '''
 183         self.crash = True
 184         self.failed = True
 185         self.crash_reason = crash_reason
 186         #Mark all Assertions as failed
 187         for a in self.asserts:
 188             a.set_exception(crash_reason)
 189         self.fail_asserts.extend(self.asserts)
 190
 191     def run(self):
 192         '''Executes proto, runs all assertions on the resulting dump file.'''
 193         self.reset()
 194
 195         try:
 196             #Create process, run & get return code
 197             p = subprocess.Popen(self.protoarg, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 198             self.proto_output = p.communicate()
 199             if p.returncode != 0:
 200                 raise subprocess.CalledProcessError(p.returncode, self.protoarg)
 201             #Find dump file, run assertions
 202             dumpfile = self.find_dump()
 203             for a in self.asserts:
 204                 a.run(dumpfile)
 205                 if a.failed:
 206                     self.fail_asserts.append(a)
 207                     self.failed = True
 208         except subprocess.CalledProcessError, exc:
 209             error_str = "Proto terminated with a non-zero return code: " + str(exc)
 210             self.test_crashed(error_str)
 211         except MissingDumpFileError, exc:
 212             error_str = "Could not find the dumpfile. " + str(exc)
 213             self.test_crashed(error_str)
 214         except:
 215             #Just in case there is some other exception (we don't want the entire program to crash)
 216             self.test_crashed("Unhandled Exception: %s" % (sys.exc_info()[1])) #exec_info returns: (type, value, traceback)
 217
 218     def get_result(self):
 219         ''' Returns a pretty-output version of the result. '''
 220         if self.crash:
 221             return self.crash_reason
 222         elif self.failed:
 223             out = ["FAIL: Failed %d out of %d assertions." \
 224                    % (len(self.fail_asserts), len(self.asserts))]
 225             for f in self.fail_asserts:
 226                 out.append(f.get_result())
 227             return '\n'.join(out)
 228         else:
 229             return "All assertions passed."
 230
 231 class TestFile():
 232     ''' Represents all the tests contained in a single file.'''
 233     def __init__(self, filename):
 234         ''' initialized by giving it the name of the test file.'''
 235         self.filename = filename
 236         self.tests = []
 237         self.failed = False
 238         #To Write Dump Files:
 239         self.randomseed = random.randint(10000, 99999)
 240
 241     def parse_file(self):
 242         ''' Builds and runs config parser on the test config file.'''
 243         self.config_parser = ConfigParser(self.filename)
 244         self.tests = self.config_parser.run()
 245
 246     def run(self):
 247         '''Runs all tests in the test file.'''
 248         self.failed = False #reset
 249         print "Running %s  " % self.filename,
 250         tests_completed = 0
 251         for test in self.tests:
 252             test.run()
 253             if test.failed:
 254                 self.failed = True
 255             #Print progress bar; one "." for every 5 tests completed.
 256             sys.stdout.flush()
 257             tests_completed += 1
 258             if tests_completed % 5 == 0 or tests_completed == 1:
 259                 print "\b.",
 260
 261         #Write Logs
 262         if self.failed:
 263             print " FAILED %d out of %d tests" % (len([t for t in self.tests if t.failed]), len(self.tests))
 264         else:
 265             print " passed all %d tests" % len(self.tests)
 266
 267         self.write_log()
 268         return self.get_result()
 269
 270     def get_result(self):
 271         '''Returns a pretty output'''
 272         tests_passed = len([t for t in self.tests if not t.failed])
 273         if tests_passed == len(self.tests):
 274             return "All tests Passed"
 275         else:
 276             return "Passed %d out of %d tests." % (tests_passed, len(self.tests))
 277
 278     def write_log(self):
 279         ''' Generates a test log for the file.'''
 280         log = [self.get_result()]
 281
 282         for i in range(len(self.tests)):
 283             test = self.tests[i]
 284             if verbosity >= 1 or test.failed:
 285                 log.extend(["Test #%d" % i,
 286                             "Running with arguments: %s" % test.protoarg,
 287                             "Dump file path: %s" % test.asserts[0].dumpfile,
 288                             "\n***PROTO OUTPUT***",
 289                             ''.join(test.proto_output),
 290                             "***END PROTO OUTPUT***\n",
 291                             test.get_result()
 292                             ])
 293                 log.append("-" * 80)
 294
 295         out_file = self.filename + ".RESULTS"
 296         out = open(out_file,"w")
 297         out.write('\n'.join(log))
 298         out.close()
 299
 300
 301 class TestSuite():
 302     '''
 303     A test suite is a collection of test files. It is provided to allow
 304     users to run a comprehensive test suite without having to create a
 305     monolithic test file.
 306     '''
 307     def __init__(self, config_path):
 308         self.config_path = config_path
 309         self.test_files = {} #Store test_name: Test
 310         self.failed = True #Default Starting Value
 311
 312     def add(self, test_filename, test):
 313         ''' Add a test file to the test suite. '''
 314         self.test_files[test_filename] = test
 315
 316     def gen_tests(self):
 317         '''
 318         Populates test suite by adding all files with a ".test" extension
 319         to the test file.
 320         '''
 321         #Directory Input
 322         if os.path.isdir(self.config_path):
 323             if recursive_dir_scan:
 324                 matches = [os.path.join(a[0],b) \
 325                             for a in os.walk(self.config_path) for b in a[2]]
 326             else:
 327                 #All files in the test directory:
 328                 matches = [os.path.join(self.config_path, x) for x in os.listdir(self.config_path)]
 329             #Filter Results:
 330             matches = self.filter_tests(matches)
 331         elif os.path.isfile(self.config_path):
 332             #Test Suite Input
 333             if self.config_path.lower().endswith('.testsuite'):
 334                 f = open(self.config_path)
 335                 matches = self.filter_tests(f.readlines(), True)
 336             else:
 337                 #Test File Input
 338                 matches = [self.config_path]
 339         else:
 340             raise InvalidConfigPathError(self.config_path)
 341
 342         if len(matches) == 0:
 343             raise InvalidConfigPathError("Invalid Config Path: %s" % self.config_path)
 344         else:
 345             print "Found %d test files." % len(matches)
 346
 347         for test_name in matches:
 348             t = TestFile(test_name)
 349             t.parse_file()
 350             if len(t.tests) == 0:
 351                 print "Warning: No tests found in %s " % test_name
 352             else:
 353                 self.add(test_name, t)
 354
 355     def filter_tests(self, filelist, test_existence = False):
 356         '''
 357         Removes all files that don't end with '.test' extension. Also performs some
 358         cleanup: removes leading and trailing whitespace, removes duplicates, removes non-existant files.
 359
 360         Intended to be used in directory mode or test suite mode
 361         '''
 362         cleaned = set([x.strip() for x in filelist \
 363                          if x.strip().lower().endswith('.test')])
 364
 365         if test_existence:
 366             for f in cleaned:
 367                 try:
 368                     z = open(f,"r")
 369                 except IOERROR:
 370                     cleaned.remove(f)
 371
 372         return cleaned
 373
 374     def run(self):
 375         '''Runs all Test files in test suite.'''
 376         for test_file_name, test_file in self.test_files.items():
 377             test_file.run()
 378             if test_file.failed:
 379                 self.failed = True
 380
 381 class ConfigParser():
 382     '''
 383     This class is responsible for parsing a "config file" which is a list of
 384     tests and assertions. Supported comparison operators are defined in the
 385     dictionaries below with a
 386         symbol: (function, description)
 387     format.
 388
 389     NOTE ON DEFINING FNS WHICH REQUIRE EXTRA ARGUMENTS:
 390     Use currying to convert a function of n-arguments into a higher order
 391     function that accepts a list of all arguments other than "expected value"
 392     and "actual value". (See the definition of ~= for an example.)
 393     '''
 394
 395     numeric_fns = {"=" : ((lambda act, exp: act == exp), " equal to "),
 396            ">"  : ((lambda act, exp: act > exp), " greater than "),
 397            "<"  : ((lambda act, exp: act < exp), " less than "),
 398            ">=" : ((lambda act, exp: act >= exp), "greater than / equal to "),
 399            "<=" : ((lambda act, exp: act <= exp), "less than / equal to "),
 400            "!=" : ((lambda act, exp: act != exp), " not equal to "),
 401            "~=" : ((lambda arg_list: (lambda act, exp: abs(act - exp) <= arg_list[0])),
 402                    " nearly equal to ")
 403            }
 404
 405     string_fns = {"is_nan" : ((lambda act, exp: act.lower() == "nan"), 1, " is "),
 406                   "is" : ((lambda act, exp: act.lower() == exp.lower()), 0, " is "),
 407                   "has": ((lambda act, exp: act.lower().find(exp.lower()) != -1), 0, " has ")}
 408
 409     def __init__(self, test_file):
 410         '''
 411         Initialized by giving it an instance of TestFile
 412         '''
 413         self.test_file = test_file
 414         self.tests = []
 415         self.randomseed = random.randint(10000, 99999)
 416
 417     def reset(self):
 418         ''' Removes all tests parsed so far. '''
 419         self.tests = []
 420
 421     def gen_dump_name(self):
 422         '''Attaches a prefix to the proto dump file.'''
 423         return "prototest" + str(self.randomseed + len(self.tests))
 424
 425     def run(self):
 426         '''Scans the config file and populates test cases and assertions.'''
 427         self.reset()
 428
 429
 430         #Open config file
 431         try:
 432             config_file = open(self.test_file, "r")
 433         except UnboundLocalError:
 434             #This is the only way to know if the file couldn't be opened.
 435             print "Could not open the config file (IO Exception)"
 436
 437         if verbosity > 0:
 438             print "Parsing %s" % self.test_file,
 439         #Parse line by line
 440         for line in config_file:
 441             line = line.rstrip("\n")
 442             #Split by whitespace, ignore empty lines
 443             split = line.split()
 444             if not split:
 445                 continue
 446
 447             cmd = split[0] #first token is the "command"
 448
 449             if cmd == "test:":
 450                 if verbosity >= 3:
 451                     print "PARSER: encountered a test: ", line
 452                 #Generate a dump file name for this test
 453                 dump_file_name = self.gen_dump_name()
 454
 455                 #Have proto use this dump file name:
 456                 protoargs = split[1:]
 457                 try:
 458                     protoargs.insert(protoargs.index("-D") + 1,
 459                                      r"-dump-stem " + dump_file_name)
 460                 except ValueError:
 461                     #User hasn't provided a dump file related switch.
 462                     protoargs.append(r"-D -dump-stem " + dump_file_name)
 463                 #If using -v0, automatically make all tests headless [exclude p2b]
 464                 if not "-headless" in protoargs and not "--test-compiler" in protoargs and verbosity == 0:
 465                     protoargs.append("-headless")
 466                 #Finally, create a test
 467                 protoargs = ' '.join(protoargs)
 468                 self.tests.append(Test(protoargs, dump_file_name))
 469             elif cmd.startswith("//"):
 470                 if verbosity >= 3:
 471                     print "PARSER: encountered a comment: ", line
 472             elif cmd in self.numeric_fns:
 473                 if verbosity >= 3:
 474                     print "PARSER: encountered a numeric assertion: ", line
 475                 #<cmd> <line #> <column #> <expected val> <other args>
 476                 if len(split[4:]) == 0:
 477                     #Binary function
 478                     fn = self.numeric_fns[cmd][0]
 479                 else:
 480                     #Curried Function
 481                     fn = self.numeric_fns[cmd][0](split[4:])
 482                 a = Assertion(split[1], split[2], split[3], fn,\
 483                               self.numeric_fns[cmd][1], is_numeric = True)
 484                 self.tests[-1].add(a)
 485             elif cmd in self.string_fns:
 486                 if verbosity >= 3:
 487                     print "PARSER: encountered a string assertion: ", line
 488                 #If fn works on entire line, split properly & strip whitespace.
 489                 if split[2] == "_":
 490                     expected = line.split(None,3)[3].strip()
 491                 else:
 492                     expected = split[3]
 493                 fn = self.string_fns[cmd][0]
 494                 a = Assertion(split[1], split[2], expected, fn, self.string_fns[cmd][2], is_numeric = False)
 495                 self.tests[-1].add(a)
 496             else:
 497                 print "Parser: encountered unrecognized command: ", line
 498
 499         self.tests = [t for t in self.tests if len(t.asserts) != 0]
 500         if verbosity > 0:
 501             print "... %d tests found" % len(self.tests)
 502         return self.tests
 503
 504
 505
 506 def main():
 507     '''Builds and runs tests. (Main Entry Point of program)'''
 508     print prototest_version
 509
 510     #Build Command Line Parser
 511     parser = optparse.OptionParser(prog="prototest", version=prototest_version)
 512     parser.add_option("-v", "--verbose", type="int", action="store",
 513                       dest="verbosity", help="Prints diagnostic messages, more when used multiple times")
 514     parser.add_option("-r", "--recursive", action="store_true", dest="recursive", help="Looks for test files recursively in subdirectories.")
 515     parser.add_option("-d", "--dumpdir", dest="dump_dir",
 516                       help="Set the path to dump files. (use only if prototest is not in the same dir. as proto)")
 517     parser.set_defaults(verbosity=1, dumpdir="")
 518
 519     #Parse Command Line Arguments
 520     (option, args) = parser.parse_args()
 521     if len(args) < 1:
 522         sys.exit("ERROR: You must provide a test config file (or path to test files)")
 523     else:
 524         global verbosity; global dump_dir; global recursive_dir_scan;
 525         (config_path, verbosity, dump_dir, recursive_dir_scan) = \
 526             (args[0], option.verbosity, option.dump_dir, option.recursive)
 527
 528     #Build and Run Tests
 529     test_suite = TestSuite(config_path)
 530     print "PARSING TEST FILE(s)"
 531     test_suite.gen_tests()
 532     print "RUNNING TEST(s)"
 533     test_suite.run()
 534
 535 if __name__ == "__main__":
 536     main()