selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11
  12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
  13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
  14
  15 import subunithelper
  16 import subunit
  17
  18
  19 def format_time(delta):
  20     minutes, seconds = divmod(delta.seconds, 60)
  21     hours, minutes = divmod(minutes, 60)
  22     ret = ""
  23     if hours:
  24         ret += "%dh" % hours
  25     if minutes:
  26         ret += "%dm" % minutes
  27     ret += "%ds" % seconds
  28     return ret
  29
  30
  31 class PlainFormatter(subunithelper.TestsuiteEnabledTestResult):
  32
  33     def __init__(self, summaryfile, verbose, immediate, statistics,
  34             totaltests=None):
  35         super(PlainFormatter, self).__init__()
  36         self.verbose = verbose
  37         self.immediate = immediate
  38         self.statistics = statistics
  39         self.start_time = None
  40         self.test_output = {}
  41         self.suitesfailed = []
  42         self.suites_ok = 0
  43         self.skips = {}
  44         self.summaryfile = summaryfile
  45         self.index = 0
  46         self.name = None
  47         self._progress_level = 0
  48         self.totalsuites = totaltests
  49         self.last_time = None
  50
  51     def progress(self, offset, whence):
  52         if whence == subunit.PROGRESS_POP:
  53             self._progress_level -= 1
  54         elif whence == subunit.PROGRESS_PUSH:
  55             self._progress_level += 1
  56         elif whence == subunit.PROGRESS_SET:
  57             if self._progress_level == 0:
  58                 self.totalsuites = offset
  59         elif whence == subunit.PROGRESS_CUR:
  60             raise NotImplementedError
  61
  62     def time(self, dt):
  63         if self.start_time is None:
  64             self.start_time = dt
  65         self.last_time = dt
  66
  67     def start_testsuite(self, name):
  68         self.index += 1
  69         self.name = name
  70
  71         if not self.verbose:
  72             self.test_output[name] = ""
  73
  74         out = "[%d" % self.index
  75         if self.totalsuites is not None:
  76             out += "/%d" % self.totalsuites
  77         if self.start_time is not None:
  78             out += " in " + format_time(self.last_time - self.start_time)
  79         if self.suitesfailed:
  80             out += ", %d errors" % (len(self.suitesfailed),)
  81         out += "] %s" % name
  82         if self.immediate:
  83             sys.stdout.write(out + "\n")
  84         else:
  85             sys.stdout.write(out + ": ")
  86
  87     def output_msg(self, output):
  88         if self.verbose:
  89             sys.stdout.write(output)
  90         elif self.name is not None:
  91             self.test_output[self.name] += output
  92         else:
  93             sys.stdout.write(output)
  94
  95     def control_msg(self, output):
  96         #$self->output_msg($output)
  97         pass
  98
  99     def end_testsuite(self, name, result, reason):
 100         out = ""
 101         unexpected = False
 102
 103         if not name in self.test_output:
 104             print "no output for name[%s]" % name
 105
 106         if result in ("success", "xfail"):
 107             self.suites_ok+=1
 108         else:
 109             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 110             if reason is not None:
 111                 self.output_msg("REASON: %s\n" % (reason,))
 112             self.suitesfailed.append(name)
 113             if self.immediate and not self.verbose and name in self.test_output:
 114                 out += self.test_output[name]
 115             unexpected = True
 116
 117         if not self.immediate:
 118             if not unexpected:
 119                 out += " ok\n"
 120             else:
 121                 out += " " + result.upper() + "\n"
 122
 123         sys.stdout.write(out)
 124
 125     def startTest(self, test):
 126         pass
 127
 128     def addSuccess(self, test):
 129         self.end_test(test.id(), "success", False)
 130
 131     def addError(self, test, details=None):
 132         self.end_test(test.id(), "error", True, details)
 133
 134     def addFailure(self, test, details=None):
 135         self.end_test(test.id(), "failure", True, details)
 136
 137     def addSkip(self, test, details=None):
 138         self.end_test(test.id(), "skip", False, details)
 139
 140     def addExpectedFail(self, test, details=None):
 141         self.end_test(test.id(), "xfail", False, details)
 142
 143     def end_test(self, testname, result, unexpected, reason=None):
 144         if not unexpected:
 145             self.test_output[self.name] = ""
 146             if not self.immediate:
 147                 sys.stdout.write({
 148                     'failure': 'f',
 149                     'xfail': 'X',
 150                     'skip': 's',
 151                     'success': '.'}.get(result, "?(%s)" % result))
 152             return
 153
 154         if not self.name in self.test_output:
 155             self.test_output[self.name] = ""
 156
 157         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 158         if reason is not None:
 159             self.test_output[self.name] += "REASON: %s\n" % (reason[1].message.encode("utf-8").strip(),)
 160
 161         if self.immediate and not self.verbose:
 162             print self.test_output[self.name]
 163             self.test_output[self.name] = ""
 164
 165         if not self.immediate:
 166             sys.stdout.write({
 167                'error': 'E',
 168                'failure': 'F',
 169                'success': 'S'}.get(result, "?"))
 170
 171     def summary(self):
 172         f = open(self.summaryfile, 'w+')
 173
 174         if self.suitesfailed:
 175             f.write("= Failed tests =\n")
 176
 177             for suite in self.suitesfailed:
 178                 f.write("== %s ==\n" % suite)
 179                 if suite in self.test_output:
 180                     f.write(self.test_output[suite]+"\n\n")
 181
 182             f.write("\n")
 183
 184         if not self.immediate and not self.verbose:
 185             for suite in self.suitesfailed:
 186                 print "=" * 78
 187                 print "FAIL: %s" % suite
 188                 if suite in self.test_output:
 189                     print self.test_output[suite]
 190                 print ""
 191
 192         f.write("= Skipped tests =\n")
 193         for reason in self.skips.keys():
 194             f.write(reason + "\n")
 195             for name in self.skips[reason]:
 196                 f.write("\t%s\n" % name)
 197             f.write("\n")
 198         f.close()
 199
 200         print "\nA summary with detailed information can be found in:"
 201         print "  %s" % self.summaryfile
 202
 203         if not self.suitesfailed:
 204             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 205                   self.statistics['TESTS_EXPECTED_FAIL'])
 206             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 207         else:
 208             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 209                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 210                 self.statistics['TESTS_ERROR'],
 211                 len(self.suitesfailed))
 212
 213     def skip_testsuite(self, name, reason="UNKNOWN"):
 214         self.skips.setdefault(reason, []).append(name)
 215         if self.totalsuites:
 216             self.totalsuites-=1
 217
 218 parser = optparse.OptionParser("format-subunit [options]")
 219 parser.add_option("--verbose", action="store_true",
 220     help="Be verbose")
 221 parser.add_option("--immediate", action="store_true",
 222     help="Show failures immediately, don't wait until test run has finished")
 223 parser.add_option("--prefix", type="string", default=".",
 224     help="Prefix to write summary to")
 225
 226 opts, args = parser.parse_args()
 227
 228 statistics = {
 229     'SUITES_FAIL': 0,
 230     'TESTS_UNEXPECTED_OK': 0,
 231     'TESTS_EXPECTED_OK': 0,
 232     'TESTS_UNEXPECTED_FAIL': 0,
 233     'TESTS_EXPECTED_FAIL': 0,
 234     'TESTS_ERROR': 0,
 235     'TESTS_SKIP': 0,
 236 }
 237
 238 def handle_sigint(sig, stack):
 239         sys.exit(0)
 240 signal.signal(signal.SIGINT, handle_sigint)
 241
 242 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 243     opts.immediate, statistics)
 244
 245 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 246
 247 msg_ops.summary()
 248
 249 sys.exit(expected_ret)