selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11
  12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
  13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
  14
  15 import subunithelper
  16 import subunit
  17
  18 def format_time(t):
  19     minutes, seconds = divmod(t, 60)
  20     hours, minutes = divmod(minutes, 60)
  21     ret = ""
  22     if hours:
  23         ret += "%dh" % hours
  24     if minutes:
  25         ret += "%dm" % minutes
  26     ret += "%ds" % seconds
  27     return ret
  28
  29
  30 class PlainFormatter(object):
  31
  32     def __init__(self, summaryfile, verbose, immediate, statistics,
  33             totaltests=None):
  34         self.verbose = verbose
  35         self.immediate = immediate
  36         self.statistics = statistics
  37         self.start_time = None
  38         self.test_output = {}
  39         self.suitesfailed = []
  40         self.suites_ok = 0
  41         self.skips = {}
  42         self.summaryfile = summaryfile
  43         self.index = 0
  44         self.name = None
  45         self._progress_level = 0
  46         self.totalsuites = totaltests
  47
  48     def progress(self, offset, whence):
  49         if whence == subunit.PROGRESS_POP:
  50             self._progress_level -= 1
  51         elif whence == subunit.PROGRESS_PUSH:
  52             self._progress_level += 1
  53         elif whence == subunit.PROGRESS_SET:
  54             if self._progress_level == 0:
  55                 self.totalsuites = offset
  56         elif whence == subunit.PROGRESS_CUR:
  57             raise NotImplementedError
  58
  59     def report_time(self, time):
  60         if self.start_time is None:
  61             self.start_time = time
  62         self.last_time = time
  63
  64     def start_testsuite(self, name):
  65         self.index += 1
  66         self.name = name
  67         testsuite_start_time = self.last_time
  68
  69         duration = testsuite_start_time - self.start_time
  70
  71         if not self.verbose:
  72             self.test_output[name] = ""
  73
  74         out = "[%d" % self.index
  75         if self.totalsuites is not None:
  76             out += "/%d" % self.totalsuites
  77         out += " in " + format_time(duration)
  78         if self.suitesfailed:
  79             out += ", %d errors" % (len(self.suitesfailed),)
  80         out += "] %s" % name
  81         if self.immediate:
  82             sys.stdout.write(out + "\n")
  83         else:
  84             sys.stdout.write(out + ": ")
  85
  86     def output_msg(self, output):
  87         if self.verbose:
  88             sys.stdout.write(output)
  89         elif self.name is not None:
  90             self.test_output[self.name] += output
  91         else:
  92             sys.stdout.write(output)
  93
  94     def control_msg(self, output):
  95         #$self->output_msg($output)
  96         pass
  97
  98     def end_testsuite(self, name, result, reason):
  99         out = ""
 100         unexpected = False
 101
 102         if not name in self.test_output:
 103             print "no output for name[%s]" % name
 104
 105         if result in ("success", "xfail"):
 106             self.suites_ok+=1
 107         else:
 108             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 109             if reason is not None:
 110                 self.output_msg("REASON: %s\n" % (reason,))
 111             self.suitesfailed.append(name)
 112             if self.immediate and not self.verbose and name in self.test_output:
 113                 out += self.test_output[name]
 114             unexpected = True
 115
 116         if not self.immediate:
 117             if not unexpected:
 118                 out += " ok\n"
 119             else:
 120                 out += " " + result.upper() + "\n"
 121
 122         sys.stdout.write(out)
 123
 124     def start_test(self, testname):
 125         pass
 126
 127     def end_test(self, testname, result, unexpected, reason=None):
 128         if not unexpected:
 129             self.test_output[self.name] = ""
 130             if not self.immediate:
 131                 sys.stdout.write({
 132                     'failure': 'f',
 133                     'xfail': 'X',
 134                     'skip': 's',
 135                     'success': '.'}.get(result, "?(%s)" % result))
 136             return
 137
 138         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 139         if reason is not None:
 140             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
 141
 142         if self.immediate and not self.verbose:
 143             print self.test_output[self.name]
 144             self.test_output[self.name] = ""
 145
 146         if not self.immediate:
 147             sys.stdout.write({
 148                'error': 'E',
 149                'failure': 'F',
 150                'success': 'S'}.get(result, "?"))
 151
 152     def summary(self):
 153         f = open(self.summaryfile, 'w+')
 154
 155         if self.suitesfailed:
 156             f.write("= Failed tests =\n")
 157
 158             for suite in self.suitesfailed:
 159                 f.write("== %s ==\n" % suite)
 160                 if suite in self.test_output:
 161                     f.write(self.test_output[suite]+"\n\n")
 162
 163             f.write("\n")
 164
 165         if not self.immediate and not self.verbose:
 166             for suite in self.suitesfailed:
 167                 print "=" * 78
 168                 print "FAIL: %s" % suite
 169                 if suite in self.test_output:
 170                     print self.test_output[suite]
 171                 print ""
 172
 173         f.write("= Skipped tests =\n")
 174         for reason in self.skips.keys():
 175             f.write(reason + "\n")
 176             for name in self.skips[reason]:
 177                 f.write("\t%s\n" % name)
 178             f.write("\n")
 179         f.close()
 180
 181         print "\nA summary with detailed information can be found in:"
 182         print "  %s" % self.summaryfile
 183
 184         if not self.suitesfailed:
 185             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 186                   self.statistics['TESTS_EXPECTED_FAIL'])
 187             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 188         else:
 189             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 190                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 191                 self.statistics['TESTS_ERROR'],
 192                 len(self.suitesfailed))
 193
 194     def skip_testsuite(self, name, reason="UNKNOWN"):
 195         self.skips.setdefault(reason, []).append(name)
 196         if self.totalsuites:
 197             self.totalsuites-=1
 198
 199 parser = optparse.OptionParser("format-subunit [options]")
 200 parser.add_option("--verbose", action="store_true",
 201     help="Be verbose")
 202 parser.add_option("--immediate", action="store_true",
 203     help="Show failures immediately, don't wait until test run has finished")
 204 parser.add_option("--prefix", type="string", default=".",
 205     help="Prefix to write summary to")
 206
 207 opts, args = parser.parse_args()
 208
 209 statistics = {
 210     'SUITES_FAIL': 0,
 211     'TESTS_UNEXPECTED_OK': 0,
 212     'TESTS_EXPECTED_OK': 0,
 213     'TESTS_UNEXPECTED_FAIL': 0,
 214     'TESTS_EXPECTED_FAIL': 0,
 215     'TESTS_ERROR': 0,
 216     'TESTS_SKIP': 0,
 217 }
 218
 219 def handle_sigint(sig, stack):
 220         sys.exit(0)
 221 signal.signal(signal.SIGINT, handle_sigint)
 222
 223 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 224     opts.immediate, statistics)
 225
 226 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 227
 228 msg_ops.summary()
 229
 230 sys.exit(expected_ret)