selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11
  12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
  13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
  14
  15 import subunithelper
  16 import subunit
  17
  18 class PlainFormatter(object):
  19
  20     def __init__(self, summaryfile, verbose, immediate, statistics,
  21             totaltests=None):
  22         self.verbose = verbose
  23         self.immediate = immediate
  24         self.statistics = statistics
  25         self.start_time = None
  26         self.test_output = {}
  27         self.suitesfailed = []
  28         self.suites_ok = 0
  29         self.skips = {}
  30         self.summaryfile = summaryfile
  31         self.index = 0
  32         self.name = None
  33         self._progress_level = 0
  34         self.totalsuites = totaltests
  35
  36     def progress(self, offset, whence):
  37         if whence == subunit.PROGRESS_POP:
  38             self._progress_level -= 1
  39         elif whence == subunit.PROGRESS_PUSH:
  40             self._progress_level += 1
  41         elif whence == subunit.PROGRESS_SET:
  42             if self._progress_level == 0:
  43                 self.totalsuites = offset
  44         elif whence == subunit.PROGRESS_CUR:
  45             raise NotImplementedError
  46
  47     def report_time(self, time):
  48         if self.start_time is None:
  49             self.start_time = time
  50         self.last_time = time
  51
  52     def start_testsuite(self, name):
  53         self.index += 1
  54         self.name = name
  55         testsuite_start_time = self.last_time
  56
  57         duration = testsuite_start_time - self.start_time
  58
  59         if not self.verbose:
  60             self.test_output[name] = ""
  61
  62         out = "[%d" % self.index
  63         if self.totalsuites is not None:
  64             out += "/%d" % self.totalsuites
  65         out += " in %ds" % duration
  66         if self.suitesfailed:
  67             out += ", %d errors" % (len(self.suitesfailed),)
  68         out += "] %s" % name
  69         if self.immediate:
  70             sys.stdout.write(out + "\n")
  71         else:
  72             sys.stdout.write(out + ": ")
  73
  74     def output_msg(self, output):
  75         if self.verbose:
  76             sys.stdout.write(output)
  77         elif self.name is not None:
  78             self.test_output[self.name] += output
  79         else:
  80             sys.stdout.write(output)
  81
  82     def control_msg(self, output):
  83         #$self->output_msg($output)
  84         pass
  85
  86     def end_testsuite(self, name, result, reason):
  87         out = ""
  88         unexpected = False
  89
  90         if not name in self.test_output:
  91             print "no output for name[%s]" % name
  92
  93         if result in ("success", "xfail"):
  94             self.suites_ok+=1
  95         else:
  96             self.output_msg("ERROR: Testsuite[%s]\n" % name)
  97             if reason is not None:
  98                 self.output_msg("REASON: %s\n" % (reason,))
  99             self.suitesfailed.append(name)
 100             if self.immediate and not self.verbose:
 101                 out += self.test_output[name]
 102             unexpected = True
 103
 104         if not self.immediate:
 105             if not unexpected:
 106                 out += " ok\n"
 107             else:
 108                 out += " " + result.upper() + "\n"
 109
 110         sys.stdout.write(out)
 111
 112     def start_test(self, testname):
 113         pass
 114
 115     def end_test(self, testname, result, unexpected, reason=None):
 116         if not unexpected:
 117             self.test_output[self.name] = ""
 118             if not self.immediate:
 119                 sys.stdout.write({
 120                     'failure': 'f',
 121                     'xfail': 'X',
 122                     'skip': 's',
 123                     'success': '.'}.get(result, "?(%s)" % result))
 124             return
 125
 126         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 127         if reason is not None:
 128             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
 129
 130         if self.immediate and not self.verbose:
 131             print self.test_output[self.name]
 132             self.test_output[self.name] = ""
 133
 134         if not self.immediate:
 135             sys.stdout.write({
 136                'error': 'E',
 137                'failure': 'F',
 138                'success': 'S'}.get(result, "?"))
 139
 140     def summary(self):
 141         f = open(self.summaryfile, 'w+')
 142
 143         if self.suitesfailed:
 144             f.write("= Failed tests =\n")
 145
 146             for suite in self.suitesfailed:
 147                 f.write("== %s ==\n" % suite)
 148                 f.write(self.test_output[suite]+"\n\n")
 149
 150             f.write("\n")
 151
 152         if not self.immediate and not self.verbose:
 153             for suite in self.suitesfailed:
 154                 print "=" * 78
 155                 print "FAIL: %s" % suite
 156                 print self.test_output[suite]
 157                 print ""
 158
 159         f.write("= Skipped tests =\n")
 160         for reason in self.skips.keys():
 161             f.write(reason + "\n")
 162             for name in self.skips[reason]:
 163                 f.write("\t%s\n" % name)
 164             f.write("\n")
 165         f.close()
 166
 167         print "\nA summary with detailed information can be found in:"
 168         print "  %s" % self.summaryfile
 169
 170         if not self.suitesfailed:
 171             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 172                   self.statistics['TESTS_EXPECTED_FAIL'])
 173             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 174         else:
 175             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 176                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 177                 self.statistics['TESTS_ERROR'],
 178                 len(self.suitesfailed))
 179
 180     def skip_testsuite(self, name, reason="UNKNOWN"):
 181         self.skips.setdefault(reason, []).append(name)
 182         if self.totalsuites:
 183             self.totalsuites-=1
 184
 185 parser = optparse.OptionParser("format-subunit [options]")
 186 parser.add_option("--verbose", action="store_true",
 187     help="Be verbose")
 188 parser.add_option("--immediate", action="store_true",
 189     help="Show failures immediately, don't wait until test run has finished")
 190 parser.add_option("--prefix", type="string", default=".",
 191     help="Prefix to write summary to")
 192
 193 opts, args = parser.parse_args()
 194
 195 statistics = {
 196     'SUITES_FAIL': 0,
 197     'TESTS_UNEXPECTED_OK': 0,
 198     'TESTS_EXPECTED_OK': 0,
 199     'TESTS_UNEXPECTED_FAIL': 0,
 200     'TESTS_EXPECTED_FAIL': 0,
 201     'TESTS_ERROR': 0,
 202     'TESTS_SKIP': 0,
 203 }
 204
 205 def handle_sigint(sig, stack):
 206         sys.exit(0)
 207 signal.signal(signal.SIGINT, handle_sigint)
 208
 209 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 210     opts.immediate, statistics)
 211
 212 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 213
 214 msg_ops.summary()
 215
 216 sys.exit(expected_ret)