selftest/subunithelper.py

   1 # Python module for parsing and generating the Subunit protocol
   2 # (Samba-specific)
   3 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 __all__ = ['parse_results']
  19
  20 import datetime
  21 import re
  22 import sys
  23 import os
  24 from samba import subunit
  25 from samba.subunit.run import TestProtocolClient
  26 import unittest
  27 try:
  28     from dateutil.parser import isoparse as iso_parse_date
  29 except ImportError:
  30     try:
  31         from iso8601 import parse_date as iso_parse_date;
  32     except ImportError:
  33         print('Install either python-dateutil >= 2.7.1 or python-iso8601')
  34
  35
  36 VALID_RESULTS = set(['success', 'successful', 'failure', 'fail', 'skip',
  37                      'knownfail', 'error', 'xfail', 'skip-testsuite',
  38                      'testsuite-failure', 'testsuite-xfail',
  39                      'testsuite-success', 'testsuite-error',
  40                      'uxsuccess', 'testsuite-uxsuccess'])
  41
  42
  43 class TestsuiteEnabledTestResult(unittest.TestResult):
  44
  45     def start_testsuite(self, name):
  46         raise NotImplementedError(self.start_testsuite)
  47
  48
  49 def parse_results(msg_ops, statistics, fh):
  50     exitcode = 0
  51     open_tests = {}
  52
  53     for l in fh:
  54         parts = l.split(None, 1)
  55         if not len(parts) == 2 or not l.startswith(parts[0]):
  56             msg_ops.output_msg(l)
  57             continue
  58         command = parts[0].rstrip(":")
  59         arg = parts[1]
  60         if command in ("test", "testing"):
  61             msg_ops.control_msg(l)
  62             name = arg.rstrip()
  63             test = subunit.RemotedTestCase(name)
  64             if name in open_tests:
  65                 msg_ops.addError(open_tests.pop(name), subunit.RemoteError(u"Test already running"))
  66             msg_ops.startTest(test)
  67             open_tests[name] = test
  68         elif command == "time":
  69             msg_ops.control_msg(l)
  70             try:
  71                 dt = iso_parse_date(arg.rstrip("\n"))
  72             except TypeError as e:
  73                 print("Unable to parse time line: %s" % arg.rstrip("\n"))
  74             else:
  75                 msg_ops.time(dt)
  76         elif command in VALID_RESULTS:
  77             msg_ops.control_msg(l)
  78             result = command
  79             grp = re.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
  80             (testname, hasreason) = (grp.group(1), grp.group(2))
  81             if hasreason:
  82                 reason = ""
  83                 # reason may be specified in next lines
  84                 terminated = False
  85                 for l in fh:
  86                     msg_ops.control_msg(l)
  87                     if l == "]\n":
  88                         terminated = True
  89                         break
  90                     else:
  91                         reason += l
  92
  93                 if isinstance(reason, bytes):
  94                     remote_error = subunit.RemoteError(reason.decode("utf-8"))
  95                 else:
  96                     remote_error = subunit.RemoteError(reason)
  97
  98                 if not terminated:
  99                     statistics['TESTS_ERROR'] += 1
 100                     msg_ops.addError(subunit.RemotedTestCase(testname),
 101                                      subunit.RemoteError(u"result (%s) reason (%s) interrupted" % (result, reason)))
 102                     return 1
 103             else:
 104                 reason = None
 105                 remote_error = subunit.RemoteError(u"No reason specified")
 106             if result in ("success", "successful"):
 107                 try:
 108                     test = open_tests.pop(testname)
 109                 except KeyError:
 110                     statistics['TESTS_ERROR'] += 1
 111                     exitcode = 1
 112                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 113                 else:
 114                     statistics['TESTS_EXPECTED_OK'] += 1
 115                     msg_ops.addSuccess(test)
 116             elif result in ("xfail", "knownfail"):
 117                 try:
 118                     test = open_tests.pop(testname)
 119                 except KeyError:
 120                     statistics['TESTS_ERROR'] += 1
 121                     exitcode = 1
 122                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 123                 else:
 124                     statistics['TESTS_EXPECTED_FAIL'] += 1
 125                     msg_ops.addExpectedFailure(test, remote_error)
 126             elif result in ("uxsuccess", ):
 127                 try:
 128                     test = open_tests.pop(testname)
 129                 except KeyError:
 130                     statistics['TESTS_ERROR'] += 1
 131                     exitcode = 1
 132                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 133                 else:
 134                     statistics['TESTS_UNEXPECTED_OK'] += 1
 135                     msg_ops.addUnexpectedSuccess(test)
 136                     exitcode = 1
 137             elif result in ("failure", "fail"):
 138                 try:
 139                     test = open_tests.pop(testname)
 140                 except KeyError:
 141                     statistics['TESTS_ERROR'] += 1
 142                     exitcode = 1
 143                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 144                 else:
 145                     statistics['TESTS_UNEXPECTED_FAIL'] += 1
 146                     exitcode = 1
 147                     msg_ops.addFailure(test, remote_error)
 148             elif result == "skip":
 149                 statistics['TESTS_SKIP'] += 1
 150                 # Allow tests to be skipped without prior announcement of test
 151                 try:
 152                     test = open_tests.pop(testname)
 153                 except KeyError:
 154                     test = subunit.RemotedTestCase(testname)
 155                 msg_ops.addSkip(test, reason)
 156             elif result == "error":
 157                 statistics['TESTS_ERROR'] += 1
 158                 exitcode = 1
 159                 try:
 160                     test = open_tests.pop(testname)
 161                 except KeyError:
 162                     test = subunit.RemotedTestCase(testname)
 163                 msg_ops.addError(test, remote_error)
 164             elif result == "skip-testsuite":
 165                 msg_ops.skip_testsuite(testname)
 166             elif result == "testsuite-success":
 167                 msg_ops.end_testsuite(testname, "success", reason)
 168             elif result == "testsuite-failure":
 169                 msg_ops.end_testsuite(testname, "failure", reason)
 170                 exitcode = 1
 171             elif result == "testsuite-xfail":
 172                 msg_ops.end_testsuite(testname, "xfail", reason)
 173             elif result == "testsuite-uxsuccess":
 174                 msg_ops.end_testsuite(testname, "uxsuccess", reason)
 175                 exitcode = 1
 176             elif result == "testsuite-error":
 177                 msg_ops.end_testsuite(testname, "error", reason)
 178                 exitcode = 1
 179             else:
 180                 raise AssertionError("Recognized but unhandled result %r" %
 181                                      result)
 182         elif command == "testsuite":
 183             msg_ops.start_testsuite(arg.strip())
 184         elif command == "progress":
 185             arg = arg.strip()
 186             if arg == "pop":
 187                 msg_ops.progress(None, subunit.PROGRESS_POP)
 188             elif arg == "push":
 189                 msg_ops.progress(None, subunit.PROGRESS_PUSH)
 190             elif arg[0] in '+-':
 191                 msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
 192             else:
 193                 msg_ops.progress(int(arg), subunit.PROGRESS_SET)
 194         else:
 195             msg_ops.output_msg(l)
 196
 197     while open_tests:
 198         test = subunit.RemotedTestCase(open_tests.popitem()[1])
 199         msg_ops.addError(test, subunit.RemoteError(u"was started but never finished!"))
 200         statistics['TESTS_ERROR'] += 1
 201         exitcode = 1
 202
 203     return exitcode
 204
 205
 206 class SubunitOps(TestProtocolClient, TestsuiteEnabledTestResult):
 207
 208     def progress(self, count, whence):
 209         if whence == subunit.PROGRESS_POP:
 210             self._stream.write("progress: pop\n")
 211         elif whence == subunit.PROGRESS_PUSH:
 212             self._stream.write("progress: push\n")
 213         elif whence == subunit.PROGRESS_SET:
 214             self._stream.write("progress: %d\n" % count)
 215         elif whence == subunit.PROGRESS_CUR:
 216             raise NotImplementedError
 217
 218     # The following are Samba extensions:
 219     def start_testsuite(self, name):
 220         self._stream.write("testsuite: %s\n" % name)
 221
 222     def skip_testsuite(self, name, reason=None):
 223         if reason:
 224             self._stream.write("skip-testsuite: %s [\n%s\n]\n" % (name, reason))
 225         else:
 226             self._stream.write("skip-testsuite: %s\n" % name)
 227
 228     def end_testsuite(self, name, result, reason=None):
 229         if reason:
 230             self._stream.write("testsuite-%s: %s [\n%s\n]\n" % (result, name, reason))
 231         else:
 232             self._stream.write("testsuite-%s: %s\n" % (result, name))
 233
 234     def output_msg(self, msg):
 235         self._stream.write(msg)
 236
 237
 238 def read_test_regexes(*names):
 239     ret = {}
 240     files = []
 241     for name in names:
 242         # if we are given a directory, we read all the files it contains
 243         # (except the ones that end with "~").
 244         if os.path.isdir(name):
 245             files.extend([os.path.join(name, x)
 246                           for x in os.listdir(name)
 247                           if x[-1] != '~'])
 248         else:
 249             files.append(name)
 250
 251     for filename in files:
 252         with open(filename, 'r') as f:
 253             for l in f:
 254                 l = l.strip()
 255                 if l == "" or l[0] == "#":
 256                     continue
 257                 if "#" in l:
 258                     (regex, reason) = l.split("#", 1)
 259                     ret[regex.strip()] = reason.strip()
 260                 else:
 261                     ret[l] = None
 262
 263     return ret
 264
 265
 266 def find_in_list(regexes, fullname):
 267     for regex, reason in regexes.items():
 268         if re.match(regex, fullname):
 269             if reason is None:
 270                 return ""
 271             return reason
 272     return None
 273
 274
 275 class ImmediateFail(Exception):
 276     """Raised to abort immediately."""
 277
 278     def __init__(self):
 279         super(ImmediateFail, self).__init__("test failed and fail_immediately set")
 280
 281
 282 class FilterOps(unittest.TestResult):
 283
 284     def control_msg(self, msg):
 285         pass  # We regenerate control messages, so ignore this
 286
 287     def time(self, time):
 288         self._ops.time(time)
 289
 290     def progress(self, delta, whence):
 291         self._ops.progress(delta, whence)
 292
 293     def output_msg(self, msg):
 294         if self.output is None:
 295             sys.stdout.write(msg)
 296         else:
 297             self.output += msg
 298
 299     def startTest(self, test):
 300         self.seen_output = True
 301         test = self._add_prefix(test)
 302         if self.strip_ok_output:
 303             self.output = ""
 304
 305         self._ops.startTest(test)
 306
 307     def _add_prefix(self, test):
 308         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 309
 310     def addError(self, test, err=None):
 311         test = self._add_prefix(test)
 312         self.error_added += 1
 313         self.total_error += 1
 314         self._ops.addError(test, err)
 315         self._ops.writeOutcome(test)
 316         self.output = None
 317         if self.fail_immediately:
 318             raise ImmediateFail()
 319
 320     def addSkip(self, test, reason=None):
 321         self.seen_output = True
 322         test = self._add_prefix(test)
 323         self._ops.addSkip(test, reason)
 324         self._ops.writeOutcome(test)
 325         self.output = None
 326
 327     def addExpectedFailure(self, test, err=None):
 328         test = self._add_prefix(test)
 329         self._ops.addExpectedFailure(test, err)
 330         self._ops.writeOutcome(test)
 331         self.output = None
 332
 333     def addUnexpectedSuccess(self, test):
 334         test = self._add_prefix(test)
 335         self.uxsuccess_added += 1
 336         self.total_uxsuccess += 1
 337         self._ops.addUnexpectedSuccess(test)
 338         self._ops.writeOutcome(test)
 339         if self.output:
 340             self._ops.output_msg(self.output)
 341         self.output = None
 342         if self.fail_immediately:
 343             raise ImmediateFail()
 344
 345     def addFailure(self, test, err=None):
 346         test = self._add_prefix(test)
 347         xfail_reason = find_in_list(self.expected_failures, test.id())
 348         if xfail_reason is None:
 349             xfail_reason = find_in_list(self.flapping, test.id())
 350         if xfail_reason is not None:
 351             self.xfail_added += 1
 352             self.total_xfail += 1
 353             self._ops.addExpectedFailure(test, err)
 354             self._ops.writeOutcome(test)
 355         else:
 356             self.fail_added += 1
 357             self.total_fail += 1
 358             self._ops.addFailure(test, err)
 359             self._ops.writeOutcome(test)
 360             if self.output:
 361                 self._ops.output_msg(self.output)
 362             if self.fail_immediately:
 363                 raise ImmediateFail()
 364         self.output = None
 365
 366     def addSuccess(self, test):
 367         test = self._add_prefix(test)
 368         xfail_reason = find_in_list(self.expected_failures, test.id())
 369         if xfail_reason is not None:
 370             self.uxsuccess_added += 1
 371             self.total_uxsuccess += 1
 372             self._ops.addUnexpectedSuccess(test)
 373             self._ops.writeOutcome(test)
 374             if self.output:
 375                 self._ops.output_msg(self.output)
 376             if self.fail_immediately:
 377                 raise ImmediateFail()
 378         else:
 379             self._ops.addSuccess(test)
 380             self._ops.writeOutcome(test)
 381         self.output = None
 382
 383     def skip_testsuite(self, name, reason=None):
 384         self._ops.skip_testsuite(name, reason)
 385
 386     def start_testsuite(self, name):
 387         self._ops.start_testsuite(name)
 388         self.error_added = 0
 389         self.fail_added = 0
 390         self.xfail_added = 0
 391         self.uxsuccess_added = 0
 392
 393     def end_testsuite(self, name, result, reason=None):
 394         xfail = False
 395
 396         if self.xfail_added > 0:
 397             xfail = True
 398         if self.fail_added > 0 or self.error_added > 0 or self.uxsuccess_added > 0:
 399             xfail = False
 400
 401         if xfail and result in ("fail", "failure"):
 402             result = "xfail"
 403
 404         if self.uxsuccess_added > 0 and result != "uxsuccess":
 405             result = "uxsuccess"
 406             if reason is None:
 407                 reason = "Subunit/Filter Reason"
 408             reason += "\n uxsuccess[%d]" % self.uxsuccess_added
 409
 410         if self.fail_added > 0 and result != "failure":
 411             result = "failure"
 412             if reason is None:
 413                 reason = "Subunit/Filter Reason"
 414             reason += "\n failures[%d]" % self.fail_added
 415
 416         if self.error_added > 0 and result != "error":
 417             result = "error"
 418             if reason is None:
 419                 reason = "Subunit/Filter Reason"
 420             reason += "\n errors[%d]" % self.error_added
 421
 422         self._ops.end_testsuite(name, result, reason)
 423         if result not in ("success", "xfail"):
 424             if self.output:
 425                 self._ops.output_msg(self.output)
 426             if self.fail_immediately:
 427                 raise ImmediateFail()
 428         self.output = None
 429
 430     def __init__(self, out, prefix=None, suffix=None, expected_failures=None,
 431                  strip_ok_output=False, fail_immediately=False,
 432                  flapping=None):
 433         self._ops = out
 434         self.seen_output = False
 435         self.output = None
 436         self.prefix = prefix
 437         self.suffix = suffix
 438         if expected_failures is not None:
 439             self.expected_failures = expected_failures
 440         else:
 441             self.expected_failures = {}
 442         if flapping is not None:
 443             self.flapping = flapping
 444         else:
 445             self.flapping = {}
 446         self.strip_ok_output = strip_ok_output
 447         self.xfail_added = 0
 448         self.fail_added = 0
 449         self.uxsuccess_added = 0
 450         self.total_xfail = 0
 451         self.total_error = 0
 452         self.total_fail = 0
 453         self.total_uxsuccess = 0
 454         self.error_added = 0
 455         self.fail_immediately = fail_immediately
 456
 457
 458 class PerfFilterOps(unittest.TestResult):
 459
 460     def progress(self, delta, whence):
 461         pass
 462
 463     def output_msg(self, msg):
 464         pass
 465
 466     def control_msg(self, msg):
 467         pass
 468
 469     def skip_testsuite(self, name, reason=None):
 470         self._ops.skip_testsuite(name, reason)
 471
 472     def start_testsuite(self, name):
 473         self.suite_has_time = False
 474
 475     def end_testsuite(self, name, result, reason=None):
 476         pass
 477
 478     def _add_prefix(self, test):
 479         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 480
 481     def time(self, time):
 482         self.latest_time = time
 483         #self._ops.output_msg("found time %s\n" % time)
 484         self.suite_has_time = True
 485
 486     def get_time(self):
 487         if self.suite_has_time:
 488             return self.latest_time
 489         return datetime.datetime.utcnow()
 490
 491     def startTest(self, test):
 492         self.seen_output = True
 493         test = self._add_prefix(test)
 494         self.starts[test.id()] = self.get_time()
 495
 496     def addSuccess(self, test):
 497         test = self._add_prefix(test)
 498         tid = test.id()
 499         if tid not in self.starts:
 500             self._ops.addError(test, "%s succeeded without ever starting!" % tid)
 501         delta = self.get_time() - self.starts[tid]
 502         self._ops.output_msg("elapsed-time: %s: %f\n" % (tid, delta.total_seconds()))
 503
 504     def addFailure(self, test, err=''):
 505         tid = test.id()
 506         delta = self.get_time() - self.starts[tid]
 507         self._ops.output_msg("failure: %s failed after %f seconds (%s)\n" %
 508                              (tid, delta.total_seconds(), err))
 509
 510     def addError(self, test, err=''):
 511         tid = test.id()
 512         delta = self.get_time() - self.starts[tid]
 513         self._ops.output_msg("error: %s failed after %f seconds (%s)\n" %
 514                              (tid, delta.total_seconds(), err))
 515
 516     def __init__(self, out, prefix='', suffix=''):
 517         self._ops = out
 518         self.prefix = prefix or ''
 519         self.suffix = suffix or ''
 520         self.starts = {}
 521         self.seen_output = False
 522         self.suite_has_time = False
 523
 524
 525 class PlainFormatter(TestsuiteEnabledTestResult):
 526
 527     def __init__(self, verbose, immediate, statistics,
 528                  totaltests=None):
 529         super(PlainFormatter, self).__init__()
 530         self.verbose = verbose
 531         self.immediate = immediate
 532         self.statistics = statistics
 533         self.start_time = None
 534         self.test_output = {}
 535         self.suitesfailed = []
 536         self.suites_ok = 0
 537         self.skips = {}
 538         self.index = 0
 539         self.name = None
 540         self._progress_level = 0
 541         self.totalsuites = totaltests
 542         self.last_time = None
 543
 544     @staticmethod
 545     def _format_time(delta):
 546         minutes, seconds = divmod(delta.seconds, 60)
 547         hours, minutes = divmod(minutes, 60)
 548         ret = ""
 549         if hours:
 550             ret += "%dh" % hours
 551         if minutes:
 552             ret += "%dm" % minutes
 553         ret += "%ds" % seconds
 554         return ret
 555
 556     def progress(self, offset, whence):
 557         if whence == subunit.PROGRESS_POP:
 558             self._progress_level -= 1
 559         elif whence == subunit.PROGRESS_PUSH:
 560             self._progress_level += 1
 561         elif whence == subunit.PROGRESS_SET:
 562             if self._progress_level == 0:
 563                 self.totalsuites = offset
 564         elif whence == subunit.PROGRESS_CUR:
 565             raise NotImplementedError
 566
 567     def time(self, dt):
 568         if self.start_time is None:
 569             self.start_time = dt
 570         self.last_time = dt
 571
 572     def start_testsuite(self, name):
 573         self.index += 1
 574         self.name = name
 575
 576         if not self.verbose:
 577             self.test_output[name] = ""
 578
 579         total_tests = (self.statistics['TESTS_EXPECTED_OK'] +
 580                        self.statistics['TESTS_EXPECTED_FAIL'] +
 581                        self.statistics['TESTS_ERROR'] +
 582                        self.statistics['TESTS_UNEXPECTED_FAIL'] +
 583                        self.statistics['TESTS_UNEXPECTED_OK'])
 584
 585         out = "[%d(%d)" % (self.index, total_tests)
 586         if self.totalsuites is not None:
 587             out += "/%d" % self.totalsuites
 588         if self.start_time is not None:
 589             out += " at " + self._format_time(self.last_time - self.start_time)
 590         if self.suitesfailed:
 591             out += ", %d errors" % (len(self.suitesfailed),)
 592         out += "] %s" % name
 593         if self.immediate:
 594             sys.stdout.write(out + "\n")
 595         else:
 596             sys.stdout.write(out + ": ")
 597
 598     def output_msg(self, output):
 599         if self.verbose:
 600             sys.stdout.write(output)
 601         elif self.name is not None:
 602             self.test_output[self.name] += output
 603         else:
 604             sys.stdout.write(output)
 605
 606     def control_msg(self, output):
 607         pass
 608
 609     def end_testsuite(self, name, result, reason):
 610         out = ""
 611         unexpected = False
 612
 613         if name not in self.test_output:
 614             print("no output for name[%s]" % name)
 615
 616         if result in ("success", "xfail"):
 617             self.suites_ok += 1
 618         else:
 619             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 620             if reason is not None:
 621                 self.output_msg("REASON: %s\n" % (reason,))
 622             self.suitesfailed.append(name)
 623             if self.immediate and not self.verbose and name in self.test_output:
 624                 out += self.test_output[name]
 625             unexpected = True
 626
 627         if not self.immediate:
 628             if not unexpected:
 629                 out += " ok\n"
 630             else:
 631                 out += " " + result.upper() + "\n"
 632
 633         sys.stdout.write(out)
 634
 635     def startTest(self, test):
 636         pass
 637
 638     def addSuccess(self, test):
 639         self.end_test(test.id(), "success", False)
 640
 641     def addError(self, test, err=None):
 642         self.end_test(test.id(), "error", True, err)
 643
 644     def addFailure(self, test, err=None):
 645         self.end_test(test.id(), "failure", True, err)
 646
 647     def addSkip(self, test, reason=None):
 648         self.end_test(test.id(), "skip", False, reason)
 649
 650     def addExpectedFailure(self, test, err=None):
 651         self.end_test(test.id(), "xfail", False, err)
 652
 653     def addUnexpectedSuccess(self, test):
 654         self.end_test(test.id(), "uxsuccess", True)
 655
 656     def end_test(self, testname, result, unexpected, err=None):
 657         if not unexpected:
 658             self.test_output[self.name] = ""
 659             if not self.immediate:
 660                 sys.stdout.write({
 661                     'failure': 'f',
 662                     'xfail': 'X',
 663                     'skip': 's',
 664                     'success': '.'}.get(result, "?(%s)" % result))
 665             return
 666
 667         if self.name not in self.test_output:
 668             self.test_output[self.name] = ""
 669
 670         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 671         if err is not None:
 672             self.test_output[self.name] += "REASON: %s\n" % str(err[1]).strip()
 673
 674         if self.immediate and not self.verbose:
 675             sys.stdout.write(self.test_output[self.name])
 676             self.test_output[self.name] = ""
 677
 678         if not self.immediate:
 679             sys.stdout.write({
 680                 'error': 'E',
 681                'failure': 'F',
 682                'uxsuccess': 'U',
 683                'success': 'S'}.get(result, "?"))
 684
 685     def write_summary(self, path):
 686         f = open(path, 'w+')
 687
 688         if self.suitesfailed:
 689             f.write("= Failed tests =\n")
 690
 691             for suite in self.suitesfailed:
 692                 f.write("== %s ==\n" % suite)
 693                 if suite in self.test_output:
 694                     f.write(self.test_output[suite] + "\n\n")
 695
 696             f.write("\n")
 697
 698         if not self.immediate and not self.verbose:
 699             for suite in self.suitesfailed:
 700                 print("=" * 78)
 701                 print("FAIL: %s" % suite)
 702                 if suite in self.test_output:
 703                     print(self.test_output[suite])
 704                 print("")
 705
 706         f.write("= Skipped tests =\n")
 707         for reason in self.skips.keys():
 708             f.write(reason + "\n")
 709             for name in self.skips[reason]:
 710                 f.write("\t%s\n" % name)
 711             f.write("\n")
 712         f.close()
 713
 714         if (not self.suitesfailed and
 715             not self.statistics['TESTS_UNEXPECTED_FAIL'] and
 716             not self.statistics['TESTS_UNEXPECTED_OK'] and
 717             not self.statistics['TESTS_ERROR']):
 718             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 719                   self.statistics['TESTS_EXPECTED_FAIL'])
 720             print("\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok))
 721         else:
 722             print("\nFAILED (%d failures, %d errors and %d unexpected successes in %d testsuites)" % (
 723                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 724                 self.statistics['TESTS_ERROR'],
 725                 self.statistics['TESTS_UNEXPECTED_OK'],
 726                 len(self.suitesfailed)))
 727
 728     def skip_testsuite(self, name, reason="UNKNOWN"):
 729         self.skips.setdefault(reason, []).append(name)
 730         if self.totalsuites:
 731             self.totalsuites -= 1