selftest/subunithelper.py

   1 # Python module for parsing and generating the Subunit protocol
   2 # (Samba-specific)
   3 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 __all__ = ['parse_results']
  19
  20 import datetime
  21 import re
  22 import sys
  23 import os
  24 from samba import subunit
  25 from samba.subunit.run import TestProtocolClient
  26 import unittest
  27 try:
  28     from dateutil.parser import isoparse as iso_parse_date
  29 except ImportError:
  30     try:
  31         from iso8601 import parse_date as iso_parse_date;
  32     except ImportError:
  33         print('Install either python-dateutil >= 2.7.1 or python-iso8601')
  34
  35
  36 VALID_RESULTS = set(['success', 'successful', 'failure', 'fail', 'skip',
  37                      'knownfail', 'error', 'xfail', 'skip-testsuite',
  38                      'testsuite-failure', 'testsuite-xfail',
  39                      'testsuite-success', 'testsuite-error',
  40                      'uxsuccess', 'testsuite-uxsuccess'])
  41
  42
  43 class TestsuiteEnabledTestResult(unittest.TestResult):
  44
  45     def start_testsuite(self, name):
  46         raise NotImplementedError(self.start_testsuite)
  47
  48
  49 def parse_results(msg_ops, statistics, fh):
  50     exitcode = 0
  51     open_tests = {}
  52
  53     for l in fh:
  54         parts = l.split(None, 1)
  55         if not len(parts) == 2 or not l.startswith(parts[0]):
  56             msg_ops.output_msg(l)
  57             continue
  58         command = parts[0].rstrip(":")
  59         arg = parts[1]
  60         if command in ("test", "testing"):
  61             msg_ops.control_msg(l)
  62             name = arg.rstrip()
  63             test = subunit.RemotedTestCase(name)
  64             if name in open_tests:
  65                 msg_ops.addError(open_tests.pop(name), subunit.RemoteError(u"Test already running"))
  66             msg_ops.startTest(test)
  67             open_tests[name] = test
  68         elif command == "time":
  69             msg_ops.control_msg(l)
  70             try:
  71                 dt = iso_parse_date(arg.rstrip("\n"))
  72             except TypeError as e:
  73                 print("Unable to parse time line: %s" % arg.rstrip("\n"))
  74             else:
  75                 msg_ops.time(dt)
  76         elif command in VALID_RESULTS:
  77             msg_ops.control_msg(l)
  78             result = command
  79             grp = re.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg)
  80             (testname, hasreason) = (grp.group(1), grp.group(2))
  81             if hasreason:
  82                 reason = ""
  83                 # reason may be specified in next lines
  84                 terminated = False
  85                 for l in fh:
  86                     msg_ops.control_msg(l)
  87                     if l == "]\n":
  88                         terminated = True
  89                         break
  90                     else:
  91                         reason += l
  92
  93                 if isinstance(reason, bytes):
  94                     remote_error = subunit.RemoteError(reason.decode("utf-8"))
  95                 else:
  96                     remote_error = subunit.RemoteError(reason)
  97
  98                 if not terminated:
  99                     statistics['TESTS_ERROR'] += 1
 100                     msg_ops.addError(subunit.RemotedTestCase(testname),
 101                                      subunit.RemoteError(u"result (%s) reason (%s) interrupted" % (result, reason)))
 102                     return 1
 103             else:
 104                 reason = None
 105                 remote_error = subunit.RemoteError(u"No reason specified")
 106             if result in ("success", "successful"):
 107                 try:
 108                     test = open_tests.pop(testname)
 109                 except KeyError:
 110                     statistics['TESTS_ERROR'] += 1
 111                     exitcode = 1
 112                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 113                 else:
 114                     statistics['TESTS_EXPECTED_OK'] += 1
 115                     msg_ops.addSuccess(test)
 116             elif result in ("xfail", "knownfail"):
 117                 try:
 118                     test = open_tests.pop(testname)
 119                 except KeyError:
 120                     statistics['TESTS_ERROR'] += 1
 121                     exitcode = 1
 122                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 123                 else:
 124                     statistics['TESTS_EXPECTED_FAIL'] += 1
 125                     msg_ops.addExpectedFailure(test, remote_error)
 126             elif result in ("uxsuccess", ):
 127                 try:
 128                     test = open_tests.pop(testname)
 129                 except KeyError:
 130                     statistics['TESTS_ERROR'] += 1
 131                     exitcode = 1
 132                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 133                 else:
 134                     statistics['TESTS_UNEXPECTED_OK'] += 1
 135                     msg_ops.addUnexpectedSuccess(test)
 136                     exitcode = 1
 137             elif result in ("failure", "fail"):
 138                 try:
 139                     test = open_tests.pop(testname)
 140                 except KeyError:
 141                     statistics['TESTS_ERROR'] += 1
 142                     exitcode = 1
 143                     msg_ops.addError(subunit.RemotedTestCase(testname), subunit.RemoteError(u"Test was never started"))
 144                 else:
 145                     statistics['TESTS_UNEXPECTED_FAIL'] += 1
 146                     exitcode = 1
 147                     msg_ops.addFailure(test, remote_error)
 148             elif result == "skip":
 149                 statistics['TESTS_SKIP'] += 1
 150                 # Allow tests to be skipped without prior announcement of test
 151                 try:
 152                     test = open_tests.pop(testname)
 153                 except KeyError:
 154                     test = subunit.RemotedTestCase(testname)
 155                 msg_ops.addSkip(test, reason)
 156             elif result == "error":
 157                 statistics['TESTS_ERROR'] += 1
 158                 exitcode = 1
 159                 try:
 160                     test = open_tests.pop(testname)
 161                 except KeyError:
 162                     test = subunit.RemotedTestCase(testname)
 163                 msg_ops.addError(test, remote_error)
 164             elif result == "skip-testsuite":
 165                 msg_ops.skip_testsuite(testname)
 166             elif result == "testsuite-success":
 167                 msg_ops.end_testsuite(testname, "success", reason)
 168             elif result == "testsuite-failure":
 169                 msg_ops.end_testsuite(testname, "failure", reason)
 170                 exitcode = 1
 171             elif result == "testsuite-xfail":
 172                 msg_ops.end_testsuite(testname, "xfail", reason)
 173             elif result == "testsuite-uxsuccess":
 174                 msg_ops.end_testsuite(testname, "uxsuccess", reason)
 175                 exitcode = 1
 176             elif result == "testsuite-error":
 177                 msg_ops.end_testsuite(testname, "error", reason)
 178                 exitcode = 1
 179             else:
 180                 raise AssertionError("Recognized but unhandled result %r" %
 181                                      result)
 182         elif command == "testsuite":
 183             msg_ops.start_testsuite(arg.strip())
 184         elif command == "progress":
 185             arg = arg.strip()
 186             if arg == "pop":
 187                 msg_ops.progress(None, subunit.PROGRESS_POP)
 188             elif arg == "push":
 189                 msg_ops.progress(None, subunit.PROGRESS_PUSH)
 190             elif arg[0] in '+-':
 191                 msg_ops.progress(int(arg), subunit.PROGRESS_CUR)
 192             else:
 193                 msg_ops.progress(int(arg), subunit.PROGRESS_SET)
 194         else:
 195             msg_ops.output_msg(l)
 196
 197     while open_tests:
 198         test = subunit.RemotedTestCase(open_tests.popitem()[1])
 199         msg_ops.addError(test, subunit.RemoteError(u"was started but never finished!"))
 200         statistics['TESTS_ERROR'] += 1
 201         exitcode = 1
 202
 203     return exitcode
 204
 205
 206 class SubunitOps(TestProtocolClient, TestsuiteEnabledTestResult):
 207
 208     def progress(self, count, whence):
 209         if whence == subunit.PROGRESS_POP:
 210             self._stream.write("progress: pop\n")
 211         elif whence == subunit.PROGRESS_PUSH:
 212             self._stream.write("progress: push\n")
 213         elif whence == subunit.PROGRESS_SET:
 214             self._stream.write("progress: %d\n" % count)
 215         elif whence == subunit.PROGRESS_CUR:
 216             raise NotImplementedError
 217
 218     # The following are Samba extensions:
 219     def start_testsuite(self, name):
 220         self._stream.write("testsuite: %s\n" % name)
 221
 222     def skip_testsuite(self, name, reason=None):
 223         if reason:
 224             self._stream.write("skip-testsuite: %s [\n%s\n]\n" % (name, reason))
 225         else:
 226             self._stream.write("skip-testsuite: %s\n" % name)
 227
 228     def end_testsuite(self, name, result, reason=None):
 229         if reason:
 230             self._stream.write("testsuite-%s: %s [\n%s\n]\n" % (result, name, reason))
 231         else:
 232             self._stream.write("testsuite-%s: %s\n" % (result, name))
 233
 234     def output_msg(self, msg):
 235         self._stream.write(msg)
 236
 237
 238 def read_test_regexes(*names):
 239     ret = []
 240     files = []
 241     for name in names:
 242         # if we are given a directory, we read all the files it contains
 243         # (except the ones that end with "~").
 244         if os.path.isdir(name):
 245             files.extend([os.path.join(name, x)
 246                           for x in os.listdir(name)
 247                           if x[-1] != '~'])
 248         else:
 249             files.append(name)
 250
 251     for filename in files:
 252         with open(filename, 'r') as f:
 253             for l in f:
 254                 l = l.strip()
 255                 if l == "" or l[0] == "#":
 256                     continue
 257                 if "#" in l:
 258                     (regex, reason) = l.split("#", 1)
 259                     ret.append(re.compile(regex.strip()))
 260                 else:
 261                     ret.append(re.compile(l))
 262
 263     return ret
 264
 265
 266 def find_in_list(regexes, fullname):
 267     for regex in regexes:
 268         if regex.match(fullname):
 269             return True
 270     return False
 271
 272
 273 class ImmediateFail(Exception):
 274     """Raised to abort immediately."""
 275
 276     def __init__(self):
 277         super(ImmediateFail, self).__init__("test failed and fail_immediately set")
 278
 279
 280 class FilterOps(unittest.TestResult):
 281
 282     def control_msg(self, msg):
 283         pass  # We regenerate control messages, so ignore this
 284
 285     def time(self, time):
 286         self._ops.time(time)
 287
 288     def progress(self, delta, whence):
 289         self._ops.progress(delta, whence)
 290
 291     def output_msg(self, msg):
 292         if self.output is None:
 293             sys.stdout.write(msg)
 294         else:
 295             self.output += msg
 296
 297     def startTest(self, test):
 298         self.seen_output = True
 299         test = self._add_prefix(test)
 300         if self.strip_ok_output:
 301             self.output = ""
 302
 303         self._ops.startTest(test)
 304
 305     def _add_prefix(self, test):
 306         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 307
 308     def addError(self, test, err=None):
 309         test = self._add_prefix(test)
 310         self.error_added += 1
 311         self.total_error += 1
 312         self._ops.addError(test, err)
 313         self._ops.writeOutcome(test)
 314         self.output = None
 315         if self.fail_immediately:
 316             raise ImmediateFail()
 317
 318     def addSkip(self, test, reason=None):
 319         self.seen_output = True
 320         test = self._add_prefix(test)
 321         self._ops.addSkip(test, reason)
 322         self._ops.writeOutcome(test)
 323         self.output = None
 324
 325     def addExpectedFailure(self, test, err=None):
 326         test = self._add_prefix(test)
 327         self._ops.addExpectedFailure(test, err)
 328         self._ops.writeOutcome(test)
 329         self.output = None
 330
 331     def addUnexpectedSuccess(self, test):
 332         test = self._add_prefix(test)
 333         self.uxsuccess_added += 1
 334         self.total_uxsuccess += 1
 335         self._ops.addUnexpectedSuccess(test)
 336         self._ops.writeOutcome(test)
 337         if self.output:
 338             self._ops.output_msg(self.output)
 339         self.output = None
 340         if self.fail_immediately:
 341             raise ImmediateFail()
 342
 343     def addFailure(self, test, err=None):
 344         test = self._add_prefix(test)
 345         xfail = find_in_list(self.expected_failures, test.id())
 346         if not xfail:
 347             xfail = find_in_list(self.flapping, test.id())
 348         if xfail:
 349             self.xfail_added += 1
 350             self.total_xfail += 1
 351             self._ops.addExpectedFailure(test, err)
 352             self._ops.writeOutcome(test)
 353         else:
 354             self.fail_added += 1
 355             self.total_fail += 1
 356             self._ops.addFailure(test, err)
 357             self._ops.writeOutcome(test)
 358             if self.output:
 359                 self._ops.output_msg(self.output)
 360             if self.fail_immediately:
 361                 raise ImmediateFail()
 362         self.output = None
 363
 364     def addSuccess(self, test):
 365         test = self._add_prefix(test)
 366         xfail = find_in_list(self.expected_failures, test.id())
 367         if xfail:
 368             self.uxsuccess_added += 1
 369             self.total_uxsuccess += 1
 370             self._ops.addUnexpectedSuccess(test)
 371             self._ops.writeOutcome(test)
 372             if self.output:
 373                 self._ops.output_msg(self.output)
 374             if self.fail_immediately:
 375                 raise ImmediateFail()
 376         else:
 377             self._ops.addSuccess(test)
 378             self._ops.writeOutcome(test)
 379         self.output = None
 380
 381     def skip_testsuite(self, name, reason=None):
 382         self._ops.skip_testsuite(name, reason)
 383
 384     def start_testsuite(self, name):
 385         self._ops.start_testsuite(name)
 386         self.error_added = 0
 387         self.fail_added = 0
 388         self.xfail_added = 0
 389         self.uxsuccess_added = 0
 390
 391     def end_testsuite(self, name, result, reason=None):
 392         xfail = False
 393
 394         if self.xfail_added > 0:
 395             xfail = True
 396         if self.fail_added > 0 or self.error_added > 0 or self.uxsuccess_added > 0:
 397             xfail = False
 398
 399         if xfail and result in ("fail", "failure"):
 400             result = "xfail"
 401
 402         if self.uxsuccess_added > 0 and result != "uxsuccess":
 403             result = "uxsuccess"
 404             if reason is None:
 405                 reason = "Subunit/Filter Reason"
 406             reason += "\n uxsuccess[%d]" % self.uxsuccess_added
 407
 408         if self.fail_added > 0 and result != "failure":
 409             result = "failure"
 410             if reason is None:
 411                 reason = "Subunit/Filter Reason"
 412             reason += "\n failures[%d]" % self.fail_added
 413
 414         if self.error_added > 0 and result != "error":
 415             result = "error"
 416             if reason is None:
 417                 reason = "Subunit/Filter Reason"
 418             reason += "\n errors[%d]" % self.error_added
 419
 420         self._ops.end_testsuite(name, result, reason)
 421         if result not in ("success", "xfail"):
 422             if self.output:
 423                 self._ops.output_msg(self.output)
 424             if self.fail_immediately:
 425                 raise ImmediateFail()
 426         self.output = None
 427
 428     def __init__(self, out, prefix=None, suffix=None, expected_failures=None,
 429                  strip_ok_output=False, fail_immediately=False,
 430                  flapping=None):
 431         self._ops = out
 432         self.seen_output = False
 433         self.output = None
 434         self.prefix = prefix
 435         self.suffix = suffix
 436         if expected_failures is not None:
 437             self.expected_failures = expected_failures
 438         else:
 439             self.expected_failures = []
 440         if flapping is not None:
 441             self.flapping = flapping
 442         else:
 443             self.flapping = []
 444         self.strip_ok_output = strip_ok_output
 445         self.xfail_added = 0
 446         self.fail_added = 0
 447         self.uxsuccess_added = 0
 448         self.total_xfail = 0
 449         self.total_error = 0
 450         self.total_fail = 0
 451         self.total_uxsuccess = 0
 452         self.error_added = 0
 453         self.fail_immediately = fail_immediately
 454
 455
 456 class PerfFilterOps(unittest.TestResult):
 457
 458     def progress(self, delta, whence):
 459         pass
 460
 461     def output_msg(self, msg):
 462         pass
 463
 464     def control_msg(self, msg):
 465         pass
 466
 467     def skip_testsuite(self, name, reason=None):
 468         self._ops.skip_testsuite(name, reason)
 469
 470     def start_testsuite(self, name):
 471         self.suite_has_time = False
 472
 473     def end_testsuite(self, name, result, reason=None):
 474         pass
 475
 476     def _add_prefix(self, test):
 477         return subunit.RemotedTestCase(self.prefix + test.id() + self.suffix)
 478
 479     def time(self, time):
 480         self.latest_time = time
 481         #self._ops.output_msg("found time %s\n" % time)
 482         self.suite_has_time = True
 483
 484     def get_time(self):
 485         if self.suite_has_time:
 486             return self.latest_time
 487         return datetime.datetime.utcnow()
 488
 489     def startTest(self, test):
 490         self.seen_output = True
 491         test = self._add_prefix(test)
 492         self.starts[test.id()] = self.get_time()
 493
 494     def addSuccess(self, test):
 495         test = self._add_prefix(test)
 496         tid = test.id()
 497         if tid not in self.starts:
 498             self._ops.addError(test, "%s succeeded without ever starting!" % tid)
 499         delta = self.get_time() - self.starts[tid]
 500         self._ops.output_msg("elapsed-time: %s: %f\n" % (tid, delta.total_seconds()))
 501
 502     def addFailure(self, test, err=''):
 503         tid = test.id()
 504         delta = self.get_time() - self.starts[tid]
 505         self._ops.output_msg("failure: %s failed after %f seconds (%s)\n" %
 506                              (tid, delta.total_seconds(), err))
 507
 508     def addError(self, test, err=''):
 509         tid = test.id()
 510         delta = self.get_time() - self.starts[tid]
 511         self._ops.output_msg("error: %s failed after %f seconds (%s)\n" %
 512                              (tid, delta.total_seconds(), err))
 513
 514     def __init__(self, out, prefix='', suffix=''):
 515         self._ops = out
 516         self.prefix = prefix or ''
 517         self.suffix = suffix or ''
 518         self.starts = {}
 519         self.seen_output = False
 520         self.suite_has_time = False
 521
 522
 523 class PlainFormatter(TestsuiteEnabledTestResult):
 524
 525     def __init__(self, verbose, immediate, statistics,
 526                  totaltests=None):
 527         super(PlainFormatter, self).__init__()
 528         self.verbose = verbose
 529         self.immediate = immediate
 530         self.statistics = statistics
 531         self.start_time = None
 532         self.test_output = {}
 533         self.suitesfailed = []
 534         self.suites_ok = 0
 535         self.skips = {}
 536         self.index = 0
 537         self.name = None
 538         self._progress_level = 0
 539         self.totalsuites = totaltests
 540         self.last_time = None
 541
 542     @staticmethod
 543     def _format_time(delta):
 544         minutes, seconds = divmod(delta.seconds, 60)
 545         hours, minutes = divmod(minutes, 60)
 546         ret = ""
 547         if hours:
 548             ret += "%dh" % hours
 549         if minutes:
 550             ret += "%dm" % minutes
 551         ret += "%ds" % seconds
 552         return ret
 553
 554     def progress(self, offset, whence):
 555         if whence == subunit.PROGRESS_POP:
 556             self._progress_level -= 1
 557         elif whence == subunit.PROGRESS_PUSH:
 558             self._progress_level += 1
 559         elif whence == subunit.PROGRESS_SET:
 560             if self._progress_level == 0:
 561                 self.totalsuites = offset
 562         elif whence == subunit.PROGRESS_CUR:
 563             raise NotImplementedError
 564
 565     def time(self, dt):
 566         if self.start_time is None:
 567             self.start_time = dt
 568         self.last_time = dt
 569
 570     def start_testsuite(self, name):
 571         self.index += 1
 572         self.name = name
 573
 574         if not self.verbose:
 575             self.test_output[name] = ""
 576
 577         total_tests = (self.statistics['TESTS_EXPECTED_OK'] +
 578                        self.statistics['TESTS_EXPECTED_FAIL'] +
 579                        self.statistics['TESTS_ERROR'] +
 580                        self.statistics['TESTS_UNEXPECTED_FAIL'] +
 581                        self.statistics['TESTS_UNEXPECTED_OK'])
 582
 583         out = "[%d(%d)" % (self.index, total_tests)
 584         if self.totalsuites is not None:
 585             out += "/%d" % self.totalsuites
 586         if self.start_time is not None:
 587             out += " at " + self._format_time(self.last_time - self.start_time)
 588         if self.suitesfailed:
 589             out += ", %d errors" % (len(self.suitesfailed),)
 590         out += "] %s" % name
 591         if self.immediate:
 592             sys.stdout.write(out + "\n")
 593         else:
 594             sys.stdout.write(out + ": ")
 595
 596     def output_msg(self, output):
 597         if self.verbose:
 598             sys.stdout.write(output)
 599         elif self.name is not None:
 600             self.test_output[self.name] += output
 601         else:
 602             sys.stdout.write(output)
 603
 604     def control_msg(self, output):
 605         pass
 606
 607     def end_testsuite(self, name, result, reason):
 608         out = ""
 609         unexpected = False
 610
 611         if name not in self.test_output:
 612             print("no output for name[%s]" % name)
 613
 614         if result in ("success", "xfail"):
 615             self.suites_ok += 1
 616         else:
 617             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 618             if reason is not None:
 619                 self.output_msg("REASON: %s\n" % (reason,))
 620             self.suitesfailed.append(name)
 621             if self.immediate and not self.verbose and name in self.test_output:
 622                 out += self.test_output[name]
 623             unexpected = True
 624
 625         if not self.immediate:
 626             if not unexpected:
 627                 out += " ok\n"
 628             else:
 629                 out += " " + result.upper() + "\n"
 630
 631         sys.stdout.write(out)
 632
 633     def startTest(self, test):
 634         pass
 635
 636     def addSuccess(self, test):
 637         self.end_test(test.id(), "success", False)
 638
 639     def addError(self, test, err=None):
 640         self.end_test(test.id(), "error", True, err)
 641
 642     def addFailure(self, test, err=None):
 643         self.end_test(test.id(), "failure", True, err)
 644
 645     def addSkip(self, test, reason=None):
 646         self.end_test(test.id(), "skip", False, reason)
 647
 648     def addExpectedFailure(self, test, err=None):
 649         self.end_test(test.id(), "xfail", False, err)
 650
 651     def addUnexpectedSuccess(self, test):
 652         self.end_test(test.id(), "uxsuccess", True)
 653
 654     def end_test(self, testname, result, unexpected, err=None):
 655         if not unexpected:
 656             self.test_output[self.name] = ""
 657             if not self.immediate:
 658                 sys.stdout.write({
 659                     'failure': 'f',
 660                     'xfail': 'X',
 661                     'skip': 's',
 662                     'success': '.'}.get(result, "?(%s)" % result))
 663             return
 664
 665         if self.name not in self.test_output:
 666             self.test_output[self.name] = ""
 667
 668         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 669         if err is not None:
 670             self.test_output[self.name] += "REASON: %s\n" % str(err[1]).strip()
 671
 672         if self.immediate and not self.verbose:
 673             sys.stdout.write(self.test_output[self.name])
 674             self.test_output[self.name] = ""
 675
 676         if not self.immediate:
 677             sys.stdout.write({
 678                 'error': 'E',
 679                'failure': 'F',
 680                'uxsuccess': 'U',
 681                'success': 'S'}.get(result, "?"))
 682
 683     def write_summary(self, path):
 684         f = open(path, 'w+')
 685
 686         if self.suitesfailed:
 687             f.write("= Failed tests =\n")
 688
 689             for suite in self.suitesfailed:
 690                 f.write("== %s ==\n" % suite)
 691                 if suite in self.test_output:
 692                     f.write(self.test_output[suite] + "\n\n")
 693
 694             f.write("\n")
 695
 696         if not self.immediate and not self.verbose:
 697             for suite in self.suitesfailed:
 698                 print("=" * 78)
 699                 print("FAIL: %s" % suite)
 700                 if suite in self.test_output:
 701                     print(self.test_output[suite])
 702                 print("")
 703
 704         f.write("= Skipped tests =\n")
 705         for reason in self.skips.keys():
 706             f.write(reason + "\n")
 707             for name in self.skips[reason]:
 708                 f.write("\t%s\n" % name)
 709             f.write("\n")
 710         f.close()
 711
 712         if (not self.suitesfailed and
 713             not self.statistics['TESTS_UNEXPECTED_FAIL'] and
 714             not self.statistics['TESTS_UNEXPECTED_OK'] and
 715             not self.statistics['TESTS_ERROR']):
 716             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 717                   self.statistics['TESTS_EXPECTED_FAIL'])
 718             print("\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok))
 719         else:
 720             print("\nFAILED (%d failures, %d errors and %d unexpected successes in %d testsuites)" % (
 721                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 722                 self.statistics['TESTS_ERROR'],
 723                 self.statistics['TESTS_UNEXPECTED_OK'],
 724                 len(self.suitesfailed)))
 725
 726     def skip_testsuite(self, name, reason="UNKNOWN"):
 727         self.skips.setdefault(reason, []).append(name)
 728         if self.totalsuites:
 729             self.totalsuites -= 1