lib/testtools/testtools/run.py

   1 # Copyright (c) 2009 testtools developers. See LICENSE for details.
   2
   3 """python -m testtools.run testspec [testspec...]
   4
   5 Run some tests with the testtools extended API.
   6
   7 For instance, to run the testtools test suite.
   8  $ python -m testtools.run testtools.tests.test_suite
   9 """
  10
  11 import os
  12 import unittest
  13 import sys
  14
  15 from testtools import TextTestResult
  16 from testtools.compat import classtypes, istext, unicode_output_stream
  17 from testtools.testsuite import iterate_tests, sorted_tests
  18
  19
  20 defaultTestLoader = unittest.defaultTestLoader
  21 defaultTestLoaderCls = unittest.TestLoader
  22
  23 if getattr(defaultTestLoader, 'discover', None) is None:
  24     try:
  25         import discover
  26         defaultTestLoader = discover.DiscoveringTestLoader()
  27         defaultTestLoaderCls = discover.DiscoveringTestLoader
  28         have_discover = True
  29     except ImportError:
  30         have_discover = False
  31 else:
  32     have_discover = True
  33
  34
  35 class TestToolsTestRunner(object):
  36     """ A thunk object to support unittest.TestProgram."""
  37
  38     def __init__(self, verbosity=None, failfast=None, buffer=None):
  39         """Create a TestToolsTestRunner.
  40
  41         :param verbosity: Ignored.
  42         :param failfast: Stop running tests at the first failure.
  43         :param buffer: Ignored.
  44         """
  45         self.failfast = failfast
  46
  47     def run(self, test):
  48         "Run the given test case or test suite."
  49         result = TextTestResult(
  50             unicode_output_stream(sys.stdout), failfast=self.failfast)
  51         result.startTestRun()
  52         try:
  53             return test.run(result)
  54         finally:
  55             result.stopTestRun()
  56
  57
  58 ####################
  59 # Taken from python 2.7 and slightly modified for compatibility with
  60 # older versions. Delete when 2.7 is the oldest supported version.
  61 # Modifications:
  62 #  - Use have_discover to raise an error if the user tries to use
  63 #    discovery on an old version and doesn't have discover installed.
  64 #  - If --catch is given check that installHandler is available, as
  65 #    it won't be on old python versions.
  66 #  - print calls have been been made single-source python3 compatibile.
  67 #  - exception handling likewise.
  68 #  - The default help has been changed to USAGE_AS_MAIN and USAGE_FROM_MODULE
  69 #    removed.
  70 #  - A tweak has been added to detect 'python -m *.run' and use a
  71 #    better progName in that case.
  72 #  - self.module is more comprehensively set to None when being invoked from
  73 #    the commandline - __name__ is used as a sentinel value.
  74 #  - --list has been added which can list tests (should be upstreamed).
  75 #  - --load-list has been added which can reduce the tests used (should be
  76 #    upstreamed).
  77 #  - The limitation of using getopt is declared to the user.
  78 #  - http://bugs.python.org/issue16709 is worked around, by sorting tests when
  79 #    discover is used.
  80
  81 FAILFAST     = "  -f, --failfast   Stop on first failure\n"
  82 CATCHBREAK   = "  -c, --catch      Catch control-C and display results\n"
  83 BUFFEROUTPUT = "  -b, --buffer     Buffer stdout and stderr during test runs\n"
  84
  85 USAGE_AS_MAIN = """\
  86 Usage: %(progName)s [options] [tests]
  87
  88 Options:
  89   -h, --help       Show this message
  90   -v, --verbose    Verbose output
  91   -q, --quiet      Minimal output
  92   -l, --list       List tests rather than executing them.
  93   --load-list      Specifies a file containing test ids, only tests matching
  94                    those ids are executed.
  95 %(failfast)s%(catchbreak)s%(buffer)s
  96 Examples:
  97   %(progName)s test_module               - run tests from test_module
  98   %(progName)s module.TestClass          - run tests from module.TestClass
  99   %(progName)s module.Class.test_method  - run specified test method
 100
 101 All options must come before [tests].  [tests] can be a list of any number of
 102 test modules, classes and test methods.
 103
 104 Alternative Usage: %(progName)s discover [options]
 105
 106 Options:
 107   -v, --verbose    Verbose output
 108 %(failfast)s%(catchbreak)s%(buffer)s  -s directory     Directory to start discovery ('.' default)
 109   -p pattern       Pattern to match test files ('test*.py' default)
 110   -t directory     Top level directory of project (default to
 111                    start directory)
 112   -l, --list       List tests rather than executing them.
 113   --load-list      Specifies a file containing test ids, only tests matching
 114                    those ids are executed.
 115
 116 For test discovery all test modules must be importable from the top
 117 level directory of the project.
 118 """
 119
 120
 121 class TestProgram(object):
 122     """A command-line program that runs a set of tests; this is primarily
 123        for making test modules conveniently executable.
 124     """
 125     USAGE = USAGE_AS_MAIN
 126
 127     # defaults for testing
 128     failfast = catchbreak = buffer = progName = None
 129
 130     def __init__(self, module=__name__, defaultTest=None, argv=None,
 131                     testRunner=None, testLoader=defaultTestLoader,
 132                     exit=True, verbosity=1, failfast=None, catchbreak=None,
 133                     buffer=None, stdout=None):
 134         if module == __name__:
 135             self.module = None
 136         elif istext(module):
 137             self.module = __import__(module)
 138             for part in module.split('.')[1:]:
 139                 self.module = getattr(self.module, part)
 140         else:
 141             self.module = module
 142         if argv is None:
 143             argv = sys.argv
 144         if stdout is None:
 145             stdout = sys.stdout
 146
 147         self.exit = exit
 148         self.failfast = failfast
 149         self.catchbreak = catchbreak
 150         self.verbosity = verbosity
 151         self.buffer = buffer
 152         self.defaultTest = defaultTest
 153         self.listtests = False
 154         self.load_list = None
 155         self.testRunner = testRunner
 156         self.testLoader = testLoader
 157         progName = argv[0]
 158         if progName.endswith('%srun.py' % os.path.sep):
 159             elements = progName.split(os.path.sep)
 160             progName = '%s.run' % elements[-2]
 161         else:
 162             progName = os.path.basename(argv[0])
 163         self.progName = progName
 164         self.parseArgs(argv)
 165         if self.load_list:
 166             # TODO: preserve existing suites (like testresources does in
 167             # OptimisingTestSuite.add, but with a standard protocol).
 168             # This is needed because the load_tests hook allows arbitrary
 169             # suites, even if that is rarely used.
 170             source = open(self.load_list, 'rb')
 171             try:
 172                 lines = source.readlines()
 173             finally:
 174                 source.close()
 175             test_ids = set(line.strip().decode('utf-8') for line in lines)
 176             filtered = unittest.TestSuite()
 177             for test in iterate_tests(self.test):
 178                 if test.id() in test_ids:
 179                     filtered.addTest(test)
 180             self.test = filtered
 181         if not self.listtests:
 182             self.runTests()
 183         else:
 184             for test in iterate_tests(self.test):
 185                 stdout.write('%s\n' % test.id())
 186
 187     def usageExit(self, msg=None):
 188         if msg:
 189             print(msg)
 190         usage = {'progName': self.progName, 'catchbreak': '', 'failfast': '',
 191                  'buffer': ''}
 192         if self.failfast != False:
 193             usage['failfast'] = FAILFAST
 194         if self.catchbreak != False:
 195             usage['catchbreak'] = CATCHBREAK
 196         if self.buffer != False:
 197             usage['buffer'] = BUFFEROUTPUT
 198         print(self.USAGE % usage)
 199         sys.exit(2)
 200
 201     def parseArgs(self, argv):
 202         if len(argv) > 1 and argv[1].lower() == 'discover':
 203             self._do_discovery(argv[2:])
 204             return
 205
 206         import getopt
 207         long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer',
 208             'list', 'load-list=']
 209         try:
 210             options, args = getopt.getopt(argv[1:], 'hHvqfcbl', long_opts)
 211             for opt, value in options:
 212                 if opt in ('-h','-H','--help'):
 213                     self.usageExit()
 214                 if opt in ('-q','--quiet'):
 215                     self.verbosity = 0
 216                 if opt in ('-v','--verbose'):
 217                     self.verbosity = 2
 218                 if opt in ('-f','--failfast'):
 219                     if self.failfast is None:
 220                         self.failfast = True
 221                     # Should this raise an exception if -f is not valid?
 222                 if opt in ('-c','--catch'):
 223                     if self.catchbreak is None:
 224                         self.catchbreak = True
 225                     # Should this raise an exception if -c is not valid?
 226                 if opt in ('-b','--buffer'):
 227                     if self.buffer is None:
 228                         self.buffer = True
 229                     # Should this raise an exception if -b is not valid?
 230                 if opt in ('-l', '--list'):
 231                     self.listtests = True
 232                 if opt == '--load-list':
 233                     self.load_list = value
 234             if len(args) == 0 and self.defaultTest is None:
 235                 # createTests will load tests from self.module
 236                 self.testNames = None
 237             elif len(args) > 0:
 238                 self.testNames = args
 239             else:
 240                 self.testNames = (self.defaultTest,)
 241             self.createTests()
 242         except getopt.error:
 243             self.usageExit(sys.exc_info()[1])
 244
 245     def createTests(self):
 246         if self.testNames is None:
 247             self.test = self.testLoader.loadTestsFromModule(self.module)
 248         else:
 249             self.test = self.testLoader.loadTestsFromNames(self.testNames,
 250                                                            self.module)
 251
 252     def _do_discovery(self, argv, Loader=defaultTestLoaderCls):
 253         # handle command line args for test discovery
 254         if not have_discover:
 255             raise AssertionError("Unable to use discovery, must use python 2.7 "
 256                     "or greater, or install the discover package.")
 257         self.progName = '%s discover' % self.progName
 258         import optparse
 259         parser = optparse.OptionParser()
 260         parser.prog = self.progName
 261         parser.add_option('-v', '--verbose', dest='verbose', default=False,
 262                           help='Verbose output', action='store_true')
 263         if self.failfast != False:
 264             parser.add_option('-f', '--failfast', dest='failfast', default=False,
 265                               help='Stop on first fail or error',
 266                               action='store_true')
 267         if self.catchbreak != False:
 268             parser.add_option('-c', '--catch', dest='catchbreak', default=False,
 269                               help='Catch ctrl-C and display results so far',
 270                               action='store_true')
 271         if self.buffer != False:
 272             parser.add_option('-b', '--buffer', dest='buffer', default=False,
 273                               help='Buffer stdout and stderr during tests',
 274                               action='store_true')
 275         parser.add_option('-s', '--start-directory', dest='start', default='.',
 276                           help="Directory to start discovery ('.' default)")
 277         parser.add_option('-p', '--pattern', dest='pattern', default='test*.py',
 278                           help="Pattern to match tests ('test*.py' default)")
 279         parser.add_option('-t', '--top-level-directory', dest='top', default=None,
 280                           help='Top level directory of project (defaults to start directory)')
 281         parser.add_option('-l', '--list', dest='listtests', default=False, action="store_true",
 282                           help='List tests rather than running them.')
 283         parser.add_option('--load-list', dest='load_list', default=None,
 284                           help='Specify a filename containing the test ids to use.')
 285
 286         options, args = parser.parse_args(argv)
 287         if len(args) > 3:
 288             self.usageExit()
 289
 290         for name, value in zip(('start', 'pattern', 'top'), args):
 291             setattr(options, name, value)
 292
 293         # only set options from the parsing here
 294         # if they weren't set explicitly in the constructor
 295         if self.failfast is None:
 296             self.failfast = options.failfast
 297         if self.catchbreak is None:
 298             self.catchbreak = options.catchbreak
 299         if self.buffer is None:
 300             self.buffer = options.buffer
 301         self.listtests = options.listtests
 302         self.load_list = options.load_list
 303
 304         if options.verbose:
 305             self.verbosity = 2
 306
 307         start_dir = options.start
 308         pattern = options.pattern
 309         top_level_dir = options.top
 310
 311         loader = Loader()
 312         # See http://bugs.python.org/issue16709
 313         # While sorting here is intrusive, its better than being random.
 314         # Rules for the sort:
 315         # - standard suites are flattened, and the resulting tests sorted by
 316         #   id.
 317         # - non-standard suites are preserved as-is, and sorted into position
 318         #   by the first test found by iterating the suite.
 319         # We do this by a DSU process: flatten and grab a key, sort, strip the
 320         # keys.
 321         loaded = loader.discover(start_dir, pattern, top_level_dir)
 322         self.test = sorted_tests(loaded)
 323
 324     def runTests(self):
 325         if (self.catchbreak
 326             and getattr(unittest, 'installHandler', None) is not None):
 327             unittest.installHandler()
 328         if self.testRunner is None:
 329             self.testRunner = TestToolsTestRunner
 330         if isinstance(self.testRunner, classtypes()):
 331             try:
 332                 testRunner = self.testRunner(verbosity=self.verbosity,
 333                                              failfast=self.failfast,
 334                                              buffer=self.buffer)
 335             except TypeError:
 336                 # didn't accept the verbosity, buffer or failfast arguments
 337                 testRunner = self.testRunner()
 338         else:
 339             # it is assumed to be a TestRunner instance
 340             testRunner = self.testRunner
 341         self.result = testRunner.run(self.test)
 342         if self.exit:
 343             sys.exit(not self.result.wasSuccessful())
 344 ################
 345
 346 def main(argv, stdout):
 347     program = TestProgram(argv=argv, testRunner=TestToolsTestRunner,
 348         stdout=stdout)
 349
 350 if __name__ == '__main__':
 351     main(sys.argv, sys.stdout)