mailman/bin/testall.py

   1 # Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
   2 #
   3 # This program is free software; you can redistribute it and/or
   4 # modify it under the terms of the GNU General Public License
   5 # as published by the Free Software Foundation; either version 2
   6 # of the License, or (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program; if not, write to the Free Software
  15 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
  16 # USA.
  17
  18 """Mailman unit test driver."""
  19
  20 from __future__ import with_statement
  21
  22 import os
  23 import re
  24 import grp
  25 import pwd
  26 import sys
  27 import random
  28 import shutil
  29 import optparse
  30 import tempfile
  31 import unittest
  32 import pkg_resources
  33
  34 from mailman import Defaults
  35 from mailman.configuration import config
  36 from mailman.i18n import _
  37 from mailman.initialize import initialize_1, initialize_2, initialize_3
  38 from mailman.testing.helpers import SMTPServer
  39 from mailman.version import MAILMAN_VERSION
  40
  41
  42 basedir = None
  43
  44
  45 \f
  46 def v_callback(option, opt, value, parser):
  47     if opt in ('-q', '--quiet'):
  48         delta = -1
  49     elif opt in ('-v', '--verbose'):
  50         delta = 1
  51     else:
  52         raise AssertionError('Unexpected option: %s' % opt)
  53     dest = getattr(parser.values, option.dest)
  54     setattr(parser.values, option.dest, max(0, dest + delta))
  55
  56
  57 def parseargs():
  58     parser = optparse.OptionParser(version=MAILMAN_VERSION,
  59                                    usage=_("""\
  60 %prog [options] [tests]
  61
  62 Run the Mailman unit test suite.  'tests' is one or more Python regular
  63 expressions matching only the tests you want to run.  Prefix the regular
  64 expression with '!' to specify a negative test."""))
  65     parser.set_defaults(verbosity=2)
  66     parser.add_option('-v', '--verbose',
  67                       action='callback', callback=v_callback,
  68                       dest='verbosity', help=_("""\
  69 Increase verbosity by 1, which defaults to %default.  Use -q to reduce
  70 verbosity.  -v and -q options accumulate."""))
  71     parser.add_option('-q', '--quiet',
  72                       action='callback', callback=v_callback,
  73                       dest='verbosity', help=_("""\
  74 Reduce verbosity by 1 (but not below 0)."""))
  75     parser.add_option('-e', '--stderr',
  76                       default=False, action='store_true',
  77                       help=_('Propagate log errors to stderr.'))
  78     parser.add_option('-c', '--coverage',
  79                       default=False, action='store_true',
  80                       help=_('Enable code coverage.'))
  81     parser.add_option('-r', '--randomize',
  82                       default=False, action='store_true',
  83                       help=_("""\
  84 Randomize the tests; good for finding subtle dependency errors.  Note that
  85 this isn't completely random though because the doctests are not mixed with
  86 the Python tests.  Each type of test is randomized within its group."""))
  87     options, arguments = parser.parse_args()
  88     if len(arguments) == 0:
  89         arguments = ['.']
  90     parser.options = options
  91     parser.arguments = arguments
  92     return parser
  93
  94
  95 \f
  96 def search():
  97     testnames = []
  98     # Walk the entire tree from the current base directory.  Look for modules
  99     # that start with 'test_'.  Calculate the full module path name to this
 100     # module, append 'test_suite' and add that to testnames.  This way, we run
 101     # all the suites defined in the test_suite() function inside all test
 102     # modules.
 103     for dirpath, dirnames, filenames in os.walk(basedir):
 104         for fn in filenames:
 105             if fn.startswith('test_') and fn.endswith('.py'):
 106                 # Start with full path
 107                 path = os.path.join(dirpath, fn)
 108                 # Convert the file path to a module path.  First, we must make
 109                 # the file path relative to the root directory.  Then strip
 110                 # off the trailing .py
 111                 path = path[len(basedir)+1:-3]
 112                 # Convert slashes to dots
 113                 modpath = path.replace(os.sep, '.') + '.test_suite'
 114                 testnames.append('mailman.' + modpath)
 115     return testnames
 116
 117
 118 def match(pat, name):
 119     if not pat:
 120         return True
 121     if pat.startswith('!'):
 122         # Negative test
 123         return re.search(pat[1:], name) is None
 124     else:
 125         # Positive test
 126         return re.search(pat, name) is not None
 127
 128
 129 def filter_tests(suite, patterns):
 130     if '.' in patterns:
 131         return suite
 132     new = unittest.TestSuite()
 133     for test in suite._tests:
 134         if isinstance(test, unittest.TestCase):
 135             # Get the fill test name: package.module.class.method
 136             name = test.id()
 137             for pat in patterns:
 138                 if match(pat, name):
 139                     new.addTest(test)
 140                     break
 141         else:
 142             filtered = filter_tests(test, patterns)
 143             if filtered:
 144                 new.addTest(filtered)
 145     return new
 146
 147
 148 def suite(patterns, randomize):
 149     if patterns is None:
 150         patterns = '.'
 151     loader = unittest.TestLoader()
 152     # Search for all tests that match the given patterns
 153     testnames = search()
 154     suite = loader.loadTestsFromNames(testnames)
 155     tests = filter_tests(suite, patterns)
 156     if randomize:
 157         random.shuffle(tests._tests)
 158     else:
 159         tests._tests.sort()
 160     return tests
 161
 162
 163 \f
 164 def main():
 165     global basedir
 166
 167     parser = parseargs()
 168
 169     # Set verbosity level for test_documentation.py.  XXX There should be a
 170     # better way to do this.
 171     class Bag: pass
 172     config.tests = Bag()
 173     config.tests.verbosity = parser.options.verbosity
 174     config.tests.randomize = parser.options.randomize
 175
 176     # Turn on code coverage if selected.
 177     if parser.options.coverage:
 178         try:
 179             import coverage
 180         except ImportError:
 181             parser.options.coverage = False
 182         else:
 183             coverage.start()
 184
 185     # Set up the testing configuration file both for this process, and for all
 186     # sub-processes testing will spawn (e.g. the qrunners).
 187     #
 188     # Also create a logging.cfg file with values reflecting verbosity and
 189     # stderr propagation.  Enable it only if necessary.
 190     fd, logging_cfg = tempfile.mkstemp(suffix='.cfg')
 191     os.close(fd)
 192     enable_logging_cfg = False
 193     with open(logging_cfg, 'w') as fp:
 194         print >> fp, '[*]'
 195         if parser.options.stderr:
 196             print >> fp, 'propagate = True'
 197             enable_logging_cfg = True
 198         if parser.options.verbosity > 2:
 199             print >> fp, 'level = DEBUG'
 200             enable_logging_cfg = True
 201
 202     cfg_in = pkg_resources.resource_string(
 203         'mailman.testing', 'testing.cfg.in')
 204     fd, cfg_out = tempfile.mkstemp(suffix='.cfg')
 205     os.close(fd)
 206     with open(cfg_out, 'w') as fp:
 207         fp.write(cfg_in)
 208         if enable_logging_cfg:
 209             print >> fp, 'LOG_CONFIG_FILE = "%s"' % logging_cfg
 210
 211     # Calculate a temporary VAR_DIR directory so that run-time artifacts of
 212     # the tests won't tread on the installation's data.  This also makes it
 213     # easier to clean up after the tests are done, and insures isolation of
 214     # test suite runs.
 215     var_dir = tempfile.mkdtemp()
 216     if parser.options.verbosity > 2:
 217         print 'VAR_DIR :', var_dir
 218         print 'config file:', cfg_out
 219         if enable_logging_cfg:
 220             print 'logging config file:', logging_cfg
 221
 222     user_id = os.getuid()
 223     user_name = pwd.getpwuid(user_id).pw_name
 224     group_id = os.getgid()
 225     group_name = grp.getgrgid(group_id).gr_name
 226
 227     try:
 228         with open(cfg_out, 'a') as fp:
 229             print >> fp, 'VAR_DIR = "%s"' % var_dir
 230             print >> fp, 'MAILMAN_USER = "%s"' % user_name
 231             print >> fp, 'MAILMAN_UID =', user_id
 232             print >> fp, 'MAILMAN_GROUP = "%s"' % group_name
 233             print >> fp, 'MAILMAN_GID =', group_id
 234             print >> fp, "LANGUAGES = 'en'"
 235             print >> fp, 'SMTPPORT =', SMTPServer.port
 236             # A fake MHonArc command, for testing.
 237             print >> fp, 'MHONARC_COMMAND = """/bin/echo', \
 238                   Defaults.MHONARC_COMMAND, '"""'
 239
 240         initialize_1(cfg_out, propagate_logs=parser.options.stderr)
 241         mailman_uid = pwd.getpwnam(config.MAILMAN_USER).pw_uid
 242         mailman_gid = grp.getgrnam(config.MAILMAN_GROUP).gr_gid
 243         os.chmod(cfg_out, 0660)
 244         os.chown(cfg_out, mailman_uid, mailman_gid)
 245
 246         # Create an empty SQLite database file with the proper permissions and
 247         # calculate the SQLAlchemy engine url to this database file.
 248         fd, config.dbfile = tempfile.mkstemp(dir=config.DATA_DIR, suffix='.db')
 249         os.close(fd)
 250         os.chmod(config.dbfile, 0660)
 251         os.chown(config.dbfile, mailman_uid, mailman_gid)
 252
 253         # Patch ups.
 254         test_engine_url = 'sqlite:///' + config.dbfile
 255         config.DEFAULT_DATABASE_URL = test_engine_url
 256
 257         # Write this to the config file so subprocesses share the same testing
 258         # database file.
 259         with open(cfg_out, 'a') as fp:
 260             print >> fp, 'DEFAULT_DATABASE_URL = "%s"' % test_engine_url
 261
 262         # With -vvv, turn on engine debugging.
 263         initialize_2(parser.options.verbosity > 3)
 264         initialize_3()
 265
 266         # Run the tests.  XXX I'm not sure if basedir can be converted to
 267         # pkg_resources.
 268         import mailman
 269         basedir = os.path.dirname(mailman.__file__)
 270         runner = unittest.TextTestRunner(verbosity=parser.options.verbosity)
 271         results = runner.run(suite(parser.arguments, parser.options.randomize))
 272     finally:
 273         os.remove(cfg_out)
 274         os.remove(logging_cfg)
 275         shutil.rmtree(var_dir)
 276
 277     # Turn off coverage and print a report
 278     if parser.options.coverage:
 279         coverage.stop()
 280         modules = [module for name, module in sys.modules.items()
 281                    if module
 282                    and name is not None
 283                    and name.split('.')[0] == 'mailman']
 284         coverage.report(modules)
 285     sys.exit(bool(results.failures or results.errors))
 286
 287
 288 \f
 289 if __name__ == '__main__':
 290     main()