svntest/testcase.py

   1 #
   2 #  testcase.py:  Control of test case execution.
   3 #
   4 #  Subversion is a tool for revision control.
   5 #  See http://subversion.tigris.org for more information.
   6 #
   7 # ====================================================================
   8 #    Licensed to the Apache Software Foundation (ASF) under one
   9 #    or more contributor license agreements.  See the NOTICE file
  10 #    distributed with this work for additional information
  11 #    regarding copyright ownership.  The ASF licenses this file
  12 #    to you under the Apache License, Version 2.0 (the
  13 #    "License"); you may not use this file except in compliance
  14 #    with the License.  You may obtain a copy of the License at
  15 #
  16 #      http://www.apache.org/licenses/LICENSE-2.0
  17 #
  18 #    Unless required by applicable law or agreed to in writing,
  19 #    software distributed under the License is distributed on an
  20 #    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  21 #    KIND, either express or implied.  See the License for the
  22 #    specific language governing permissions and limitations
  23 #    under the License.
  24 ######################################################################
  25
  26 import os, types, sys
  27
  28 import svntest
  29
  30 # if somebody does a "from testcase import *", they only get these names
  31 __all__ = ['_XFail', '_Wimp', '_Skip', '_SkipUnless']
  32
  33 RESULT_OK = 'ok'
  34 RESULT_FAIL = 'fail'
  35 RESULT_SKIP = 'skip'
  36
  37
  38 class TextColors:
  39   '''Some ANSI terminal constants for output color'''
  40   ENDC = '\033[0;m'
  41   FAILURE = '\033[1;31m'
  42   SUCCESS = '\033[1;32m'
  43
  44   @classmethod
  45   def disable(cls):
  46     cls.ENDC = ''
  47     cls.FAILURE = ''
  48     cls.SUCCESS = ''
  49
  50   @classmethod
  51   def success(cls, str):
  52     return lambda: cls.SUCCESS + str + cls.ENDC
  53
  54   @classmethod
  55   def failure(cls, str):
  56     return lambda: cls.FAILURE + str + cls.ENDC
  57
  58
  59 if not sys.stdout.isatty() or sys.platform == 'win32':
  60   TextColors.disable()
  61
  62
  63 class TestCase:
  64   """A thing that can be tested.  This is an abstract class with
  65   several methods that need to be overridden."""
  66
  67   _result_map = {
  68     RESULT_OK:   (0, TextColors.success('PASS: '), True),
  69     RESULT_FAIL: (1, TextColors.failure('FAIL: '), False),
  70     RESULT_SKIP: (2, TextColors.success('SKIP: '), True),
  71     }
  72
  73   def __init__(self, delegate=None, cond_func=lambda: True, doc=None, wip=None,
  74                issues=None):
  75     """Create a test case instance based on DELEGATE.
  76
  77     COND_FUNC is a callable that is evaluated at test run time and should
  78     return a boolean value that determines how a pass or failure is
  79     interpreted: see the specialized kinds of test case such as XFail and
  80     Skip for details.  The evaluation of COND_FUNC is deferred so that it
  81     can base its decision on useful bits of information that are not
  82     available at __init__ time (like the fact that we're running over a
  83     particular RA layer).
  84
  85     DOC is ...
  86
  87     WIP is a string describing the reason for the work-in-progress
  88     """
  89     assert hasattr(cond_func, '__call__')
  90
  91     self._delegate = delegate
  92     self._cond_func = cond_func
  93     self.description = doc or delegate.description
  94     self.inprogress = wip
  95     self.issues = issues
  96
  97   def get_function_name(self):
  98     """Return the name of the python function implementing the test."""
  99     return self._delegate.get_function_name()
 100
 101   def get_sandbox_name(self):
 102     """Return the name that should be used for the sandbox.
 103
 104     If a sandbox should not be constructed, this method returns None.
 105     """
 106     return self._delegate.get_sandbox_name()
 107
 108   def set_issues(self, issues):
 109     """Set the issues associated with this test."""
 110     self.issues = issues
 111
 112   def run(self, sandbox):
 113     """Run the test within the given sandbox."""
 114     return self._delegate.run(sandbox)
 115
 116   def list_mode(self):
 117     return ''
 118
 119   def results(self, result):
 120     # if our condition applied, then use our result map. otherwise, delegate.
 121     if self._cond_func():
 122       val = list(self._result_map[result])
 123       val[1] = val[1]()
 124       return val
 125     return self._delegate.results(result)
 126
 127
 128 class FunctionTestCase(TestCase):
 129   """A TestCase based on a naked Python function object.
 130
 131   FUNC should be a function that returns None on success and throws an
 132   svntest.Failure exception on failure.  It should have a brief
 133   docstring describing what it does (and fulfilling certain conditions).
 134   FUNC must take one argument, an Sandbox instance.  (The sandbox name
 135   is derived from the file name in which FUNC was defined)
 136   """
 137
 138   def __init__(self, func, issues=None):
 139     # it better be a function that accepts an sbox parameter and has a
 140     # docstring on it.
 141     assert isinstance(func, types.FunctionType)
 142
 143     name = func.func_name
 144
 145     assert func.func_code.co_argcount == 1, \
 146         '%s must take an sbox argument' % name
 147
 148     doc = func.__doc__.strip()
 149     assert doc, '%s must have a docstring' % name
 150
 151     # enforce stylistic guidelines for the function docstrings:
 152     # - no longer than 50 characters
 153     # - should not end in a period
 154     # - should not be capitalized
 155     assert len(doc) <= 50, \
 156         "%s's docstring must be 50 characters or less" % name
 157     assert doc[-1] != '.', \
 158         "%s's docstring should not end in a period" % name
 159     assert doc[0].lower() == doc[0], \
 160         "%s's docstring should not be capitalized" % name
 161
 162     TestCase.__init__(self, doc=doc, issues=issues)
 163     self.func = func
 164
 165   def get_function_name(self):
 166     return self.func.func_name
 167
 168   def get_sandbox_name(self):
 169     """Base the sandbox's name on the name of the file in which the
 170     function was defined."""
 171
 172     filename = self.func.func_code.co_filename
 173     return os.path.splitext(os.path.basename(filename))[0]
 174
 175   def run(self, sandbox):
 176     return self.func(sandbox)
 177
 178
 179 class _XFail(TestCase):
 180   """A test that is expected to fail, if its condition is true."""
 181
 182   _result_map = {
 183     RESULT_OK:   (1, TextColors.failure('XPASS:'), False),
 184     RESULT_FAIL: (0, TextColors.success('XFAIL:'), True),
 185     RESULT_SKIP: (2, TextColors.success('SKIP: '), True),
 186     }
 187
 188   def __init__(self, test_case, cond_func=lambda: True, wip=None,
 189                issues=None):
 190     """Create an XFail instance based on TEST_CASE.  COND_FUNC is a
 191     callable that is evaluated at test run time and should return a
 192     boolean value.  If COND_FUNC returns true, then TEST_CASE is
 193     expected to fail (and a pass is considered an error); otherwise,
 194     TEST_CASE is run normally.  The evaluation of COND_FUNC is
 195     deferred so that it can base its decision on useful bits of
 196     information that are not available at __init__ time (like the fact
 197     that we're running over a particular RA layer).
 198
 199     WIP is ...
 200
 201     ISSUES is an issue number (or a list of issue numbers) tracking this."""
 202
 203     TestCase.__init__(self, create_test_case(test_case), cond_func, wip=wip,
 204                       issues=issues)
 205
 206   def list_mode(self):
 207     # basically, the only possible delegate is a Skip test. favor that mode.
 208     return self._delegate.list_mode() or 'XFAIL'
 209
 210
 211 class _Wimp(_XFail):
 212   """Like XFail, but indicates a work-in-progress: an unexpected pass
 213   is not considered a test failure."""
 214
 215   _result_map = {
 216     RESULT_OK:   (0, TextColors.success('XPASS:'), True),
 217     RESULT_FAIL: (0, TextColors.success('XFAIL:'), True),
 218     RESULT_SKIP: (2, TextColors.success('SKIP: '), True),
 219     }
 220
 221   def __init__(self, wip, test_case, cond_func=lambda: True):
 222     _XFail.__init__(self, test_case, cond_func, wip)
 223
 224
 225 class _Skip(TestCase):
 226   """A test that will be skipped if its conditional is true."""
 227
 228   def __init__(self, test_case, cond_func=lambda: True, issues=None):
 229     """Create a Skip instance based on TEST_CASE.  COND_FUNC is a
 230     callable that is evaluated at test run time and should return a
 231     boolean value.  If COND_FUNC returns true, then TEST_CASE is
 232     skipped; otherwise, TEST_CASE is run normally.
 233     The evaluation of COND_FUNC is deferred so that it can base its
 234     decision on useful bits of information that are not available at
 235     __init__ time (like the fact that we're running over a
 236     particular RA layer)."""
 237
 238     TestCase.__init__(self, create_test_case(test_case), cond_func,
 239                       issues=issues)
 240
 241   def list_mode(self):
 242     if self._cond_func():
 243       return 'SKIP'
 244     return self._delegate.list_mode()
 245
 246   def get_sandbox_name(self):
 247     if self._cond_func():
 248       return None
 249     return self._delegate.get_sandbox_name()
 250
 251   def run(self, sandbox):
 252     if self._cond_func():
 253       raise svntest.Skip
 254     return self._delegate.run(sandbox)
 255
 256
 257 class _SkipUnless(_Skip):
 258   """A test that will be skipped if its conditional is false."""
 259
 260   def __init__(self, test_case, cond_func):
 261     _Skip.__init__(self, test_case, lambda c=cond_func: not c())
 262
 263
 264 def create_test_case(func, issues=None):
 265   if isinstance(func, TestCase):
 266     return func
 267   else:
 268     return FunctionTestCase(func, issues=issues)
 269
 270
 271 # Various decorators to make declaring tests as such simpler
 272 def XFail_deco(cond_func = lambda: True):
 273   def _second(func):
 274     if isinstance(func, TestCase):
 275       return _XFail(func, cond_func, issues=func.issues)
 276     else:
 277       return _XFail(func, cond_func)
 278
 279   return _second
 280
 281
 282 def Wimp_deco(wip, cond_func = lambda: True):
 283   def _second(func):
 284     if isinstance(func, TestCase):
 285       return _Wimp(wip, func, cond_func, issues=func.issues)
 286     else:
 287       return _Wimp(wip, func, cond_func)
 288
 289   return _second
 290
 291
 292 def Skip_deco(cond_func = lambda: True):
 293   def _second(func):
 294     if isinstance(func, TestCase):
 295       return _Skip(func, cond_func, issues=func.issues)
 296     else:
 297       return _Skip(func, cond_func)
 298
 299   return _second
 300
 301
 302 def SkipUnless_deco(cond_func):
 303   def _second(func):
 304     if isinstance(func, TestCase):
 305       return _Skip(func, lambda c=cond_func: not c(), issues=func.issues)
 306     else:
 307       return _Skip(func, lambda c=cond_func: not c())
 308
 309   return _second
 310
 311
 312 def Issues_deco(*issues):
 313   def _second(func):
 314     if isinstance(func, TestCase):
 315       # if the wrapped thing is already a test case, just set the issues
 316       func.set_issues(issues)
 317       return func
 318
 319     else:
 320       # we need to wrap the function
 321       return create_test_case(func, issues=issues)
 322
 323   return _second
 324
 325 # Create a singular alias, for linguistic correctness
 326 Issue_deco = Issues_deco