lib/subunit/filters/subunit-filter

   1 #!/usr/bin/env python
   2 #  subunit: extensions to python unittest to get test results from subprocesses.
   3 #  Copyright (C) 2008  Robert Collins <robertc@robertcollins.net>
   4 #            (C) 2009  Martin Pool
   5 #
   6 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
   7 #  license at the users choice. A copy of both licenses are available in the
   8 #  project source as Apache-2.0 and BSD. You may not use this file except in
   9 #  compliance with one of these two licences.
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
  13 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
  14 #  license you chose for the specific language governing permissions and
  15 #  limitations under that license.
  16 #
  17
  18 """Filter a subunit stream to include/exclude tests.
  19
  20 The default is to strip successful tests.
  21
  22 Tests can be filtered by Python regular expressions with --with and --without,
  23 which match both the test name and the error text (if any).  The result
  24 contains tests which match any of the --with expressions and none of the
  25 --without expressions.  For case-insensitive matching prepend '(?i)'.
  26 Remember to quote shell metacharacters.
  27 """
  28
  29 from optparse import OptionParser
  30 import sys
  31 import re
  32
  33 from subunit import (
  34     DiscardStream,
  35     ProtocolTestCase,
  36     TestProtocolClient,
  37     read_test_list,
  38     )
  39 from subunit.test_results import TestResultFilter
  40
  41 parser = OptionParser(description=__doc__)
  42 parser.add_option("--error", action="store_false",
  43     help="include errors", default=False, dest="error")
  44 parser.add_option("-e", "--no-error", action="store_true",
  45     help="exclude errors", dest="error")
  46 parser.add_option("--failure", action="store_false",
  47     help="include failures", default=False, dest="failure")
  48 parser.add_option("-f", "--no-failure", action="store_true",
  49     help="exclude failures", dest="failure")
  50 parser.add_option("--passthrough", action="store_false",
  51     help="Show all non subunit input.", default=False, dest="no_passthrough")
  52 parser.add_option("--no-passthrough", action="store_true",
  53     help="Hide all non subunit input.", default=False, dest="no_passthrough")
  54 parser.add_option("-s", "--success", action="store_false",
  55     help="include successes", dest="success")
  56 parser.add_option("--no-success", action="store_true",
  57     help="exclude successes", default=True, dest="success")
  58 parser.add_option("--no-skip", action="store_true",
  59     help="exclude skips", dest="skip")
  60 parser.add_option("--xfail", action="store_false",
  61     help="include expected falures", default=True, dest="xfail")
  62 parser.add_option("--no-xfail", action="store_true",
  63     help="exclude expected falures", default=True, dest="xfail")
  64 parser.add_option("-m", "--with", type=str,
  65     help="regexp to include (case-sensitive by default)",
  66     action="append", dest="with_regexps")
  67 parser.add_option("--fixup-expected-failures", type=str,
  68     help="File with list of test ids that are expected to fail; on failure "
  69          "their result will be changed to xfail; on success they will be "
  70          "changed to error.", dest="fixup_expected_failures", action="append")
  71 parser.add_option("--without", type=str,
  72     help="regexp to exclude (case-sensitive by default)",
  73     action="append", dest="without_regexps")
  74
  75 def only_genuine_failures_callback(option, opt, value, parser):
  76     parser.rargs.insert(0, '--no-passthrough')
  77     parser.rargs.insert(0, '--no-xfail')
  78     parser.rargs.insert(0, '--no-skip')
  79     parser.rargs.insert(0, '--no-success')
  80
  81 parser.add_option("-F", "--only-genuine-failures", action="callback",
  82     callback=only_genuine_failures_callback,
  83     help="Only pass through failures and exceptions.")
  84
  85 (options, args) = parser.parse_args()
  86
  87 def _compile_re_from_list(l):
  88     return re.compile("|".join(l), re.MULTILINE)
  89
  90
  91 def _make_regexp_filter(with_regexps, without_regexps):
  92     """Make a callback that checks tests against regexps.
  93
  94     with_regexps and without_regexps are each either a list of regexp strings,
  95     or None.
  96     """
  97     with_re = with_regexps and _compile_re_from_list(with_regexps)
  98     without_re = without_regexps and _compile_re_from_list(without_regexps)
  99
 100     def check_regexps(test, outcome, err, details):
 101         """Check if this test and error match the regexp filters."""
 102         test_str = str(test) + outcome + str(err) + str(details)
 103         if with_re and not with_re.search(test_str):
 104             return False
 105         if without_re and without_re.search(test_str):
 106             return False
 107         return True
 108     return check_regexps
 109
 110
 111 regexp_filter = _make_regexp_filter(options.with_regexps,
 112         options.without_regexps)
 113 fixup_expected_failures = set()
 114 for path in options.fixup_expected_failures or ():
 115     fixup_expected_failures.update(read_test_list(path))
 116 result = TestProtocolClient(sys.stdout)
 117 result = TestResultFilter(result, filter_error=options.error,
 118     filter_failure=options.failure, filter_success=options.success,
 119     filter_skip=options.skip, filter_xfail=options.xfail,
 120     filter_predicate=regexp_filter,
 121     fixup_expected_failures=fixup_expected_failures)
 122 if options.no_passthrough:
 123     passthrough_stream = DiscardStream()
 124 else:
 125     passthrough_stream = None
 126 test = ProtocolTestCase(sys.stdin, passthrough=passthrough_stream)
 127 test.run(result)
 128 sys.exit(0)