1 # Python module for parsing and generating the Subunit protocol
3 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 __all__
= ['parse_results']
23 import subunit
.iso8601
26 VALID_RESULTS
= ['success', 'successful', 'failure', 'fail', 'skip', 'knownfail', 'error', 'xfail', 'skip-testsuite', 'testsuite-failure', 'testsuite-xfail', 'testsuite-success', 'testsuite-error']
28 class TestsuiteEnabledTestResult(testtools
.testresult
.TestResult
):
30 def start_testsuite(self
, name
):
31 raise NotImplementedError(self
.start_testsuite
)
34 def parse_results(msg_ops
, statistics
, fh
):
42 parts
= l
.split(None, 1)
43 if not len(parts
) == 2 or not l
.startswith(parts
[0]):
46 command
= parts
[0].rstrip(":")
48 if command
in ("test", "testing"):
49 msg_ops
.control_msg(l
)
51 test
= subunit
.RemotedTestCase(name
)
52 if name
in open_tests
:
53 msg_ops
.addError(open_tests
.pop(name
), subunit
.RemoteError(u
"Test already running"))
54 msg_ops
.startTest(test
)
55 open_tests
[name
] = test
56 elif command
== "time":
57 msg_ops
.control_msg(l
)
59 dt
= subunit
.iso8601
.parse_date(arg
.rstrip("\n"))
61 print "Unable to parse time line: %s" % arg
.rstrip("\n")
64 elif command
in VALID_RESULTS
:
65 msg_ops
.control_msg(l
)
67 grp
= re
.match("(.*?)( \[)?([ \t]*)( multipart)?\n", arg
)
68 (testname
, hasreason
) = (grp
.group(1), grp
.group(2))
71 # reason may be specified in next lines
77 msg_ops
.control_msg(l
)
84 remote_error
= subunit
.RemoteError(reason
.decode("utf-8"))
87 statistics
['TESTS_ERROR']+=1
88 msg_ops
.addError(subunit
.RemotedTestCase(testname
), subunit
.RemoteError(u
"reason (%s) interrupted" % result
))
92 remote_error
= subunit
.RemoteError(u
"No reason specified")
93 if result
in ("success", "successful"):
95 test
= open_tests
.pop(testname
)
97 statistics
['TESTS_ERROR']+=1
98 msg_ops
.addError(subunit
.RemotedTestCase(testname
), subunit
.RemoteError(u
"Test was never started"))
100 statistics
['TESTS_EXPECTED_OK']+=1
101 msg_ops
.addSuccess(test
)
102 elif result
in ("xfail", "knownfail"):
104 test
= open_tests
.pop(testname
)
106 statistics
['TESTS_ERROR']+=1
107 msg_ops
.addError(subunit
.RemotedTestCase(testname
), subunit
.RemoteError(u
"Test was never started"))
109 statistics
['TESTS_EXPECTED_FAIL']+=1
110 msg_ops
.addExpectedFailure(test
, remote_error
)
112 elif result
in ("failure", "fail"):
114 test
= open_tests
.pop(testname
)
116 statistics
['TESTS_ERROR']+=1
117 msg_ops
.addError(subunit
.RemotedTestCase(testname
), subunit
.RemoteError(u
"Test was never started"))
119 statistics
['TESTS_UNEXPECTED_FAIL']+=1
120 msg_ops
.addFailure(test
, remote_error
)
121 elif result
== "skip":
122 statistics
['TESTS_SKIP']+=1
123 # Allow tests to be skipped without prior announcement of test
125 test
= open_tests
.pop(testname
)
127 test
= subunit
.RemotedTestCase(testname
)
128 msg_ops
.addSkip(test
, reason
)
129 elif result
== "error":
130 statistics
['TESTS_ERROR']+=1
132 test
= open_tests
.pop(testname
)
134 test
= subunit
.RemotedTestCase(testname
)
135 msg_ops
.addError(test
, remote_error
)
136 elif result
== "skip-testsuite":
137 msg_ops
.skip_testsuite(testname
)
138 elif result
== "testsuite-success":
139 msg_ops
.end_testsuite(testname
, "success", reason
)
140 elif result
== "testsuite-failure":
141 msg_ops
.end_testsuite(testname
, "failure", reason
)
142 elif result
== "testsuite-xfail":
143 msg_ops
.end_testsuite(testname
, "xfail", reason
)
144 elif result
== "testsuite-error":
145 msg_ops
.end_testsuite(testname
, "error", reason
)
147 raise AssertionError("Recognized but unhandled result %r" %
149 elif command
== "testsuite":
150 msg_ops
.start_testsuite(arg
.strip())
151 elif command
== "progress":
154 msg_ops
.progress(None, subunit
.PROGRESS_POP
)
156 msg_ops
.progress(None, subunit
.PROGRESS_PUSH
)
158 msg_ops
.progress(int(arg
), subunit
.PROGRESS_CUR
)
160 msg_ops
.progress(int(arg
), subunit
.PROGRESS_SET
)
162 msg_ops
.output_msg(l
)
165 test
= subunit
.RemotedTestCase(open_tests
.popitem()[1])
166 msg_ops
.addError(test
, subunit
.RemoteError(u
"was started but never finished!"))
167 statistics
['TESTS_ERROR']+=1
169 if statistics
['TESTS_ERROR'] > 0:
171 if statistics
['TESTS_UNEXPECTED_FAIL'] > 0:
176 class SubunitOps(subunit
.TestProtocolClient
,TestsuiteEnabledTestResult
):
178 # The following are Samba extensions:
179 def start_testsuite(self
, name
):
180 self
._stream
.write("testsuite: %s\n" % name
)
182 def skip_testsuite(self
, name
, reason
=None):
184 self
._stream
.write("skip-testsuite: %s [\n%s\n]\n" % (name
, reason
))
186 self
._stream
.write("skip-testsuite: %s\n" % name
)
188 def end_testsuite(self
, name
, result
, reason
=None):
190 self
._stream
.write("testsuite-%s: %s [\n%s\n]\n" % (result
, name
, reason
))
192 self
._stream
.write("testsuite-%s: %s\n" % (result
, name
))
194 def output_msg(self
, msg
):
195 self
._stream
.write(msg
)
198 def read_test_regexes(name
):
204 if l
== "" or l
[0] == "#":
207 (regex
, reason
) = l
.split("#", 1)
208 ret
[regex
.strip()] = reason
.strip()
216 def find_in_list(regexes
, fullname
):
217 for regex
, reason
in regexes
.iteritems():
218 if re
.match(regex
, fullname
):
225 class FilterOps(testtools
.testresult
.TestResult
):
227 def control_msg(self
, msg
):
228 pass # We regenerate control messages, so ignore this
230 def time(self
, time
):
233 def progress(self
, delta
, whence
):
234 self
._ops
.progress(delta
, whence
)
236 def output_msg(self
, msg
):
237 if self
.output
is None:
238 sys
.stdout
.write(msg
)
242 def startTest(self
, test
):
243 test
= self
._add
_prefix
(test
)
244 if self
.strip_ok_output
:
247 self
._ops
.startTest(test
)
249 def _add_prefix(self
, test
):
250 if self
.prefix
is not None:
251 return subunit
.RemotedTestCase(self
.prefix
+ test
.id())
255 def addError(self
, test
, details
=None):
256 test
= self
._add
_prefix
(test
)
259 self
._ops
.addError(test
, details
)
262 def addSkip(self
, test
, details
=None):
263 test
= self
._add
_prefix
(test
)
264 self
._ops
.addSkip(test
, details
)
267 def addExpectedFailure(self
, test
, details
=None):
268 test
= self
._add
_prefix
(test
)
269 self
._ops
.addExpectedFailure(test
, details
)
272 def addFailure(self
, test
, details
=None):
273 test
= self
._add
_prefix
(test
)
274 xfail_reason
= find_in_list(self
.expected_failures
, test
.id())
275 if xfail_reason
is not None:
278 if details
is not None:
279 details
= subunit
.RemoteError(details
[1].message
+ xfail_reason
.decode("utf-8"))
281 details
= subunit
.RemoteError(xfail_reason
.decode("utf-8"))
282 self
._ops
.addExpectedFailure(test
, details
)
286 self
._ops
.addFailure(test
, details
)
288 self
._ops
.output_msg(self
.output
)
291 def addSuccess(self
, test
, details
=None):
292 test
= self
._add
_prefix
(test
)
293 self
._ops
.addSuccess(test
, details
)
296 def skip_testsuite(self
, name
, reason
=None):
297 self
._ops
.skip_testsuite(name
, reason
)
299 def start_testsuite(self
, name
):
300 self
._ops
.start_testsuite(name
)
305 def end_testsuite(self
, name
, result
, reason
=None):
308 if self
.xfail_added
> 0:
310 if self
.fail_added
> 0 or self
.error_added
> 0:
313 if xfail
and result
in ("fail", "failure"):
316 if self
.fail_added
> 0 and result
!= "failure":
319 reason
= "Subunit/Filter Reason"
320 reason
+= "\n failures[%d]" % self
.fail_added
322 if self
.error_added
> 0 and result
!= "error":
325 reason
= "Subunit/Filter Reason"
326 reason
+= "\n errors[%d]" % self
.error_added
328 self
._ops
.end_testsuite(name
, result
, reason
)
330 def __init__(self
, out
, prefix
, expected_failures
, strip_ok_output
):
334 self
.expected_failures
= expected_failures
335 self
.strip_ok_output
= strip_ok_output