2 # Copyright (C) 2011-2015 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Auxiliary script for tests on TAP support: checking testsuite summary.
21 br
='============================================================================'
25 *) fatal_
"invalid \$use_colors value '$use_colors'"
30 cat > configure.ac
<<END
31 AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
33 AC_CONFIG_FILES([Makefile])
37 cat > Makefile.am
<< 'END'
38 TEST_LOG_DRIVER
= $
(srcdir
)/tap-driver
39 TEST_LOG_COMPILER
= cat
43 # The following shell variables are influential for this function:
49 1,--pass) expect_failure
=no
;;
50 1,--fail) expect_failure
=yes;;
51 *) fatal_
"invalid usage of 'do_check'";;
56 if test $use_colors = yes; then
57 # Forced colorization should take place also with non-ANSI terminals;
58 # hence the "TERM=dumb" definition.
59 make_args
='TERM=dumb AM_COLOR_TESTS=always'
63 run_make
-O -e IGNORE
$make_args check
64 if test $expect_failure = yes; then
65 test $am_make_rc -gt 0 ||
exit 1
67 test $am_make_rc -eq 0 ||
exit 1
69 $PERL "$am_testaux_srcdir"/extract-testsuite-summary.pl stdout
>summary.got \
70 || fatal_
"cannot extract testsuite summary"
73 if test $use_colors = yes; then
74 # Use cmp, not diff, because the files might contain binary data.
79 $compare summary.exp summary.got ||
exit 1
82 if test $use_colors = yes; then
91 red
= grn
= lgn
= blu
= mgn
= brg
= std
=
96 ${grn}Testsuite summary for GNU AutoTAP 5.12${std}
99 success_footer
=${grn}${br}${std}
103 ${red}Testsuite summary for GNU AutoTAP 5.12${std}
108 ${red}See ./test-suite.log${std}
109 ${red}Please report to bug-automake@gnu.org${std}
114 $AUTOMAKE --add-missing
119 { echo 1.
.1 && echo ok
; } > all.
test
120 do_check
--pass <<END
122 ${brg}# TOTAL: 1${std}
123 ${grn}# PASS: 1${std}
133 { echo 1.
.1 && echo 'ok # SKIP'; } > all.
test
134 do_check
--pass <<END
136 ${brg}# TOTAL: 1${std}
138 ${blu}# SKIP: 1${std}
147 { echo 1.
.1 && echo 'not ok # TODO'; } > all.
test
148 do_check
--pass <<END
150 ${brg}# TOTAL: 1${std}
153 ${lgn}# XFAIL: 1${std}
161 { echo 1.
.1 && echo not ok
; } > all.
test
162 do_check
--fail <<END
164 ${brg}# TOTAL: 1${std}
168 ${red}# FAIL: 1${std}
175 { echo 1.
.1 && echo 'ok # TODO'; } > all.
test
176 do_check
--fail <<END
178 ${brg}# TOTAL: 1${std}
183 ${red}# XPASS: 1${std}
189 { echo 1.
.1 && echo 'Bail out!'; } > all.
test
190 do_check
--fail <<END
192 ${brg}# TOTAL: 1${std}
198 ${mgn}# ERROR: 1${std}
202 # 3 non-failing results.
209 do_check
--pass <<END
211 ${brg}# TOTAL: 3${std}
212 ${grn}# PASS: 1${std}
213 ${blu}# SKIP: 1${std}
214 ${lgn}# XFAIL: 1${std}
221 # 1 pass, 1 skip, 1 fail.
228 do_check
--fail <<END
230 ${brg}# TOTAL: 3${std}
231 ${grn}# PASS: 1${std}
232 ${blu}# SKIP: 1${std}
234 ${red}# FAIL: 1${std}
240 # 1 pass, 1 xfail, 1 xpass.
247 do_check
--fail <<END
249 ${brg}# TOTAL: 3${std}
250 ${grn}# PASS: 1${std}
252 ${lgn}# XFAIL: 1${std}
254 ${red}# XPASS: 1${std}
259 # 1 skip, 1 xfail, 1 error.
266 do_check
--fail <<END
268 ${brg}# TOTAL: 3${std}
270 ${blu}# SKIP: 1${std}
271 ${lgn}# XFAIL: 1${std}
274 ${mgn}# ERROR: 1${std}
288 do_check
--fail <<END
290 ${brg}# TOTAL: 6${std}
291 ${grn}# PASS: 1${std}
292 ${blu}# SKIP: 1${std}
293 ${lgn}# XFAIL: 1${std}
294 ${red}# FAIL: 1${std}
295 ${red}# XPASS: 1${std}
296 ${mgn}# ERROR: 1${std}
300 # Prepare some common data for later.
301 for i
in 0 1 2 3 4 5 6 7 8 9; do
302 for j
in 0 1 2 3 4 5 6 7 8 9; do
309 # Lots of non-failures (300 per kind).
310 (cat tap
&& cat tap
&& cat tap
) > all.
test
311 test $
(wc -l <all.
test) -eq 900 ||
exit 99 # Sanity check.
312 echo 1.
.900 >> all.
test # Test plan.
313 do_check
--pass <<END
315 ${brg}# TOTAL: 900${std}
316 ${grn}# PASS: 300${std}
317 ${blu}# SKIP: 300${std}
318 ${lgn}# XFAIL: 300${std}
325 # 1 failure and lots of non-failures means failure.
326 (cat tap
&& echo "not ok" && cat tap
) > all.
test
327 test $
(wc -l <all.
test) -eq 601 ||
exit 99 # Sanity check.
328 echo 1.
.601 >> all.
test # Test plan.
329 do_check
--fail <<END
331 ${brg}# TOTAL: 601${std}
332 ${grn}# PASS: 200${std}
333 ${blu}# SKIP: 200${std}
334 ${lgn}# XFAIL: 200${std}
335 ${red}# FAIL: 1${std}
341 # 1 error and lots of non-failures means failure.
342 (cat tap
&& sed 30q tap
&& echo 'Bail out!') > all.
test
343 test $
(wc -l <all.
test) -eq 331 ||
exit 99 # Sanity check.
344 echo 1.
.331 >> all.
test # Test plan.
345 do_check
--fail <<END
347 ${brg}# TOTAL: 331${std}
348 ${grn}# PASS: 110${std}
349 ${blu}# SKIP: 110${std}
350 ${lgn}# XFAIL: 110${std}
353 ${mgn}# ERROR: 1${std}