2 # Copyright (C) 2011-2024 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
18 # - unplanned tests are properly reported as errors
29 run_make
-O -e FAIL check
30 count_test_results total
=3 pass
=1 fail
=0 xpass
=0 xfail
=0 skip
=0 error
=2
31 grep '^ERROR: all\.test - too many tests run (expected 1, got 2)$' stdout
32 grep '^ERROR: all\.test 2 # UNPLANNED$' stdout
40 run_make
-O -e FAIL check
41 count_test_results total
=4 pass
=2 fail
=0 xpass
=0 xfail
=0 skip
=0 error
=2
42 grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
43 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
45 # Interaction with XFAIL_TESTS.
54 run_make
-O -e FAIL XFAIL_TESTS
=all.
test check
55 count_test_results total
=6 pass
=0 fail
=0 xpass
=0 xfail
=1 skip
=1 error
=4
56 grep '^ERROR: all\.test - too many tests run (expected 2, got 5)$' stdout
57 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
58 grep '^ERROR: all\.test 4 # UNPLANNED$' stdout
59 grep '^ERROR: all\.test 5 # UNPLANNED$' stdout
78 ok 13 - bar bar # TODO
83 not ok 17 - bar bar # TODO
88 ok 21 - bar bar # SKIP
120 run_make
-O -e FAIL check
121 count_test_results total
=22 pass
=1 fail
=0 xpass
=0 xfail
=0 skip
=0 error
=21
123 echo 'PASS: all.test 1' > exp
124 sed -e '/^$/d' -e 's/.*/ERROR: all.test & # UNPLANNED/' t
>> exp
125 echo 'ERROR: all.test - too many tests run (expected 1, got 21)' >> exp
127 $FGREP ': all.test' stdout
> got
133 # Note that, if the TAP input has a trailing plan, it is not possible
134 # to flag unplanned tests as such, since we do not know they're unplanned
135 # until the plan is reached; still, we should give at least an error
136 # message about the unmatched number of tests once we've got the plan.
138 for x
in 'ok' 'ok 3' 'not ok' 'not ok # TODO' 'ok # TODO' 'ok # SKIP'; do
139 unindent
> all.
test <<END
145 run_make
-O -e FAIL check
146 test $
($FGREP -c ': all.test' stdout
) -eq 4
147 $EGREP '^PASS: all\.test 1($| )' stdout
148 $EGREP '^SKIP: all\.test 2($| )' stdout
149 $EGREP ': all\.test 3($| )' stdout
150 grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout