2 # Copyright (C) 2009-2012 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Basic checks on parallel-tests support:
19 # - log files, and what goes in 'test-suite.log'
21 # - dependencies between tests
22 # - TESTS redefinition at runtime
23 # - TEST_LOGS redefinition at runtime
24 # - RECHECK_LOGS redefinition at runtime
28 cat >> configure.ac
<< 'END'
32 cat > Makefile.am
<< 'END'
33 TESTS
= foo.
test bar.
test baz.
test
34 XFAIL_TESTS
= bar.
test
39 # foo.test and bar.test sleep to ensure their logs are always strictly newer
40 # than the logs of their prerequisites, for HP-UX make. The quoting pleases
42 cat > foo.
test <<'END'
48 cat > bar.
test <<'END'
54 cat > baz.
test <<'END'
59 chmod a
+x foo.
test bar.
test baz.
test
67 $MAKE check
>stdout
&& { cat stdout
; exit 1; }
69 count_test_results total
=3 pass
=1 fail
=1 skip
=0 xfail
=0 xpass
=0 error
=1
70 test -f test-suite.log
72 test $
(grep -c '^FAIL:' test-suite.log
) -eq 1
73 test $
(grep -c '^ERROR:' test-suite.log
) -eq 1
74 $EGREP '^(X?PASS|XFAIL|SKIP)' test-suite.log
&& exit 1
83 test ! -e test-suite.log
85 # Check dependencies: baz.test needs to run before bar.test,
86 # but foo.test is not needed.
87 # Note that this usage has a problem: the summary will only
88 # take bar.log into account, because the $(TEST_SUITE_LOG) rule
89 # does not "see" baz.log. Hmm.
90 env TESTS
='bar.test' $MAKE -e check
>stdout
&& { cat stdout
; exit 1; }
92 grep '^FAIL: baz\.test$' stdout
93 grep '^ERROR: bar\.test$' stdout
98 test -f test-suite.log
100 # Upon a lazy rerun, foo.test should be run, but the others shouldn't.
101 # Note that the lazy rerun still exits with a failure, due to the previous
103 # Note that the previous test and this one taken together expose the timing
104 # issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG
105 # before running the tests lazily.
106 env RECHECK_LOGS
= $MAKE -e check
> stdout
&& { cat stdout
; exit 1; }
109 grep '^PASS: foo\.test$' stdout
110 grep bar.
test stdout
&& exit 1
111 grep baz.
test stdout
&& exit 1
112 grep '^# PASS: *1$' stdout
113 grep '^# FAIL: *1$' stdout
114 grep '^# ERROR: *1$' stdout
116 # Now, explicitly retry with all test logs already updated, and ensure
117 # that the summary is still displayed.
118 env RECHECK_LOGS
= $MAKE -e check
> stdout
&& { cat stdout
; exit 1; }
120 grep foo.
test stdout
&& exit 1
121 grep bar.
test stdout
&& exit 1
122 grep baz.
test stdout
&& exit 1
123 grep '^# PASS: *1$' stdout
124 grep '^# FAIL: *1$' stdout
125 grep '^# ERROR: *1$' stdout
127 # Lazily rerunning only foo should only rerun this one test.
128 env RECHECK_LOGS
=foo.log
$MAKE -e check
> stdout
&& { cat stdout
; exit 1; }
131 grep bar.
test stdout
&& exit 1
132 grep baz.
test stdout
&& exit 1
133 grep '^# PASS: *1$' stdout
134 grep '^# FAIL: *1$' stdout
135 grep '^# ERROR: *1$' stdout
138 env TEST_LOGS
=baz.log
$MAKE -e check
> stdout
&& { cat stdout
; exit 1; }
140 grep foo.
test stdout
&& exit 1
141 grep bar.
test stdout
&& exit 1
145 env TESTS
=baz.
test $MAKE -e check
> stdout
&& { cat stdout
; exit 1; }
147 grep foo.
test stdout
&& exit 1
148 grep bar.
test stdout
&& exit 1