2 # Copyright (C) Lumiera.org
3 # 2007 - 2008, Christian Thaeter <ct@pipapo.org>
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License as
7 # published by the Free Software Foundation; either version 2 of the
8 # License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 # TESTMODE=FULL yet unimplemented
20 # run all tests, PLANNED which fail count as error
23 # run only tests which recently failed
26 # stop testing on the first failure
41 #config HEAD- Configuration; configuration; configure tests
43 #config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr
44 #config LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
46 #config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for
47 #config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out.
48 #config The default as shown above filters some NoBug annotations and non fatal logging out.
50 LOGSUPPRESS
='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
52 #config PARA Resource Limits; ulimit; constrain resource limits
53 #config It is possible to set some limits for tests to protect the system against really broken cases.
54 #config Since running under valgrind takes consinderable more resources there are separate variants for
55 #config limits when running under valgrind.
58 #config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks.
61 #config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks.
63 #config LIMIT_VSZ=524288
64 #config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached.
65 #config Protects against memory leaks.
67 #config LIMIT_VG_CPU=20
68 #config LIMIT_VG_TIME=30
69 #config LIMIT_VG_VSZ=524288
70 #config Same variables again with limits when running under valgrind.
80 #configf HEAD~ Configuration Files; configuration files; define variables to configure the test
82 #configf `test.sh` reads config files from the following location if they are exist
83 #configf * 'test.conf' from the current directory
84 #configf * '$srcdir/test.conf' `$srcdir` is set by autotools
85 #configf * '$TEST_CONF' a user defineable variable to point to a config file
87 test -f 'test.conf' && source test.conf
88 test "$srcdir" -a -f '$srcdir/test.conf' && source "$srcdir/test.conf"
89 test "$TEST_CONF" -a -f "$TEST_CONF" && source "$TEST_CONF"
92 TESTDIR
="$(dirname "$arg0")"
95 #libtool HEAD Libtool; libtool; support for libtool
96 #libtool When test.sh detects the presence of './libtool' it runs all tests with
97 #libtool `./libtool --mode=execute`.
100 if test -x .
/libtool
; then
101 LIBTOOL_EX
="./libtool --mode=execute"
104 #valgrind HEAD- Valgrind; valgrind; valgrind support
105 #valgrind Test are run under valgrind supervision by default, if not disabled.
107 #valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options
108 #valgrind VALGRINDFLAGS="--leak-check=yes --show-reachable=yes"
110 #valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override
111 #valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable
112 #valgrind valgrind for the tests.
114 #valgrind HEAD~ Generating Valgrind Suppression Files; vgsuppression; ignore false positives
115 #valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then
116 #valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress
117 #valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known
118 #valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does
119 #valgrind not generate true positives.
121 ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288}
123 LIMIT_TIME_REAL
="$LIMIT_TIME"
124 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
125 echo "valgrind explicit disabled"
127 if [ "$(which valgrind)" ]; then
128 ulimit -S -t ${ULIMIT_VG_CPU:-20} -v ${ULIMIT_VG_VSZ:-524288}
129 LIMIT_TIME_REAL
="$LIMIT_VG_TIME"
130 if [[ -x 'vgsuppression' ]]; then
131 if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
132 echo 'generating valgrind supression file'
134 $LIBTOOL_EX $
(which valgrind
) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression
2>&1 \
135 |
awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
137 valgrind
="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q"
139 valgrind
="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
142 echo "no valgrind found, go without it"
147 echo "================ ${0##*/} ================"
154 # the old testlog if existing will be used to check for previous test states
155 if test -f ,testlog
; then
156 mv ,testlog
,testlog.pre
163 function compare_regex
() # rxfile plainfile
171 IFS
='' read -u 3 -r regex ||
return 0
172 IFS
='' read -u 4 -r line ||
{ echo "no output"; return 1; }
174 if [[ $line =~
$regex ]]; then
175 IFS
='' read -u 4 -r line ||
176 if IFS
='' read -u 3 -r regex
; then
177 echo "premature end in output, expecting: '$regex':$regexno"
185 if [[ $
((++miss
)) -gt 1 ]]; then
186 echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno"
189 IFS
='' read -u 3 -r regex ||
{ echo "more output than expected: '$line':$lineno"; return 1; }
196 #tests HEAD- Writing Tests; tests; how to write testsuites
221 echo -e "\n#### $1" >>,testlog
240 local valgrind
="$valgrind"
241 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
245 while read -r line
; do
249 if [[ ! "$arg" ]]; then
255 echo "$arg" >>,send_stdin
258 echo "$arg" >>,expect_stdout
261 echo "$arg" >>,expect_stderr
270 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
275 echo -n "TEST $name: "
276 echo -en "\nTEST $name: $* " >>,testlog
280 if grep "^TEST $name: .* FAILED" ,testlog.pre
>&/dev
/null
; then
282 MSGFAIL
=" (still broken)"
283 elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre
>&/dev
/null
; then
284 echo ".. SKIPPED (ok)"
285 echo ".. SKIPPED (ok)" >>,testlog
286 SKIPCNT
=$
(($SKIPCNT + 1))
287 TESTCNT
=$
(($TESTCNT + 1))
300 TESTCNT
=$
(($TESTCNT + 1))
307 if declare -F |
grep $TESTBIN >&/dev
/null
; then
309 elif test -x $TESTBIN; then
310 CALL
="env $LIBTOOL_EX $valgrind"
314 echo "test binary '$TESTBIN' not found" >,stderr
318 if test "$CALL" != '-'; then
320 if test -f ,send_stdin
; then
321 $CALL $TESTBIN "$@" <,send_stdin
2>,stderr
>,stdout
323 $CALL $TESTBIN "$@" 2>,stderr
>,stdout
327 if test -f ,expect_stdout
; then
328 grep -v "$LOGSUPPRESS" <,stdout
>,tmp
329 if ! compare_regex
,expect_stdout
,tmp
>>,cmptmp
; then
330 echo "unexpected data on stdout" >>,testtmp
331 cat ,cmptmp
>>,testtmp
337 if test -f ,expect_stderr
; then
338 grep -v "$LOGSUPPRESS" <,stderr
>,tmp
340 if ! compare_regex
,expect_stderr
,tmp
>>,cmptmp
; then
341 echo "unexpected data on stderr" >>,testtmp
342 cat ,cmptmp
>>,testtmp
348 if [[ "${expect_return:0:1}" = '!' ]]; then
349 if [[ "${expect_return#\!}" = "$return" ]]; then
350 echo "unexpected return value $return, expected $expect_return" >>,testtmp
354 if [[ "${expect_return}" != "$return" ]]; then
355 echo "unexpected return value $return, expected $expect_return" >>,testtmp
361 if test $fails -eq 0; then
363 echo ".. OK$MSGOK" >>,testlog
365 echo ".. FAILED$MSGFAIL";
366 echo ".. FAILED$MSGFAIL" >>,testlog
367 cat ,testtmp
>>,testlog
369 echo "stderr was:" >>,testlog
370 cat ,stderr
>>,testlog
372 FAILCNT
=$
(($FAILCNT + 1))
383 echo -n "PLANNED $1: "
384 echo -en "\nPLANNED $* " >>,testlog
385 echo ".. SKIPPED (planned)"
386 echo ".. SKIPPED (planned)" >>,testlog
387 SKIPCNT
=$
(($SKIPCNT + 1))
388 TESTCNT
=$
(($TESTCNT + 1))
393 if test \
( ! "${TESTSUITES/*,*/}" \
) -a "$TESTSUITES"; then
394 TESTSUITES
="{$TESTSUITES}"
396 for t
in $
(eval echo "$TESTDIR/*$TESTSUITES*.tests"); do
398 done |
sort |
uniq |
{
407 if [ $FAILCNT = 0 ]; then
408 echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
411 echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
412 echo " ... FAILED $FAILCNT TESTS"
413 echo " ... SKIPPED $SKIPCNT TESTS"
414 echo " see ',testlog' for details"
420 TESTSUITES
="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"