tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)         Lumiera.org
   3 #    2007 - 2008,        Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28 export LC_ALL=C
  29 NOBUG_LOGREGEX='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  30
  31 arg0="$0"
  32 srcdir="$(dirname "$arg0")"
  33
  34 ulimit -S -t 5 -v 524288
  35 valgrind=""
  36 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
  37     echo "valgrind explicit disabled"
  38 else
  39     if [ "$(which valgrind)" ]; then
  40         ulimit -S -t 20
  41         if [[ -x 'vgsuppression' ]]; then
  42             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
  43                 echo 'generating valgrind supression file'
  44
  45                 if [[ -x ".libs/vgsuppression" ]]; then
  46                     ./libtool --mode=execute valgrind ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \
  47                         | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
  48                 else
  49                     valgrind ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all ./vgsuppression 2>&1 \
  50                         | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
  51                 fi
  52             fi
  53             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no } --suppressions=vgsuppression.supp -q"
  54         else
  55             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
  56         fi
  57     else
  58         echo "no valgrind found, go without it"
  59     fi
  60 fi
  61
  62 echo
  63 echo "================ ${0##*/} ================"
  64
  65 TESTCNT=0
  66 SKIPCNT=0
  67 FAILCNT=0
  68
  69
  70 # the old testlog if existing will be used to check for previous test states
  71 if test -f ,testlog; then
  72     mv ,testlog ,testlog.pre
  73 else
  74     touch ,testlog.pre
  75 fi
  76
  77 date >,testlog
  78
  79 function compare_regex() # rxfile plainfile
  80 {
  81     local regex
  82     local line
  83     local miss
  84     local lineno=1
  85     local regexno=1
  86     {
  87         IFS='' read -u 3 -r regex || return 0
  88         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
  89         while true; do
  90             if [[ $line =~ $regex ]]; then
  91                 IFS='' read -u 4 -r line ||
  92                 if IFS='' read -u 3 -r regex; then
  93                     echo "premature end in output, expecting: '$regex':$regexno"
  94                     return 1
  95                 else
  96                     return 0
  97                 fi
  98                 : $((++lineno))
  99                 miss=0
 100             else
 101                 if [[ $((++miss)) -gt 1 ]]; then
 102                     echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno"
 103                     return 1
 104                 fi
 105                 IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; }
 106                 : $((++regexno))
 107             fi
 108         done
 109     } 3<"$1" 4<"$2"
 110 }
 111
 112
 113 function TEST()
 114 {
 115         name="$1"
 116         shift
 117         rm -f ,send_stdin
 118         rm -f ,expect_stdout
 119         rm -f ,expect_stderr
 120         expect_return=0
 121
 122         local valgrind="$valgrind"
 123         if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 124             valgrind=
 125         fi
 126
 127         while read -r line; do
 128             cmd="${line%%:*}"
 129             arg="${line#*:}"
 130             arg="${arg:1}"
 131             if [[ ! "$arg" ]]; then
 132                 arg='^$'
 133             fi
 134
 135             case $cmd in
 136             'in')
 137                 echo "$arg" >>,send_stdin
 138                 ;;
 139             'out')
 140                 echo "$arg" >>,expect_stdout
 141                 ;;
 142             'err')
 143                 echo "$arg" >>,expect_stderr
 144                 ;;
 145             'return')
 146                 expect_return="$arg"
 147                 ;;
 148             '#'*|'')
 149                 :
 150                 ;;
 151             *)
 152                 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 153                 exit
 154                 ;;
 155             esac
 156         done
 157         echo -n "TEST $name: "
 158         echo -en "\nTEST $name: $* " >>,testlog
 159
 160         case "$TESTMODE" in
 161         *FAST*)
 162             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 163                 MSGOK=" (fixed)"
 164                 MSGFAIL=" (still broken)"
 165             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 166                 echo ".. SKIPPED (ok)"
 167                 echo ".. SKIPPED (ok)" >>,testlog
 168                 SKIPCNT=$(($SKIPCNT + 1))
 169                 TESTCNT=$(($TESTCNT + 1))
 170                 return
 171             else
 172                 MSGOK=" (new)"
 173                 MSGFAIL=" (new)"
 174             fi
 175             ;;
 176         *)
 177             MSGOK=""
 178             MSGFAIL=""
 179             ;;
 180         esac
 181
 182         TESTCNT=$(($TESTCNT + 1))
 183
 184         fails=0
 185
 186         echo -n >,testtmp
 187
 188         local CALL
 189         if declare -F | grep $TESTBIN >&/dev/null; then
 190             CALL=
 191         elif test -x $TESTBIN; then
 192             CALL="env $TESTBIN_PREFIX $valgrind"
 193         else
 194             CALL='-'
 195             echo -n >,stdout
 196             echo "test binary '$TESTBIN' not found" >,stderr
 197             ((fails+=1))
 198         fi
 199
 200         if test "$CALL" != '-'; then
 201
 202             if test -f ,send_stdin; then
 203                 $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 204             else
 205                 $CALL $TESTBIN "$@" 2>,stderr >,stdout
 206             fi &>/dev/null
 207             return=$?
 208
 209             if test -f ,expect_stdout; then
 210                 grep -v "$NOBUG_LOGREGEX" <,stdout >,tmp
 211                 if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then
 212                     echo "unexpected data on stdout" >>,testtmp
 213                     cat ,cmptmp >>,testtmp
 214                     ((fails+=1))
 215                 fi
 216                 rm ,tmp ,cmptmp
 217             fi
 218
 219             if test -f ,expect_stderr; then
 220                 grep -v "$NOBUG_LOGREGEX" <,stderr >,tmp
 221                     cat ,tmp >>,testtmp
 222                 if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then
 223                     echo "unexpected data on stderr" >>,testtmp
 224                     cat ,cmptmp >>,testtmp
 225                     ((fails+=1))
 226                 fi
 227                 rm ,tmp ,cmptmp
 228             fi
 229
 230             if [[ "${expect_return:0:1}" = '!' ]]; then
 231                 if [[ "${expect_return#\!}" = "$return" ]]; then
 232                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 233                     ((fails+=1))
 234                 fi
 235             else
 236                 if [[ "${expect_return}" != "$return" ]]; then
 237                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 238                     ((fails+=1))
 239                 fi
 240             fi
 241         fi
 242
 243         if test $fails -eq 0; then
 244             echo ".. OK$MSGOK"
 245             echo ".. OK$MSGOK" >>,testlog
 246         else
 247             echo ".. FAILED$MSGFAIL";
 248             echo ".. FAILED$MSGFAIL" >>,testlog
 249             cat ,testtmp >>,testlog
 250             rm ,testtmp
 251             echo "stderr was:" >>,testlog
 252             cat ,stderr >>,testlog
 253             echo END >>,testlog
 254             FAILCNT=$(($FAILCNT + 1))
 255             case $TESTMODE in
 256             *FIRSTFAIL*)
 257                 break 2
 258                 ;;
 259             esac
 260         fi
 261 }
 262
 263 function PLANNED()
 264 {
 265         echo -n "PLANNED $1: "
 266         echo -en "\nPLANNED $* " >>,testlog
 267         echo ".. SKIPPED (planned)"
 268         echo ".. SKIPPED (planned)" >>,testlog
 269         SKIPCNT=$(($SKIPCNT + 1))
 270         TESTCNT=$(($TESTCNT + 1))
 271 }
 272
 273 function RUNTESTS()
 274 {
 275     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 276         TESTSUITES="{$TESTSUITES}"
 277     fi
 278     for t in $(eval echo "$srcdir/*$TESTSUITES*.tests"); do
 279         echo "$t"
 280     done | sort | uniq | {
 281         while read i; do
 282             echo
 283             echo "### $i" >&2
 284             if test -f $i; then
 285                 source $i
 286             fi
 287         done
 288         echo
 289         if [ $FAILCNT = 0 ]; then
 290             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 291             #rm ,testlog
 292         else
 293             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 294             echo " ... FAILED $FAILCNT TESTS"
 295             echo " ... SKIPPED $SKIPCNT TESTS"
 296             echo " see ',testlog' for details"
 297             exit 1
 298         fi
 299     }
 300 }
 301
 302 function TESTING()
 303 {
 304     echo
 305     echo "$1"
 306     echo -e "\n#### $1" >>,testlog
 307
 308     if [[ -x ".libs/$2" ]]; then
 309         TESTBIN_PREFIX="./libtool --mode=execute"
 310     else
 311         TESTBIN_PREFIX=
 312     fi
 313     TESTBIN="$2"
 314 }
 315
 316 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 317
 318 RUNTESTS