tests/test.sh

   1 #!/bin/bash
   2 #  Copyright (C)         Lumiera.org
   3 #    2007 - 2008,        Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28 export LC_ALL=C
  29 NOBUG_LOGREGEX='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\|TODO\|PLANNED\|FIXME\|DEPRECATED\|UNIMPLEMENTED\|NOTREACHED\):'
  30
  31 arg0="$0"
  32 srcdir="$(dirname "$arg0")"
  33
  34 ulimit -S -t 5 -v 524288
  35 valgrind=""
  36 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
  37     echo "valgrind explicit disabled"
  38 else
  39     if [ "$(which valgrind)" ]; then
  40         ulimit -S -t 20
  41         if [[ -x 'vgsuppression' ]]; then
  42             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
  43                 echo 'generating valgrind supression file'
  44
  45                 if [[ -x ".libs/vgsuppression" ]]; then
  46                     ./libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all vgsuppression 2>&1 \
  47                         | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp
  48                 else
  49                     valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all ./vgsuppression 2>&1 \
  50                         | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp
  51                 fi
  52             fi
  53             valgrind="$(which valgrind) --leak-check=yes --show-reachable=no --suppressions=vgsuppression.supp -q $VALGRINDFLAGS"
  54         else
  55             valgrind="$(which valgrind) --leak-check=yes --show-reachable=no -q $VALGRINDFLAGS"
  56         fi
  57     else
  58         echo "no valgrind found, go without it"
  59     fi
  60 fi
  61
  62 echo
  63 echo "================ ${0##*/} ================"
  64
  65 TESTCNT=0
  66 SKIPCNT=0
  67 FAILCNT=0
  68
  69
  70 # the old testlog if existing will be used to check for previous test states
  71 if test -f ,testlog; then
  72     mv ,testlog ,testlog.pre
  73 else
  74     touch ,testlog.pre
  75 fi
  76
  77 date >,testlog
  78
  79 function compare_regex() # rxfile plainfile
  80 {
  81     local regex
  82     local line
  83     local miss
  84     local lineno=1
  85     local regexno=1
  86     {
  87         IFS='' read -u 3 -r regex || return 0
  88         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
  89         while true; do
  90             if [[ $line =~ $regex ]]; then
  91                 IFS='' read -u 4 -r line ||
  92                 if IFS='' read -u 3 -r regex; then
  93                     echo "premature end in output, expecting: '$regex':$regexno"
  94                     return 1
  95                 else
  96                     return 0
  97                 fi
  98                 : $((++lineno))
  99                 miss=0
 100             else
 101                 if [[ $((++miss)) -gt 1 ]]; then
 102                     echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno"
 103                     return 1
 104                 fi
 105                 IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; }
 106                 : $((++regexno))
 107             fi
 108         done
 109     } 3<"$1" 4<"$2"
 110 }
 111
 112
 113 function TEST()
 114 {
 115         name="$1"
 116         shift
 117         rm -f ,send_stdin
 118         rm -f ,expect_stdout
 119         rm -f ,expect_stderr
 120
 121         while read -r line; do
 122             cmd="${line%%:*}"
 123             arg="${line#*:}"
 124             arg="${arg:1}"
 125             if [[ ! "$arg" ]]; then
 126                 arg='^$'
 127             fi
 128             expect_return=0
 129
 130             case $cmd in
 131             'in')
 132                 echo "$arg" >>,send_stdin
 133                 ;;
 134             'out')
 135                 echo "$arg" >>,expect_stdout
 136                 ;;
 137             'err')
 138                 echo "$arg" >>,expect_stderr
 139                 ;;
 140             'return')
 141                 expect_return="$arg"
 142                 ;;
 143             '#'*|'')
 144                 :
 145                 ;;
 146             *)
 147                 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 148                 exit
 149                 ;;
 150             esac
 151         done
 152         echo -n "TEST $name: "
 153         echo -en "\nTEST $name: $* " >>,testlog
 154
 155         case "$TESTMODE" in
 156         *FAST*)
 157             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 158                 MSGOK=" (fixed)"
 159                 MSGFAIL=" (still broken)"
 160             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 161                 echo ".. SKIPPED (ok)"
 162                 echo ".. SKIPPED (ok)" >>,testlog
 163                 SKIPCNT=$(($SKIPCNT + 1))
 164                 TESTCNT=$(($TESTCNT + 1))
 165                 return
 166             else
 167                 MSGOK=" (new)"
 168                 MSGFAIL=" (new)"
 169             fi
 170             ;;
 171         *)
 172             MSGOK=""
 173             MSGFAIL=""
 174             ;;
 175         esac
 176
 177         TESTCNT=$(($TESTCNT + 1))
 178
 179         fails=0
 180
 181         echo -n >,testtmp
 182
 183         if ! test -x $TESTBIN; then
 184             echo -n >,stdout
 185             echo "test binary '$TESTBIN' not found" >,stderr
 186             ((fails+=1))
 187
 188         else
 189
 190             if test -f ,send_stdin; then
 191                 env $TESTBIN_PREFIX $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 192             else
 193                 env $TESTBIN_PREFIX $TESTBIN "$@" 2>,stderr >,stdout
 194             fi &>/dev/null
 195             return=$?
 196
 197             if test -f ,expect_stdout; then
 198                 grep -v "$NOBUG_LOGREGEX" <,stdout >,tmp
 199                 if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then
 200                     echo "unexpected data on stdout" >>,testtmp
 201                     cat ,cmptmp >>,testtmp
 202                     ((fails+=1))
 203                 fi
 204                 rm ,tmp ,cmptmp
 205             fi
 206
 207             if test -f ,expect_stderr; then
 208                 grep -v "$NOBUG_LOGREGEX" <,stderr >,tmp
 209                     cat ,tmp >>,testtmp
 210                 if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then
 211                     echo "unexpected data on stderr" >>,testtmp
 212                     cat ,cmptmp >>,testtmp
 213                     ((fails+=1))
 214                 fi
 215                 rm ,tmp ,cmptmp
 216             fi
 217
 218             if [[ "${expect_return:0:1}" = '!' ]]; then
 219                 if [[ "${expect_return#\!}" = "$return" ]]; then
 220                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 221                     ((fails+=1))
 222                 fi
 223             else
 224                 if [[ "${expect_return}" != "$return" ]]; then
 225                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 226                     ((fails+=1))
 227                 fi
 228             fi
 229         fi
 230
 231         if test $fails -eq 0; then
 232             echo ".. OK$MSGOK"
 233             echo ".. OK$MSGOK" >>,testlog
 234         else
 235             echo ".. FAILED$MSGFAIL";
 236             echo ".. FAILED$MSGFAIL" >>,testlog
 237             cat ,testtmp >>,testlog
 238             rm ,testtmp
 239             echo "stderr was:" >>,testlog
 240             cat ,stderr >>,testlog
 241             echo END >>,testlog
 242             FAILCNT=$(($FAILCNT + 1))
 243             case $TESTMODE in
 244             *FIRSTFAIL*)
 245                 break 2
 246                 ;;
 247             esac
 248         fi
 249 }
 250
 251 function PLANNED()
 252 {
 253         echo -n "PLANNED $1: "
 254         echo -en "\nPLANNED $* " >>,testlog
 255         echo ".. SKIPPED (planned)"
 256         echo ".. SKIPPED (planned)" >>,testlog
 257         SKIPCNT=$(($SKIPCNT + 1))
 258         TESTCNT=$(($TESTCNT + 1))
 259 }
 260
 261 function RUNTESTS()
 262 {
 263     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 264         TESTSUITES="{$TESTSUITES}"
 265     fi
 266     for t in $(eval echo "$srcdir/*$TESTSUITES*.tests"); do
 267         echo "$t"
 268     done | sort | uniq | {
 269         while read i; do
 270             echo
 271             echo "### $i" >&2
 272             if test -f $i; then
 273                 source $i
 274             fi
 275         done
 276         echo
 277         if [ $FAILCNT = 0 ]; then
 278             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 279             #rm ,testlog
 280         else
 281             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 282             echo " ... FAILED $FAILCNT TESTS"
 283             echo " ... SKIPPED $SKIPCNT TESTS"
 284             echo " see ',testlog' for details"
 285             exit 1
 286         fi
 287     }
 288 }
 289
 290 function TESTING()
 291 {
 292     echo
 293     echo "$1"
 294     echo -e "\n#### $1" >>,testlog
 295
 296     if [[ -x ".libs/$2" ]]; then
 297         TESTBIN_PREFIX="./libtool --mode=execute $valgrind"
 298     else
 299         TESTBIN_PREFIX="$valgrind"
 300     fi
 301     TESTBIN="$2"
 302 }
 303
 304 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 305
 306 RUNTESTS