tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)         Lumiera.org
   3 #    2007 - 2008,        Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28 export LC_ALL=C
  29 NOBUG_LOGREGEX='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  30
  31 arg0="$0"
  32 srcdir="$(dirname "$arg0")"
  33
  34 ulimit -S -t 5 -v 524288
  35 valgrind=""
  36 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
  37     echo "valgrind explicit disabled"
  38 else
  39     if [ "$(which valgrind)" ]; then
  40         ulimit -S -t 20
  41         if [[ -x 'vgsuppression' ]]; then
  42             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
  43                 echo 'generating valgrind supression file'
  44
  45                 if [[ -x ".libs/vgsuppression" ]]; then
  46                     ./libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all vgsuppression 2>&1 \
  47                         | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp
  48                 else
  49                     valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all ./vgsuppression 2>&1 \
  50                         | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp
  51                 fi
  52             fi
  53             valgrind="$(which valgrind) --leak-check=yes --show-reachable=no --suppressions=vgsuppression.supp -q $VALGRINDFLAGS"
  54         else
  55             valgrind="$(which valgrind) --leak-check=yes --show-reachable=no -q $VALGRINDFLAGS"
  56         fi
  57     else
  58         echo "no valgrind found, go without it"
  59     fi
  60 fi
  61
  62 echo
  63 echo "================ ${0##*/} ================"
  64
  65 TESTCNT=0
  66 SKIPCNT=0
  67 FAILCNT=0
  68
  69
  70 # the old testlog if existing will be used to check for previous test states
  71 if test -f ,testlog; then
  72     mv ,testlog ,testlog.pre
  73 else
  74     touch ,testlog.pre
  75 fi
  76
  77 date >,testlog
  78
  79 function compare_regex() # rxfile plainfile
  80 {
  81     local regex
  82     local line
  83     local miss
  84     local lineno=1
  85     local regexno=1
  86     {
  87         IFS='' read -u 3 -r regex || return 0
  88         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
  89         while true; do
  90             if [[ $line =~ $regex ]]; then
  91                 IFS='' read -u 4 -r line ||
  92                 if IFS='' read -u 3 -r regex; then
  93                     echo "premature end in output, expecting: '$regex':$regexno"
  94                     return 1
  95                 else
  96                     return 0
  97                 fi
  98                 : $((++lineno))
  99                 miss=0
 100             else
 101                 if [[ $((++miss)) -gt 1 ]]; then
 102                     echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno"
 103                     return 1
 104                 fi
 105                 IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; }
 106                 : $((++regexno))
 107             fi
 108         done
 109     } 3<"$1" 4<"$2"
 110 }
 111
 112
 113 function TEST()
 114 {
 115         name="$1"
 116         shift
 117         rm -f ,send_stdin
 118         rm -f ,expect_stdout
 119         rm -f ,expect_stderr
 120         expect_return=0
 121
 122         while read -r line; do
 123             cmd="${line%%:*}"
 124             arg="${line#*:}"
 125             arg="${arg:1}"
 126             if [[ ! "$arg" ]]; then
 127                 arg='^$'
 128             fi
 129
 130             case $cmd in
 131             'in')
 132                 echo "$arg" >>,send_stdin
 133                 ;;
 134             'out')
 135                 echo "$arg" >>,expect_stdout
 136                 ;;
 137             'err')
 138                 echo "$arg" >>,expect_stderr
 139                 ;;
 140             'return')
 141                 expect_return="$arg"
 142                 ;;
 143             '#'*|'')
 144                 :
 145                 ;;
 146             *)
 147                 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 148                 exit
 149                 ;;
 150             esac
 151         done
 152         echo -n "TEST $name: "
 153         echo -en "\nTEST $name: $* " >>,testlog
 154
 155         case "$TESTMODE" in
 156         *FAST*)
 157             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 158                 MSGOK=" (fixed)"
 159                 MSGFAIL=" (still broken)"
 160             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 161                 echo ".. SKIPPED (ok)"
 162                 echo ".. SKIPPED (ok)" >>,testlog
 163                 SKIPCNT=$(($SKIPCNT + 1))
 164                 TESTCNT=$(($TESTCNT + 1))
 165                 return
 166             else
 167                 MSGOK=" (new)"
 168                 MSGFAIL=" (new)"
 169             fi
 170             ;;
 171         *)
 172             MSGOK=""
 173             MSGFAIL=""
 174             ;;
 175         esac
 176
 177         TESTCNT=$(($TESTCNT + 1))
 178
 179         fails=0
 180
 181         echo -n >,testtmp
 182
 183         local CALL
 184         if declare -F | grep $TESTBIN >&/dev/null; then
 185             CALL=
 186         elif test -x $TESTBIN; then
 187             CALL="env $TESTBIN_PREFIX"
 188         else
 189             CALL='-'
 190             echo -n >,stdout
 191             echo "test binary '$TESTBIN' not found" >,stderr
 192             ((fails+=1))
 193         fi
 194
 195         if test "$CALL" != '-'; then
 196
 197             if test -f ,send_stdin; then
 198                 $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 199             else
 200                 $CALL $TESTBIN "$@" 2>,stderr >,stdout
 201             fi &>/dev/null
 202             return=$?
 203
 204             if test -f ,expect_stdout; then
 205                 grep -v "$NOBUG_LOGREGEX" <,stdout >,tmp
 206                 if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then
 207                     echo "unexpected data on stdout" >>,testtmp
 208                     cat ,cmptmp >>,testtmp
 209                     ((fails+=1))
 210                 fi
 211                 rm ,tmp ,cmptmp
 212             fi
 213
 214             if test -f ,expect_stderr; then
 215                 grep -v "$NOBUG_LOGREGEX" <,stderr >,tmp
 216                     cat ,tmp >>,testtmp
 217                 if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then
 218                     echo "unexpected data on stderr" >>,testtmp
 219                     cat ,cmptmp >>,testtmp
 220                     ((fails+=1))
 221                 fi
 222                 rm ,tmp ,cmptmp
 223             fi
 224
 225             if [[ "${expect_return:0:1}" = '!' ]]; then
 226                 if [[ "${expect_return#\!}" = "$return" ]]; then
 227                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 228                     ((fails+=1))
 229                 fi
 230             else
 231                 if [[ "${expect_return}" != "$return" ]]; then
 232                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 233                     ((fails+=1))
 234                 fi
 235             fi
 236         fi
 237
 238         if test $fails -eq 0; then
 239             echo ".. OK$MSGOK"
 240             echo ".. OK$MSGOK" >>,testlog
 241         else
 242             echo ".. FAILED$MSGFAIL";
 243             echo ".. FAILED$MSGFAIL" >>,testlog
 244             cat ,testtmp >>,testlog
 245             rm ,testtmp
 246             echo "stderr was:" >>,testlog
 247             cat ,stderr >>,testlog
 248             echo END >>,testlog
 249             FAILCNT=$(($FAILCNT + 1))
 250             case $TESTMODE in
 251             *FIRSTFAIL*)
 252                 break 2
 253                 ;;
 254             esac
 255         fi
 256 }
 257
 258 function PLANNED()
 259 {
 260         echo -n "PLANNED $1: "
 261         echo -en "\nPLANNED $* " >>,testlog
 262         echo ".. SKIPPED (planned)"
 263         echo ".. SKIPPED (planned)" >>,testlog
 264         SKIPCNT=$(($SKIPCNT + 1))
 265         TESTCNT=$(($TESTCNT + 1))
 266 }
 267
 268 function RUNTESTS()
 269 {
 270     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 271         TESTSUITES="{$TESTSUITES}"
 272     fi
 273     for t in $(eval echo "$srcdir/*$TESTSUITES*.tests"); do
 274         echo "$t"
 275     done | sort | uniq | {
 276         while read i; do
 277             echo
 278             echo "### $i" >&2
 279             if test -f $i; then
 280                 source $i
 281             fi
 282         done
 283         echo
 284         if [ $FAILCNT = 0 ]; then
 285             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 286             #rm ,testlog
 287         else
 288             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 289             echo " ... FAILED $FAILCNT TESTS"
 290             echo " ... SKIPPED $SKIPCNT TESTS"
 291             echo " see ',testlog' for details"
 292             exit 1
 293         fi
 294     }
 295 }
 296
 297 function TESTING()
 298 {
 299     echo
 300     echo "$1"
 301     echo -e "\n#### $1" >>,testlog
 302
 303     if [[ -x ".libs/$2" ]]; then
 304         TESTBIN_PREFIX="./libtool --mode=execute $valgrind"
 305     else
 306         TESTBIN_PREFIX="$valgrind"
 307     fi
 308     TESTBIN="$2"
 309 }
 310
 311 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 312
 313 RUNTESTS