tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)         Lumiera.org
   3 #    2007 - 2008,        Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28
  29 #=intro
  30 #=tests Writing tests
  31 # =run
  32 #=config
  33 #=configf
  34 # =make
  35 # =control
  36 #=valgrind
  37 #=libtool
  38
  39 LC_ALL=C
  40
  41 #config HEAD- Configuration; configuration; configure tests
  42 #config
  43 #config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr
  44 #config  LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  45 #config
  46 #config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for
  47 #config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out.
  48 #config The default as shown above filters some NoBug annotations and non fatal logging out.
  49 #config
  50 LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  51
  52 #config PARA Resource Limits; ulimit; constrain resource limits
  53 #config It is possible to set some limits for tests to protect the system against really broken cases.
  54 #config Since running under valgrind takes consinderable more resources there are separate variants for
  55 #config limits when running under valgrind.
  56 #config
  57 #config  LIMIT_CPU=5
  58 #config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks.
  59 #config
  60 #config  LIMIT_TIME=10
  61 #config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks.
  62 #config
  63 #config  LIMIT_VSZ=524288
  64 #config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached.
  65 #config Protects against memory leaks.
  66 #config
  67 #config  LIMIT_VG_CPU=20
  68 #config  LIMIT_VG_TIME=30
  69 #config  LIMIT_VG_VSZ=524288
  70 #config Same variables again with limits when running under valgrind.
  71 #config
  72 LIMIT_CPU=5
  73 LIMIT_TIME=10
  74 LIMIT_VSZ=524288
  75 LIMIT_VG_CPU=20
  76 LIMIT_VG_TIME=30
  77 LIMIT_VG_VSZ=524288
  78
  79
  80 #configf HEAD~ Configuration Files; configuration files; define variables to configure the test
  81 #configf
  82 #configf `test.sh` reads config files from the following location if they are exist
  83 #configf * 'test.conf' from the current directory
  84 #configf * '$srcdir/test.conf' `$srcdir` is set by autotools
  85 #configf * '$TEST_CONF' a user defineable variable to point to a config file
  86 #configf
  87 test -f 'test.conf' && source test.conf
  88 test "$srcdir" -a -f '$srcdir/test.conf' && source "$srcdir/test.conf"
  89 test "$TEST_CONF" -a -f "$TEST_CONF" && source "$TEST_CONF"
  90
  91 arg0="$0"
  92 TESTDIR="$(dirname "$arg0")"
  93
  94
  95 #libtool HEAD Libtool; libtool; support for libtool
  96 #libtool When test.sh detects the presence of './libtool' it runs all tests with
  97 #libtool `./libtool --mode=execute`.
  98 #libtool
  99 LIBTOOL_EX=
 100 if test -x ./libtool; then
 101     LIBTOOL_EX="./libtool --mode=execute"
 102 fi
 103
 104 #valgrind HEAD- Valgrind; valgrind; valgrind support
 105 #valgrind Test are run under valgrind supervision by default, if not disabled.
 106 #valgrind
 107 #valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options
 108 #valgrind  VALGRINDFLAGS="--leak-check=yes --show-reachable=yes"
 109 #valgrind
 110 #valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override
 111 #valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable
 112 #valgrind valgrind for the tests.
 113 #valgrind
 114 #valgrind HEAD~ Generating Valgrind Suppression Files; vgsuppression; ignore false positives
 115 #valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then
 116 #valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress
 117 #valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known
 118 #valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does
 119 #valgrind not generate true positives.
 120 #valgrind
 121 ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288}
 122 valgrind=""
 123 LIMIT_TIME_REAL="$LIMIT_TIME"
 124 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 125     echo "valgrind explicit disabled"
 126 else
 127     if [ "$(which valgrind)" ]; then
 128         ulimit -S -t ${ULIMIT_VG_CPU:-20} -v ${ULIMIT_VG_VSZ:-524288}
 129         LIMIT_TIME_REAL="$LIMIT_VG_TIME"
 130         if [[ -x 'vgsuppression' ]]; then
 131             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
 132                 echo 'generating valgrind supression file'
 133
 134                 $LIBTOOL_EX $(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \
 135                     | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
 136             fi
 137             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q"
 138         else
 139             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
 140         fi
 141     else
 142         echo "no valgrind found, go without it"
 143     fi
 144 fi
 145
 146 echo
 147 echo "================ ${0##*/} ================"
 148
 149 TESTCNT=0
 150 SKIPCNT=0
 151 FAILCNT=0
 152
 153
 154 # the old testlog if existing will be used to check for previous test states
 155 if test -f ,testlog; then
 156     mv ,testlog ,testlog.pre
 157 else
 158     touch ,testlog.pre
 159 fi
 160
 161 date >,testlog
 162
 163 function compare_regex() # rxfile plainfile
 164 {
 165     local regex
 166     local line
 167     local miss
 168     local lineno=1
 169     local regexno=1
 170     {
 171         IFS='' read -u 3 -r regex || return 0
 172         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
 173         while true; do
 174             if [[ $line =~ $regex ]]; then
 175                 IFS='' read -u 4 -r line ||
 176                 if IFS='' read -u 3 -r regex; then
 177                     echo "premature end in output, expecting: '$regex':$regexno"
 178                     return 1
 179                 else
 180                     return 0
 181                 fi
 182                 : $((++lineno))
 183                 miss=0
 184             else
 185                 if [[ $((++miss)) -gt 1 ]]; then
 186                     echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno"
 187                     return 1
 188                 fi
 189                 IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; }
 190                 : $((++regexno))
 191             fi
 192         done
 193     } 3<"$1" 4<"$2"
 194 }
 195
 196 #tests HEAD- Writing Tests; tests; how to write testsuites
 197 #tests
 198 #tests
 199 #tests
 200 #tests
 201 #tests
 202 #tests
 203 #tests
 204 #tests
 205 #tests
 206 #tests
 207 #tests
 208 #tests
 209 #tests
 210 #tests
 211 #tests
 212 #tests
 213 #tests
 214 #tests
 215 #tests
 216 #tests
 217 function TESTING()
 218 {
 219     echo
 220     echo "$1"
 221     echo -e "\n#### $1" >>,testlog
 222
 223     TESTBIN="$2"
 224 }
 225
 226 #tests
 227 #tests
 228 #tests
 229 #tests
 230 #tests
 231 function TEST()
 232 {
 233         name="$1"
 234         shift
 235         rm -f ,send_stdin
 236         rm -f ,expect_stdout
 237         rm -f ,expect_stderr
 238         expect_return=0
 239
 240         local valgrind="$valgrind"
 241         if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 242             valgrind=
 243         fi
 244
 245         while read -r line; do
 246             cmd="${line%%:*}"
 247             arg="${line#*:}"
 248             arg="${arg:1}"
 249             if [[ ! "$arg" ]]; then
 250                 arg='^$'
 251             fi
 252
 253             case $cmd in
 254             'in')
 255                 echo "$arg" >>,send_stdin
 256                 ;;
 257             'out')
 258                 echo "$arg" >>,expect_stdout
 259                 ;;
 260             'err')
 261                 echo "$arg" >>,expect_stderr
 262                 ;;
 263             'return')
 264                 expect_return="$arg"
 265                 ;;
 266             '#'*|'')
 267                 :
 268                 ;;
 269             *)
 270                 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 271                 exit
 272                 ;;
 273             esac
 274         done
 275         echo -n "TEST $name: "
 276         echo -en "\nTEST $name: $* " >>,testlog
 277
 278         case "$TESTMODE" in
 279         *FAST*)
 280             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 281                 MSGOK=" (fixed)"
 282                 MSGFAIL=" (still broken)"
 283             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 284                 echo ".. SKIPPED (ok)"
 285                 echo ".. SKIPPED (ok)" >>,testlog
 286                 SKIPCNT=$(($SKIPCNT + 1))
 287                 TESTCNT=$(($TESTCNT + 1))
 288                 return
 289             else
 290                 MSGOK=" (new)"
 291                 MSGFAIL=" (new)"
 292             fi
 293             ;;
 294         *)
 295             MSGOK=""
 296             MSGFAIL=""
 297             ;;
 298         esac
 299
 300         TESTCNT=$(($TESTCNT + 1))
 301
 302         fails=0
 303
 304         echo -n >,testtmp
 305
 306         local CALL
 307         if declare -F | grep $TESTBIN >&/dev/null; then
 308             CALL=
 309         elif test -x $TESTBIN; then
 310             CALL="env $LIBTOOL_EX $valgrind"
 311         else
 312             CALL='-'
 313             echo -n >,stdout
 314             echo "test binary '$TESTBIN' not found" >,stderr
 315             ((fails+=1))
 316         fi
 317
 318         if test "$CALL" != '-'; then
 319
 320             if test -f ,send_stdin; then
 321                 $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 322             else
 323                 $CALL $TESTBIN "$@" 2>,stderr >,stdout
 324             fi &>/dev/null
 325             return=$?
 326
 327             if test -f ,expect_stdout; then
 328                 grep -v "$LOGSUPPRESS" <,stdout >,tmp
 329                 if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then
 330                     echo "unexpected data on stdout" >>,testtmp
 331                     cat ,cmptmp >>,testtmp
 332                     ((fails+=1))
 333                 fi
 334                 rm ,tmp ,cmptmp
 335             fi
 336
 337             if test -f ,expect_stderr; then
 338                 grep -v "$LOGSUPPRESS" <,stderr >,tmp
 339                     cat ,tmp >>,testtmp
 340                 if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then
 341                     echo "unexpected data on stderr" >>,testtmp
 342                     cat ,cmptmp >>,testtmp
 343                     ((fails+=1))
 344                 fi
 345                 rm ,tmp ,cmptmp
 346             fi
 347
 348             if [[ "${expect_return:0:1}" = '!' ]]; then
 349                 if [[ "${expect_return#\!}" = "$return" ]]; then
 350                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 351                     ((fails+=1))
 352                 fi
 353             else
 354                 if [[ "${expect_return}" != "$return" ]]; then
 355                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 356                     ((fails+=1))
 357                 fi
 358             fi
 359         fi
 360
 361         if test $fails -eq 0; then
 362             echo ".. OK$MSGOK"
 363             echo ".. OK$MSGOK" >>,testlog
 364         else
 365             echo ".. FAILED$MSGFAIL";
 366             echo ".. FAILED$MSGFAIL" >>,testlog
 367             cat ,testtmp >>,testlog
 368             rm ,testtmp
 369             echo "stderr was:" >>,testlog
 370             cat ,stderr >>,testlog
 371             echo END >>,testlog
 372             FAILCNT=$(($FAILCNT + 1))
 373             case $TESTMODE in
 374             *FIRSTFAIL*)
 375                 break 2
 376                 ;;
 377             esac
 378         fi
 379 }
 380
 381 function PLANNED()
 382 {
 383         echo -n "PLANNED $1: "
 384         echo -en "\nPLANNED $* " >>,testlog
 385         echo ".. SKIPPED (planned)"
 386         echo ".. SKIPPED (planned)" >>,testlog
 387         SKIPCNT=$(($SKIPCNT + 1))
 388         TESTCNT=$(($TESTCNT + 1))
 389 }
 390
 391 function RUNTESTS()
 392 {
 393     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 394         TESTSUITES="{$TESTSUITES}"
 395     fi
 396     for t in $(eval echo "$TESTDIR/*$TESTSUITES*.tests"); do
 397         echo "$t"
 398     done | sort | uniq | {
 399         while read i; do
 400             echo
 401             echo "### $i" >&2
 402             if test -f $i; then
 403                 source $i
 404             fi
 405         done
 406         echo
 407         if [ $FAILCNT = 0 ]; then
 408             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 409             #rm ,testlog
 410         else
 411             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 412             echo " ... FAILED $FAILCNT TESTS"
 413             echo " ... SKIPPED $SKIPCNT TESTS"
 414             echo " see ',testlog' for details"
 415             exit 1
 416         fi
 417     }
 418 }
 419
 420 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 421
 422 RUNTESTS