tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)         Lumiera.org
   3 #    2007 - 2008,        Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28
  29 #=intro
  30 #=tests Writing tests
  31 # =run
  32 #=config
  33 #=configf
  34 # =make
  35 # =control
  36 #=valgrind
  37 #=libtool
  38
  39 LC_ALL=C
  40
  41 #config HEAD- Configuration; configuration; configure tests
  42 #config
  43 #config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr
  44 #config  LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  45 #config
  46 #config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for
  47 #config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out.
  48 #config The default as shown above filters some NoBug annotations and non fatal logging out.
  49 #config
  50 LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  51
  52 #config PARA Resource Limits; ulimit; constrain resource limits
  53 #config It is possible to set some limits for tests to protect the system against really broken cases.
  54 #config Since running under valgrind takes consinderable more resources there are separate variants for
  55 #config limits when running under valgrind.
  56 #config
  57 #config  LIMIT_CPU=5
  58 #config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks.
  59 #config
  60 #config  LIMIT_TIME=10
  61 #config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks.
  62 #config
  63 #config  LIMIT_VSZ=524288
  64 #config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached.
  65 #config Protects against memory leaks.
  66 #config
  67 #config  LIMIT_VG_CPU=20
  68 #config  LIMIT_VG_TIME=30
  69 #config  LIMIT_VG_VSZ=524288
  70 #config Same variables again with limits when running under valgrind.
  71 #config
  72 LIMIT_CPU=5
  73 LIMIT_TIME=10
  74 LIMIT_VSZ=524288
  75 LIMIT_VG_CPU=20
  76 LIMIT_VG_TIME=30
  77 LIMIT_VG_VSZ=524288
  78
  79
  80 #configf HEAD~ Configuration Files; configuration files; define variables to configure the test
  81 #configf
  82 #configf `test.sh` reads config files from the following location if they are exist
  83 #configf * 'test.conf' from the current directory
  84 #configf * '$srcdir/test.conf' `$srcdir` is set by autotools
  85 #configf * '$TEST_CONF' a user defineable variable to point to a config file
  86 #configf
  87 test -f 'test.conf' && source test.conf
  88 test "$srcdir" -a -f '$srcdir/test.conf' && source "$srcdir/test.conf"
  89 test "$TEST_CONF" -a -f "$TEST_CONF" && source "$TEST_CONF"
  90
  91 arg0="$0"
  92 TESTDIR="$(dirname "$arg0")"
  93
  94
  95 #libtool HEAD Libtool; libtool; support for libtool
  96 #libtool When test.sh detects the presence of './libtool' it runs all tests with
  97 #libtool `./libtool --mode=execute`.
  98 #libtool
  99 LIBTOOL_EX=
 100 if test -x ./libtool; then
 101     LIBTOOL_EX="./libtool --mode=execute"
 102 fi
 103
 104 #valgrind HEAD- Valgrind; valgrind; valgrind support
 105 #valgrind Test are run under valgrind supervision by default, if not disabled.
 106 #valgrind
 107 #valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options
 108 #valgrind  VALGRINDFLAGS="--leak-check=yes --show-reachable=yes"
 109 #valgrind
 110 #valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override
 111 #valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable
 112 #valgrind valgrind for the tests.
 113 #valgrind
 114 #valgrind HEAD~ Generating Valgrind Suppression Files; vgsuppression; ignore false positives
 115 #valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then
 116 #valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress
 117 #valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known
 118 #valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does
 119 #valgrind not generate true positives.
 120 #valgrind
 121 ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288}
 122 valgrind=""
 123 LIMIT_TIME_REAL="$LIMIT_TIME"
 124 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 125     echo "valgrind explicit disabled"
 126 else
 127     if [ "$(which valgrind)" ]; then
 128         ulimit -S -t ${ULIMIT_VG_CPU:-20} -v ${ULIMIT_VG_VSZ:-524288}
 129         LIMIT_TIME_REAL="$LIMIT_VG_TIME"
 130         if [[ -x 'vgsuppression' ]]; then
 131             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
 132                 echo 'generating valgrind supression file'
 133
 134                 $LIBTOOL_EX $(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \
 135                     | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
 136             fi
 137             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q"
 138         else
 139             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
 140         fi
 141     else
 142         echo "no valgrind found, go without it"
 143     fi
 144 fi
 145
 146 echo
 147 echo "================ ${0##*/} ================"
 148
 149 TESTCNT=0
 150 SKIPCNT=0
 151 FAILCNT=0
 152
 153
 154 # the old testlog if existing will be used to check for previous test states
 155 if test -f ,testlog; then
 156     mv ,testlog ,testlog.pre
 157 else
 158     touch ,testlog.pre
 159 fi
 160
 161 date >,testlog
 162
 163 function compare_template() # template plainfile
 164 {
 165     local template
 166     local line
 167     local miss
 168     local lineno=1
 169     local templateno=1
 170     {
 171         IFS='' read -u 3 -r template || return 0
 172         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
 173         while true; do
 174             local cmd="${template%%:*}:"
 175             local arg="${template#*: }"
 176
 177             case $cmd in
 178             'regex_cont:')
 179                 if [[ $line =~ $arg ]]; then
 180                     IFS='' read -u 4 -r line ||
 181                     if IFS='' read -u 3 -r template; then
 182                         echo "premature end in output, expecting $template:$templateno"
 183                         return 1
 184                     else
 185                         return 0
 186                     fi
 187                     : $((++lineno))
 188                     miss=0
 189                 else
 190                     if [[ $((++miss)) -gt 1 ]]; then
 191                         echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 192                         return 1
 193                     fi
 194                     IFS='' read -u 3 -r template || { echo "more output than expected: '$line':$lineno"; return 1; }
 195                     : $((++templateno))
 196                 fi
 197                 ;;
 198             'literal:')
 199                 if [[ "$line" = "$arg" ]]; then
 200                     IFS='' read -u 3 -r template && IFS='' read -u 4 -r line || {
 201                         return 1
 202                     }
 203                 else
 204                     echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 205                     return 1
 206                 fi
 207                 ;;
 208             *)
 209                 echo "UNKOWN MATCH COMMAND '$cmd'" 1>&2
 210                 exit
 211                 ;;
 212             esac
 213         done
 214     } 3<"$1" 4<"$2"
 215 }
 216
 217 #tests HEAD- Writing Tests; tests; how to write testsuites
 218 #tests
 219 #tests
 220 #tests
 221 #tests
 222 #tests
 223 #tests
 224 #tests
 225 #tests
 226 #tests
 227 #tests
 228 #tests
 229 #tests
 230 #tests
 231 #tests
 232 #tests
 233 #tests
 234 #tests
 235 #tests
 236 #tests
 237 #tests
 238 function TESTING()
 239 {
 240     echo
 241     echo "$1"
 242     echo -e "\n#### $1" >>,testlog
 243
 244     TESTBIN="$2"
 245 }
 246
 247 #tests
 248 #tests
 249 #tests
 250 #tests
 251 #tests
 252 function TEST()
 253 {
 254         name="$1"
 255         shift
 256         rm -f ,send_stdin
 257         rm -f ,expect_stdout
 258         rm -f ,expect_stderr
 259         expect_return=0
 260
 261         local valgrind="$valgrind"
 262         if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 263             valgrind=
 264         fi
 265
 266         local condstack="1"
 267         while read -r line; do
 268             local cmd="${line%%:*}:"
 269             local arg="${line#*: }"
 270
 271             if [[ ! "$arg" ]]; then
 272                 arg='^$'
 273             fi
 274
 275             case $cmd in
 276             'if:')
 277                 if $arg; then
 278                     condstack="1$condstack"
 279                 else
 280                     condstack="0$condstack"
 281                 fi
 282                 ;;
 283             'else:')
 284                 if [[ "${condstack:0:1}" = "1" ]]; then
 285                     condstack="0${condstack:1}"
 286                 else
 287                     condstack="1${condstack:1}"
 288                 fi
 289                 ;;
 290             'elseif:')
 291                 if [[ "${condstack:0:1}" = "0" ]] && $arg; then
 292                     condstack="1${condstack:1}"
 293                 else
 294                     condstack="0${condstack:1}"
 295                 fi
 296                 ;;
 297             'endif:')
 298                 condstack="${condstack:1}"
 299                 ;;
 300             *)
 301                 if [[ "${condstack:0:1}" = "1" ]]; then
 302                     case $cmd in
 303                     'in:')
 304                         echo "$arg" >>,send_stdin
 305                         ;;
 306                     'out:')
 307                         echo "regex_cont: $arg" >>,expect_stdout
 308                         ;;
 309                     'err:')
 310                         echo "regex_cont: $arg" >>,expect_stderr
 311                         ;;
 312                     'out-lit:')
 313                         echo "literal: $arg" >>,expect_stdout
 314                         ;;
 315                     'err-lit:')
 316                         echo "literal: $arg" >>,expect_stderr
 317                         ;;
 318                     'return:')
 319                         expect_return="$arg"
 320                         ;;
 321                     '#'*|':')
 322                         :
 323                         ;;
 324                     *)
 325                         echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 326                         exit
 327                         ;;
 328                     esac
 329                 fi
 330                 ;;
 331             esac
 332         done
 333         echo -n "TEST $name: "
 334         echo -en "\nTEST $name: $* " >>,testlog
 335
 336         case "$TESTMODE" in
 337         *FAST*)
 338             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 339                 MSGOK=" (fixed)"
 340                 MSGFAIL=" (still broken)"
 341             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 342                 echo ".. SKIPPED (ok)"
 343                 echo ".. SKIPPED (ok)" >>,testlog
 344                 SKIPCNT=$(($SKIPCNT + 1))
 345                 TESTCNT=$(($TESTCNT + 1))
 346                 return
 347             else
 348                 MSGOK=" (new)"
 349                 MSGFAIL=" (new)"
 350             fi
 351             ;;
 352         *)
 353             MSGOK=""
 354             MSGFAIL=""
 355             ;;
 356         esac
 357
 358         TESTCNT=$(($TESTCNT + 1))
 359
 360         fails=0
 361
 362         echo -n >,testtmp
 363
 364         local CALL
 365         if declare -F | grep $TESTBIN >&/dev/null; then
 366             CALL=
 367         elif test -x $TESTBIN; then
 368             CALL="env $LIBTOOL_EX $valgrind"
 369         else
 370             CALL='-'
 371             echo -n >,stdout
 372             echo "test binary '$TESTBIN' not found" >,stderr
 373             ((fails+=1))
 374         fi
 375
 376         if test "$CALL" != '-'; then
 377             if test -f ,send_stdin; then
 378                 (
 379                     $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 380                     echo $?
 381                 ) >,return &
 382             else
 383                 (
 384                     $CALL $TESTBIN "$@" 2>,stderr >,stdout
 385                     echo $?
 386                 ) >,return &
 387             fi &>/dev/null
 388             pid=$!
 389
 390             # watchdog
 391             ( sleep $LIMIT_TIME_REAL && kill -KILL $pid ) &>/dev/null &
 392             wait $pid
 393             return=$(<,return)
 394
 395             if test -f ,expect_stdout; then
 396                 grep -v "$LOGSUPPRESS" <,stdout >,tmp
 397                 if ! compare_template ,expect_stdout ,tmp >>,cmptmp; then
 398                     echo "unexpected data on stdout" >>,testtmp
 399                     cat ,cmptmp >>,testtmp
 400                     ((fails+=1))
 401                 fi
 402                 rm ,tmp ,cmptmp
 403             fi
 404
 405             if test -f ,expect_stderr; then
 406                 grep -v "$LOGSUPPRESS" <,stderr >,tmp
 407                     cat ,tmp >>,testtmp
 408                 if ! compare_template ,expect_stderr ,tmp >>,cmptmp; then
 409                     echo "unexpected data on stderr" >>,testtmp
 410                     cat ,cmptmp >>,testtmp
 411                     ((fails+=1))
 412                 fi
 413                 rm ,tmp ,cmptmp
 414             fi
 415
 416             if [[ "${expect_return:0:1}" = '!' ]]; then
 417                 if [[ "${expect_return#\!}" = "$return" ]]; then
 418                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 419                     ((fails+=1))
 420                 fi
 421             else
 422                 if [[ "${expect_return}" != "$return" ]]; then
 423                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 424                     ((fails+=1))
 425                 fi
 426             fi
 427         fi
 428
 429         if test $fails -eq 0; then
 430             echo ".. OK$MSGOK"
 431             echo ".. OK$MSGOK" >>,testlog
 432         else
 433             echo ".. FAILED$MSGFAIL";
 434             echo ".. FAILED$MSGFAIL" >>,testlog
 435             cat ,testtmp >>,testlog
 436             rm ,testtmp
 437             echo "stderr was:" >>,testlog
 438             cat ,stderr >>,testlog
 439             echo END >>,testlog
 440             FAILCNT=$(($FAILCNT + 1))
 441             case $TESTMODE in
 442             *FIRSTFAIL*)
 443                 break 2
 444                 ;;
 445             esac
 446         fi
 447 }
 448
 449 function PLANNED()
 450 {
 451         echo -n "PLANNED $1: "
 452         echo -en "\nPLANNED $* " >>,testlog
 453         echo ".. SKIPPED (planned)"
 454         echo ".. SKIPPED (planned)" >>,testlog
 455         SKIPCNT=$(($SKIPCNT + 1))
 456         TESTCNT=$(($TESTCNT + 1))
 457 }
 458
 459 function RUNTESTS()
 460 {
 461     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 462         TESTSUITES="{$TESTSUITES}"
 463     fi
 464     for t in $(eval echo "$TESTDIR/*$TESTSUITES*.tests"); do
 465         echo "$t"
 466     done | sort | uniq | {
 467         while read i; do
 468             echo
 469             echo "### $i" >&2
 470             if test -f $i; then
 471                 source $i
 472             fi
 473         done
 474         echo
 475         if [ $FAILCNT = 0 ]; then
 476             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 477             #rm ,testlog
 478         else
 479             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 480             echo " ... FAILED $FAILCNT TESTS"
 481             echo " ... SKIPPED $SKIPCNT TESTS"
 482             echo " see ',testlog' for details"
 483             exit 1
 484         fi
 485     }
 486 }
 487
 488 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 489
 490 RUNTESTS