tests/test.sh

   1 #!/usr/bin/env bash
   2 #  Copyright (C)              Lumiera.org
   3 #    2007, 2008, 2009, 2010,  Christian Thaeter <ct@pipapo.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License as
   7 #  published by the Free Software Foundation; either version 2 of the
   8 #  License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18
  19 # TESTMODE=FULL yet unimplemented
  20 #   run all tests, PLANNED which fail count as error
  21 #
  22 # TESTMODE=FAST
  23 #   run only tests which recently failed
  24 #
  25 # TESTMODE=FIRSTFAIL
  26 #   stop testing on the first failure
  27
  28 #intro Test.sh
  29 #intro =======
  30 #intro Christian Thäter
  31 #intro
  32 #intro A shell script driving software tests.
  33 #intro
  34
  35 #=intro
  36 #=tests Writing tests
  37 # =run
  38 #=config
  39 #=configf
  40 # =make
  41 # =control
  42 #=valgrind
  43 #=libtool
  44 #_
  45 #_ Index
  46 #_ -----
  47 #_
  48 #=index
  49 #_
  50
  51 LC_ALL=C
  52
  53 #config HEAD- Configuration; configuration; configure tests
  54 #config
  55 #config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr
  56 #config  LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  57 #config
  58 #config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for
  59 #config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out.
  60 #config The default as shown above filters some NoBug annotations and non fatal logging out.
  61 #config
  62 LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}[:!] \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):'
  63
  64 #config PARA Resource Limits; ulimit; constrain resource limits
  65 #config It is possible to set some limits for tests to protect the system against really broken cases.
  66 #config Since running under valgrind takes consinderable more resources there are separate variants for
  67 #config limits when running under valgrind.
  68 #config
  69 #config  LIMIT_CPU=5
  70 #config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks.
  71 #config
  72 #config  LIMIT_TIME=10
  73 #config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks.
  74 #config
  75 #config  LIMIT_VSZ=524288
  76 #config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached.
  77 #config Protects against memory leaks.
  78 #config
  79 #config  LIMIT_VG_CPU=20
  80 #config  LIMIT_VG_TIME=30
  81 #config  LIMIT_VG_VSZ=524288
  82 #config Same variables again with limits when running under valgrind.
  83 #config
  84 LIMIT_CPU=5
  85 LIMIT_TIME=10
  86 LIMIT_VSZ=524288
  87 LIMIT_VG_CPU=20
  88 LIMIT_VG_TIME=30
  89 LIMIT_VG_VSZ=524288
  90
  91
  92 #configf HEAD~ Configuration Files; configuration files; define variables to configure the test
  93 #configf
  94 #configf `test.sh` reads config files from the following location if they are exist
  95 #configf * 'test.conf' from the current directory
  96 #configf * '$srcdir/test.conf' `$srcdir` is set by autotools
  97 #configf * '$srcdir/tests/test.conf' `tests/` is suspected as default directory for tests
  98 #configf * '$TEST_CONF' a user defineable variable to point to a config file
  99 #configf
 100 test -f 'test.conf' && source test.conf
 101 test -n "$srcdir" -a -e "$srcdir/test.conf" && source "$srcdir/test.conf"
 102 test -n "$srcdir" -a -e "$srcdir/tests/test.conf" && source "$srcdir/tests/test.conf"
 103 test -n "$TEST_CONF" -a -e "$TEST_CONF" && source "$TEST_CONF"
 104
 105
 106
 107 arg0="$0"
 108 TESTDIR="$(dirname "$arg0")"
 109
 110
 111 #libtool HEAD Libtool; libtool; support for libtool
 112 #libtool When test.sh detects the presence of './libtool' it runs all tests with
 113 #libtool `./libtool --mode=execute`.
 114 #libtool
 115 LIBTOOL_EX=
 116 if test -x ./libtool; then
 117     LIBTOOL_EX="./libtool --mode=execute"
 118 fi
 119
 120 #valgrind HEAD- Valgrind; valgrind; valgrind support
 121 #valgrind Test are run under valgrind supervision by default, if not disabled.
 122 #valgrind
 123 #valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options
 124 #valgrind  VALGRINDFLAGS="--leak-check=yes --show-reachable=yes"
 125 #valgrind
 126 #valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override
 127 #valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable
 128 #valgrind valgrind for the tests.
 129 #valgrind
 130 #valgrind HEAD~ Generating Valgrind Suppression Files; vgsuppression; ignore false positives
 131 #valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then
 132 #valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress
 133 #valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known
 134 #valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does
 135 #valgrind not generate true positives.
 136 #valgrind
 137 ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288}
 138 valgrind=""
 139 LIMIT_TIME_REAL="$LIMIT_TIME"
 140 if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 141     echo "valgrind explicit disabled"
 142 else
 143     if [ "$(which valgrind)" ]; then
 144         ulimit -S -t ${ULIMIT_VG_CPU:-20} -v ${ULIMIT_VG_VSZ:-524288}
 145         LIMIT_TIME_REAL="$LIMIT_VG_TIME"
 146         if [[ -x 'vgsuppression' ]]; then
 147             if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then
 148                 echo 'generating valgrind supression file'
 149
 150                 $LIBTOOL_EX $(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \
 151                     | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp
 152             fi
 153             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q"
 154         else
 155             valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}"
 156         fi
 157     else
 158         echo "no valgrind found, go without it"
 159     fi
 160 fi
 161
 162 echo
 163 echo "================ ${0##*/} ================"
 164
 165 TESTCNT=0
 166 SKIPCNT=0
 167 FAILCNT=0
 168
 169
 170 # the old testlog if existing will be used to check for previous test states
 171 if test -f ,testlog; then
 172     mv ,testlog ,testlog.pre
 173 else
 174     touch ,testlog.pre
 175 fi
 176
 177 date >,testlog
 178
 179 function compare_template() # template plainfile
 180 {
 181     local template
 182     local line
 183     local miss
 184     local lineno=1
 185     local templateno=1
 186     {
 187         IFS='' read -u 3 -r template || return 0
 188         IFS='' read -u 4 -r line || { echo "no output"; return 1; }
 189         while true; do
 190             local cmd="${template%%:*}:"
 191             local arg="${template#*: }"
 192
 193             case $cmd in
 194             'regex_cont:')
 195                 if [[ $line =~ $arg ]]; then
 196                     IFS='' read -u 4 -r line ||
 197                     if IFS='' read -u 3 -r template; then
 198                         echo "premature end in output, expecting $template:$templateno"
 199                         return 1
 200                     else
 201                         return 0
 202                     fi
 203                     : $((++lineno))
 204                     miss=0
 205                 else
 206                     if [[ $((++miss)) -gt 1 ]]; then
 207                         echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 208                         return 1
 209                     fi
 210                     IFS='' read -u 3 -r template || { echo "more output than expected: '$line':$lineno"; return 1; }
 211                     : $((++templateno))
 212                 fi
 213                 ;;
 214             'literal:')
 215                 if [[ "$line" = "$arg" ]]; then
 216                     IFS='' read -u 3 -r template && IFS='' read -u 4 -r line || {
 217                         return 1
 218                     }
 219                 else
 220                     echo -e "'$line':$lineno\ndoes not match\n$template:$templateno"
 221                     return 1
 222                 fi
 223                 ;;
 224             *)
 225                 echo "UNKOWN MATCH COMMAND '$cmd'" 1>&2
 226                 exit
 227                 ;;
 228             esac
 229         done
 230     } 3<"$1" 4<"$2"
 231 }
 232
 233 #tests HEAD- Writing Tests; tests; how to write testsuites
 234 #tests Tests are nothing more than bash scripts with some functions from the test.sh
 235 #tests framework defined. Test.sh looks in the current directory for all files which ending in .test
 236 #tests and runs them in alphabetical order. The selection of this tests can be constrained with the
 237 #tests `TESTSUITES` environment variable.
 238 #tests
 239 #tests HEAD~ Testsuites; test files; writing tests
 240 #tests It is common to start the name of the '.test' files with a 2 digi number to give them a proper
 241 #tests order: '10foo.test', '20bar.test' and so on. Each such test should only test a certain aspect of
 242 #tests the system. You have to select the testing binary with the `TESTING` function and then write
 243 #tests certain TEST's defining how the test should react. Since tests are shell scripts it is possible
 244 #tests to add some supplemental commands there to set and clean up the given test environment.
 245 #tests
 246 #tests HEAD^ TESTING; TESTING; set the test binary
 247 #tests  TESTING "message" test_program
 248 #tests
 249 #tests Selects the test binary for the follwing tests, prints an informal message.
 250 #tests
 251 #tests  `message`::
 252 #tests    message to be printed
 253 #tests  `test_program`::
 254 #tests    an existing program to drive the tests or a shell function
 255 #tests
 256 #tests ----
 257 #tests TESTING "Testing a.out" ./a.out
 258 #tests ----
 259 #tests
 260 function TESTING()
 261 {
 262     echo
 263     echo "$1"
 264     echo -e "\n#### $1, $TESTFILE, $2" >>,testlog
 265
 266     TESTBIN="$2"
 267 }
 268
 269 #tests HEAD^ TEST; TEST; single test
 270 #tests  TEST "title" arguments.. <<END
 271 #tests
 272 #tests Runs a single test
 273 #tests
 274 #tests  `title`::
 275 #tests    describes this test and is also used as identifier for this test,
 276 #tests    must be unique for all your tests
 277 #tests  `arguments`::
 278 #tests    the following arguments are passed to the test program
 279 #tests  `<<END .. END`::
 280 #tests    a list of control commands expected in and outputs is given as 'heredoc'.
 281 #tests
 282 #tests Each line of the test specification in the heredoc starts with an arbitary number of spaces
 283 #tests followed by a command, followed by a colon and a space, followed by additional arguments or
 284 #tests being an empty or comment line.
 285 #tests
 286 #tests
 287 #tests HEAD+ Test Commands; commands; define expected in and outputs
 288 #tests
 289 #tests PARA in; in; stdin data for a test
 290 #tests  in: text
 291 #tests
 292 #tests Send `text` to stdin of the test binary. If no `in:` commands are given, nothing is send to the
 293 #tests tests input.
 294 #tests
 295 #tests PARA out; out; expected stdout (regex) from a test
 296 #tests  out: regex
 297 #tests
 298 #tests Expect `regex` on stdout. This regexes have a 'triggering' semantic. That means it is tried to match
 299 #tests a given regex on as much lines as possible (`.*` will match any remaining output), if the match fails,
 300 #tests the next expected output line is tried. When that fails too the test is aborted and counted as failure.
 301 #tests
 302 #tests When no `out:` or `out-lit:` commands are given, then stdout is not checked, any output is ignored.
 303 #tests
 304 #tests PARA err; err; expected stderr (regex) from a test
 305 #tests  out: regex
 306 #tests
 307 #tests Same as 'out:' but expects data on stderr. When no `err:` or `err-lit:` commands are given, then stdout is
 308 #tests not checked, any output there is ignored.
 309 #tests
 310 #tests PARA out-lit; out-lit; expected stdout (literal) from a test
 311 #tests  out-lit: text
 312 #tests
 313 #tests Expect `text` on stdout, must match exactly or will fail.
 314 #tests
 315 #tests PARA err-lit; err-lit; expected stderr (literal) from a test
 316 #tests  err-lit: text
 317 #tests
 318 #tests Same as 'out-lit:' but expects data on stderr.
 319 #tests
 320 #tests PARA return; return; expected exit value of a test
 321 #tests  return: value
 322 #tests
 323 #tests Expects `value` as exit code of the tested program. The check can be negated by prepending the value with
 324 #tests an exclamation mark, `return: !0` expects any exist code except zero.
 325 #tests
 326 #tests If no `return:` command is given then a zero (success) return from the test program is expected.
 327 #tests
 328 #tests HEAD+ Conditional Tests; conditional tests; switch tests on conditions
 329 #tests Sometimes tests need to be adapted to the environment/platform they are running on. This can be archived
 330 #tests with common if-else-elseif-endif statements. This statements can be nested.
 331 #tests
 332 #tests PARA if; if; conditional test
 333 #tests  if: check
 334 #tests
 335 #tests Executes `check` as shell command, if its return is zero (success) then the following test parts are used.
 336 #tests
 337 #tests PARA else; else; conditional alternative
 338 #tests  else:
 339 #tests
 340 #tests If the previous `if` failed then the following test parts are included in the test, otherwise they
 341 #tests are excluded.
 342 #tests
 343 #tests PARA elseif; elseif; conditional alternative with test
 344 #tests  elseif: check
 345 #tests
 346 #tests Composition of else and if, only includes the following test parts if the if's and elseif's before failed
 347 #tests and `check` succeeded.
 348 #tests
 349 #tests PARA endif; endif; end of conditonal test part
 350 #tests  endif:
 351 #tests
 352 #tests Ends an `if` statement.
 353 #tests
 354 #tests HEAD+ Other Elements;;
 355 #tests
 356 #tests PARA msg; msg; print a diagnostic message
 357 #tests  msg: message..
 358 #tests
 359 #tests Prints `message` while processing the test suite.
 360 #tests
 361 #tests PARA comments; comments; adding comments to tests
 362 #tests  # anything
 363 #tests
 364 #tests Lines starting with the hash mark and empty lines count as comment and are not used.
 365 #tests
 366 function TEST()
 367 {
 368         name="$1"
 369         shift
 370         rm -f ,send_stdin
 371         rm -f ,expect_stdout
 372         rm -f ,expect_stderr
 373         expect_return=0
 374
 375         local valgrind="$valgrind"
 376         if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
 377             valgrind=
 378         fi
 379
 380         local condstack="1"
 381         while read -r line; do
 382             local cmd="${line%%:*}:"
 383             local arg="${line#*: }"
 384
 385             if [[ ! "$arg" ]]; then
 386                 arg='^$'
 387             fi
 388
 389             case $cmd in
 390             'if:')
 391                 if $arg; then
 392                     condstack="1$condstack"
 393                 else
 394                     condstack="0$condstack"
 395                 fi
 396                 ;;
 397             'elseif:')
 398                 if [[ "${condstack:0:1}" = "0" ]]; then
 399                     if $arg; then
 400                         condstack="1${condstack:1}"
 401                     else
 402                         condstack="0${condstack:1}"
 403                     fi
 404                 else
 405                     condstack="2${condstack:1}"
 406                 fi
 407                 ;;
 408             'else:')
 409                 if [[ "${condstack:0:1}" != "0" ]]; then
 410                     condstack="0${condstack:1}"
 411                 else
 412                     condstack="1${condstack:1}"
 413                 fi
 414                 ;;
 415             'endif:')
 416                 condstack="${condstack:1}"
 417                 ;;
 418             *)
 419                 if [[ "${condstack:0:1}" = "1" ]]; then
 420                     case $cmd in
 421                     'msg:')
 422                         echo "MSG $arg"
 423                         ;;
 424                     'in:')
 425                         echo "$arg" >>,send_stdin
 426                         ;;
 427                     'out:')
 428                         echo "regex_cont: $arg" >>,expect_stdout
 429                         ;;
 430                     'err:')
 431                         echo "regex_cont: $arg" >>,expect_stderr
 432                         ;;
 433                     'out-lit:')
 434                         echo "literal: $arg" >>,expect_stdout
 435                         ;;
 436                     'err-lit:')
 437                         echo "literal: $arg" >>,expect_stderr
 438                         ;;
 439                     'return:')
 440                         expect_return="$arg"
 441                         ;;
 442                     '#'*|':')
 443                         :
 444                         ;;
 445                     *)
 446                         echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
 447                         exit
 448                         ;;
 449                     esac
 450                 fi
 451                 ;;
 452             esac
 453         done
 454         echo -n "TEST $name: "
 455         echo -en "\nTEST $name: $* " >>,testlog
 456
 457         case "$TESTMODE" in
 458         *FAST*)
 459             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
 460                 MSGOK=" (fixed)"
 461                 MSGFAIL=" (still broken)"
 462             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
 463                 echo ".. SKIPPED (ok)"
 464                 echo ".. SKIPPED (ok)" >>,testlog
 465                 SKIPCNT=$(($SKIPCNT + 1))
 466                 TESTCNT=$(($TESTCNT + 1))
 467                 return
 468             else
 469                 MSGOK=" (new)"
 470                 MSGFAIL=" (new)"
 471             fi
 472             ;;
 473         *)
 474             MSGOK=""
 475             MSGFAIL=""
 476             ;;
 477         esac
 478
 479         TESTCNT=$(($TESTCNT + 1))
 480
 481         fails=0
 482
 483         echo -n >,testtmp
 484
 485         local CALL
 486         if declare -F | grep $TESTBIN >&/dev/null; then
 487             CALL=
 488         elif test -x $TESTBIN; then
 489             CALL="env $LIBTOOL_EX $valgrind"
 490         else
 491             CALL='-'
 492             echo -n >,stdout
 493             echo "test binary '$TESTBIN' not found" >,stderr
 494             ((fails+=1))
 495         fi
 496
 497         if test "$CALL" != '-'; then
 498             if test -f ,send_stdin; then
 499                 (
 500                     $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout
 501                     echo $? >,return
 502                 ) &
 503             else
 504                 (
 505                     $CALL $TESTBIN "$@" 2>,stderr >,stdout
 506                     echo $? >,return
 507                 ) &
 508             fi &>/dev/null
 509             pid=$!
 510
 511             # watchdog
 512             ( sleep $LIMIT_TIME_REAL && kill -KILL $pid ) &>/dev/null &
 513             wpid=$!
 514             wait $pid
 515             return=$(<,return)
 516             if [[ "$return" -le 128 ]]; then
 517                 kill -INT $wpid >&/dev/null
 518             fi
 519
 520             if test -f ,expect_stdout; then
 521                 grep -v "$LOGSUPPRESS" <,stdout >,tmp
 522                 if ! compare_template ,expect_stdout ,tmp >>,cmptmp; then
 523                     echo "unexpected data on stdout" >>,testtmp
 524                     cat ,cmptmp >>,testtmp
 525                     ((fails+=1))
 526                 fi
 527                 rm ,tmp ,cmptmp
 528             fi
 529
 530             if test -f ,expect_stderr; then
 531                 grep -v "$LOGSUPPRESS" <,stderr >,tmp
 532                     cat ,tmp >>,testtmp
 533                 if ! compare_template ,expect_stderr ,tmp >>,cmptmp; then
 534                     echo "unexpected data on stderr" >>,testtmp
 535                     cat ,cmptmp >>,testtmp
 536                     ((fails+=1))
 537                 fi
 538                 rm ,tmp ,cmptmp
 539             fi
 540
 541             if [[ "${expect_return:0:1}" = '!' ]]; then
 542                 if [[ "${expect_return#\!}" = "$return" ]]; then
 543                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 544                     ((fails+=1))
 545                 fi
 546             else
 547                 if [[ "${expect_return}" != "$return" ]]; then
 548                     echo "unexpected return value $return, expected $expect_return" >>,testtmp
 549                     ((fails+=1))
 550                 fi
 551             fi
 552         fi
 553
 554         if test $fails -eq 0; then
 555             echo ".. OK$MSGOK"
 556             echo ".. OK$MSGOK" >>,testlog
 557         else
 558             echo ".. FAILED$MSGFAIL";
 559             echo ".. FAILED$MSGFAIL" >>,testlog
 560             cat ,testtmp >>,testlog
 561             rm ,testtmp
 562             echo "stderr was:" >>,testlog
 563             cat ,stderr >>,testlog
 564             echo END >>,testlog
 565             FAILCNT=$(($FAILCNT + 1))
 566             case $TESTMODE in
 567             *FIRSTFAIL*)
 568                 break 2
 569                 ;;
 570             esac
 571         fi
 572 }
 573
 574 #tests HEAD^ PLANNED; PLANNED; deactivated test
 575 #tests  PLANNED "title" arguments.. <<END
 576 #tests
 577 #tests Skip a single test.
 578 #tests
 579 #tests  `title`::
 580 #tests    describes this test and is also used as identifier for this test,
 581 #tests    must be unique for all your tests
 582 #tests  `arguments`::
 583 #tests    the following arguments are passed to the test program
 584 #tests  `<<END .. END`::
 585 #tests    a list of control commands expected in and outputs is given as 'heredoc'.
 586 #tests
 587 #tests `PLANNED` acts as dropin replacement for `TEST`. Each such test is skipped (and counted as skipped)
 588 #tests This can be used to specify tests in advance and activate them as soon development goes on or
 589 #tests deactivate intentional broken tests to be fixed later.
 590 #tests
 591 function PLANNED()
 592 {
 593         echo -n "PLANNED $1: "
 594         echo -en "\nPLANNED $* " >>,testlog
 595         echo ".. SKIPPED (planned)"
 596         echo ".. SKIPPED (planned)" >>,testlog
 597         SKIPCNT=$(($SKIPCNT + 1))
 598         TESTCNT=$(($TESTCNT + 1))
 599 }
 600
 601 function RUNTESTS()
 602 {
 603     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
 604         TESTSUITES="{$TESTSUITES}"
 605     fi
 606     for t in $(eval echo "$TESTDIR/*$TESTSUITES*.tests"); do
 607         echo "$t"
 608     done | sort | uniq | {
 609         while read TESTFILE; do
 610             echo
 611             echo "### $TESTFILE" >&2
 612             if test -f $TESTFILE; then
 613                 source $TESTFILE
 614             fi
 615         done
 616         echo
 617         if [ $FAILCNT = 0 ]; then
 618             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
 619             #rm ,testlog
 620         else
 621             echo " ... SUCCEEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
 622             echo " ... FAILED $FAILCNT TESTS"
 623             echo " ... SKIPPED $SKIPCNT TESTS"
 624             echo " see ',testlog' for details"
 625             exit 1
 626         fi
 627     }
 628 }
 629
 630 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
 631
 632 RUNTESTS