validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
  10 prog_name=`basename $0`
  11
  12 if [ ! -x "$default_path/sparse-llvm" ]; then
  13         disabled_cmds="sparsec sparsei sparse-llvm"
  14 fi
  15
  16 # flags:
  17 #       - some tests gave an unexpected result
  18 failed=0
  19
  20 # counts:
  21 #       - tests that have not been converted to test-suite format
  22 #       - tests that are disabled
  23 #       - tests that passed
  24 #       - tests that failed
  25 #       - tests that failed but are known to fail
  26 unhandled_tests=0
  27 disabled_tests=0
  28 ok_tests=0
  29 ko_tests=0
  30 known_ko_tests=0
  31
  32 # defaults to not verbose
  33 [ -z "$V" ] && V=0
  34 [ $V -eq 0 ] && quiet=1 || quiet=0
  35
  36 ##
  37 # get_tag_value(file) - get the 'check-<...>' tags & values
  38 get_tag_value()
  39 {
  40         check_name=""
  41         check_command="$default_cmd"
  42         check_exit_value=0
  43         check_timeout=0
  44         check_known_to_fail=0
  45         check_error_ignore=0
  46         check_output_ignore=0
  47         check_output_contains=0
  48         check_output_excludes=0
  49         check_output_pattern=0
  50
  51         lines=$(grep 'check-[a-z-]*' $1 | \
  52                 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
  53
  54         while read tag val; do
  55                 #echo "-> tag: '$tag'"
  56                 #echo "-> val: '$val'"
  57                 case $tag in
  58                 check-name:)            check_name="$val" ;;
  59                 check-command:)         check_command="$val" ;;
  60                 check-exit-value:)      check_exit_value="$val" ;;
  61                 check-timeout:)         [ -z "$val" ] && val=1
  62                                         check_timeout="$val" ;;
  63                 check-known-to-fail)    check_known_to_fail=1 ;;
  64                 check-error-ignore)     check_error_ignore=1 ;;
  65                 check-output-ignore)    check_output_ignore=1 ;;
  66                 check-output-contains:) check_output_contains=1 ;;
  67                 check-output-excludes:) check_output_excludes=1 ;;
  68                 check-output-pattern-)  check_output_pattern=1 ;;
  69                 esac
  70         done << EOT
  71         $lines
  72 EOT
  73 }
  74
  75 ##
  76 # helper for has_(each|none)_patterns()
  77 has_patterns()
  78 {
  79         ifile="$1"
  80         patt="$2"
  81         ofile="$3"
  82         cmp="$4"
  83         grep "$patt:" "$ifile" | \
  84         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
  85         while read val; do
  86                 grep -s -q "$val" "$ofile"
  87                 if [ "$?" $cmp 0 ]; then
  88                         return 1
  89                 fi
  90         done
  91
  92         return $?
  93 }
  94
  95 ##
  96 # has_each_patterns(ifile tag ofile) - does ofile contains some
  97 #                        of the patterns given by ifile's tags?
  98 #
  99 # returns 0 if all present, 1 otherwise
 100 has_each_patterns()
 101 {
 102         has_patterns "$1" "$2" "$3" -ne
 103 }
 104
 105 ##
 106 # has_none_patterns(ifile tag ofile) - does ofile contains some
 107 #                        of the patterns given by ifile's tags?
 108 #
 109 # returns 1 if any present, 0 otherwise
 110 has_none_patterns()
 111 {
 112         has_patterns "$1" "$2" "$3" -eq
 113 }
 114
 115 ##
 116 # nbr_patterns(ifile tag ofile) - does ofile contains the
 117 #                        the patterns given by ifile's tags
 118 #                        the right number of time?
 119 nbr_patterns()
 120 {
 121         ifile="$1"
 122         patt="$2"
 123         ofile="$3"
 124         grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
 125         sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
 126         while read nbr pat; do
 127                 n=$(grep -s "$pat" "$ofile" | wc -l)
 128                 if [ "$n" -ne "$nbr" ]; then
 129                         return 1
 130                 fi
 131         done
 132
 133         return $?
 134 }
 135
 136 ##
 137 # verbose(string) - prints string if we are in verbose mode
 138 verbose()
 139 {
 140         [ "$V" -eq "1" ] && echo "        $1"
 141         return 0
 142 }
 143
 144 ##
 145 # error(string[, die]) - prints an error and exits with value die if given
 146 error()
 147 {
 148         [ "$quiet" -ne 1 ] && echo "error: $1"
 149         [ -n "$2" ] && exit $2
 150         return 0
 151 }
 152
 153 do_usage()
 154 {
 155 echo "$prog_name - a tiny automatic testing script"
 156 echo "Usage: $prog_name [command] [command arguments]"
 157 echo
 158 echo "commands:"
 159 echo "    none                       runs the whole test suite"
 160 echo "    single file                runs the test in 'file'"
 161 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
 162 echo
 163 echo "    help                       prints usage"
 164 }
 165
 166 ##
 167 # do_test(file) - tries to validate a test case
 168 #
 169 # it "parses" file, looking for check-* tags and tries to validate
 170 # the test against an expected result
 171 # returns:
 172 #       - 0 if the test passed,
 173 #       - 1 if it failed,
 174 #       - 2 if it is not a "test-suite" test.
 175 #       - 3 if the test is disabled.
 176 do_test()
 177 {
 178         test_failed=0
 179         file="$1"
 180
 181         get_tag_value $file
 182
 183         # can this test be handled by test-suite ?
 184         # (it has to have a check-name key in it)
 185         if [ "$check_name" = "" ]; then
 186                 echo "warning: test '$file' unhandled"
 187                 unhandled_tests=$(($unhandled_tests + 1))
 188                 return 2
 189         fi
 190         test_name="$check_name"
 191
 192         # does the test provide a specific command ?
 193         if [ "$check_command" = "" ]; then
 194                 check_command="$defaut_command"
 195         fi
 196
 197         # check for disabled commands
 198         set -- $check_command
 199         base_cmd=$1
 200         for i in $disabled_cmds; do
 201                 if [ "$i" = "$base_cmd" ] ; then
 202                         disabled_tests=$(($disabled_tests + 1))
 203                         echo "     DISABLE $test_name ($file)"
 204                         return 3
 205                 fi
 206         done
 207
 208         cmd=`eval echo $default_path/$check_command`
 209
 210         echo "     TEST    $test_name ($file)"
 211
 212         verbose "Using command       : $cmd"
 213
 214         # grab the expected exit value
 215         expected_exit_value=$check_exit_value
 216         verbose "Expecting exit value: $expected_exit_value"
 217
 218         # do we want a timeout?
 219         if [ $check_timeout -ne 0 ]; then
 220                 cmd="timeout -k 1s $check_timeout $cmd"
 221         fi
 222
 223         # grab the actual output & exit value
 224         $cmd 1> $file.output.got 2> $file.error.got
 225         actual_exit_value=$?
 226
 227         must_fail=$check_known_to_fail
 228         quiet=0
 229         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 230         known_ko_tests=$(($known_ko_tests + $must_fail))
 231
 232         for stream in output error; do
 233                 eval ignore=\$check_${stream}_ignore
 234                 [ $ignore -eq 1 ] && continue
 235
 236                 # grab the expected output
 237                 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
 238                 | grep -v check-$stream > "$file".$stream.expected
 239
 240                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 241                 if [ "$?" -ne "0" ]; then
 242                         error "actual $stream text does not match expected $stream text."
 243                         error  "see $file.$stream.* for further investigation."
 244                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 245                         test_failed=1
 246                 fi
 247         done
 248
 249         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 250                 error "Actual exit value does not match the expected one."
 251                 error "expected $expected_exit_value, got $actual_exit_value."
 252                 test_failed=1
 253         fi
 254
 255         # verify the 'check-output-contains/excludes' tags
 256         if [ $check_output_contains -eq 1 ]; then
 257                 has_each_patterns "$file" 'check-output-contains' $file.output.got
 258                 if [ "$?" -ne "0" ]; then
 259                         error "Actual output doesn't contain some of the expected patterns."
 260                         test_failed=1
 261                 fi
 262         fi
 263         if [ $check_output_excludes -eq 1 ]; then
 264                 has_none_patterns "$file" 'check-output-excludes' $file.output.got
 265                 if [ "$?" -ne "0" ]; then
 266                         error "Actual output contains some patterns which are not expected."
 267                         test_failed=1
 268                 fi
 269         fi
 270         if [ $check_output_pattern -eq 1 ]; then
 271                 # verify the 'check-output-pattern-X-times' tags
 272                 nbr_patterns "$file" 'check-output-pattern' $file.output.got
 273                 if [ "$?" -ne "0" ]; then
 274                         error "Actual output doesn't contain the pattern the expected number."
 275                         test_failed=1
 276                 fi
 277         fi
 278
 279         [ "$test_failed" -eq "$must_fail" ] || failed=1
 280
 281         if [ "$must_fail" -eq "1" ]; then
 282                 if [ "$test_failed" -eq "1" ]; then
 283                         echo "info: test '$file' is known to fail"
 284                 else
 285                         echo "error: test '$file' is known to fail but succeed!"
 286                         test_failed=1
 287                 fi
 288         fi
 289
 290         if [ "$test_failed" -eq "1" ]; then
 291                 ko_tests=$(($ko_tests + 1))
 292         else
 293                 ok_tests=$(($ok_tests + 1))
 294                 rm -f $file.{error,output}.{expected,got,diff}
 295         fi
 296         return $test_failed
 297 }
 298
 299 do_test_suite()
 300 {
 301         for i in $tests_list; do
 302                 do_test "$i"
 303         done
 304
 305         # prints some numbers
 306         tests_nr=$(($ok_tests + $ko_tests))
 307         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 308         echo " ($known_ko_tests of them are known to fail)"
 309         if [ "$unhandled_tests" -ne "0" ]; then
 310                 echo "$unhandled_tests tests could not be handled by $prog_name"
 311         fi
 312         if [ "$disabled_tests" -ne "0" ]; then
 313                 echo "$disabled_tests tests were disabled"
 314         fi
 315 }
 316
 317 ##
 318 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 319 do_format()
 320 {
 321         if [ -z "$2" ]; then
 322                 fname="$1"
 323                 fcmd=$default_cmd
 324         elif [ -z "$3" ]; then
 325                 fname="$2"
 326                 fcmd=$default_cmd
 327         else
 328                 fname="$2"
 329                 fcmd="$3"
 330         fi
 331         file="$1"
 332         cmd=`eval echo $default_path/$fcmd`
 333         $cmd 1> $file.output.got 2> $file.error.got
 334         fexit_value=$?
 335         cat <<_EOF
 336 /*
 337  * check-name: $fname
 338 _EOF
 339         if [ "$fcmd" != "$default_cmd" ]; then
 340                 echo " * check-command: $fcmd"
 341         fi
 342         if [ "$fexit_value" -ne "0" ]; then
 343                 echo " * check-exit-value: $fexit_value"
 344         fi
 345         for stream in output error; do
 346                 if [ -s "$file.$stream.got" ]; then
 347                         echo " *"
 348                         echo " * check-$stream-start"
 349                         cat "$file.$stream.got"
 350                         echo " * check-$stream-end"
 351                 fi
 352         done
 353         echo " */"
 354         return 0
 355 }
 356
 357 ##
 358 # arg_file(filename) - checks if filename exists
 359 arg_file()
 360 {
 361         [ -z "$1" ] && {
 362                 do_usage
 363                 exit 1
 364         }
 365         [ -e "$1" ] || {
 366                 error "Can't open file $1"
 367                 exit 1
 368         }
 369         return 0
 370 }
 371
 372 case "$1" in
 373         '')
 374                 do_test_suite
 375                 ;;
 376         single)
 377                 arg_file "$2"
 378                 do_test "$2"
 379                 case "$?" in
 380                         0) echo "$2 passed !";;
 381                         1) echo "$2 failed !";;
 382                         2) echo "$2 can't be handled by $prog_name";;
 383                 esac
 384                 ;;
 385         format)
 386                 arg_file "$2"
 387                 do_format "$2" "$3" "$4"
 388                 ;;
 389         help | *)
 390                 do_usage
 391                 exit 1
 392                 ;;
 393 esac
 394
 395 exit $failed
 396