validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
  10 prog_name=`basename $0`
  11
  12 if [ ! -x "$default_path/sparse-llvm" ]; then
  13         disabled_cmds="sparsec sparsei sparse-llvm"
  14 fi
  15
  16 # flags:
  17 #       - some tests gave an unexpected result
  18 failed=0
  19
  20 # counts:
  21 #       - tests that have not been converted to test-suite format
  22 #       - tests that are disabled
  23 #       - tests that passed
  24 #       - tests that failed
  25 #       - tests that failed but are known to fail
  26 unhandled_tests=0
  27 disabled_tests=0
  28 ok_tests=0
  29 ko_tests=0
  30 known_ko_tests=0
  31
  32 # defaults to not verbose
  33 [ -z "$V" ] && V=0
  34
  35 ##
  36 # get_tag_value(file) - get the 'check-<...>' tags & values
  37 get_tag_value()
  38 {
  39         check_name=""
  40         check_command="$default_cmd"
  41         check_exit_value=0
  42         check_known_to_fail=0
  43         check_error_ignore=0
  44         check_output_ignore=0
  45         check_output_contains=0
  46         check_output_excludes=0
  47         check_output_pattern=0
  48
  49         lines=$(grep 'check-[a-z-]*' $1 | \
  50                 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
  51
  52         while read tag val; do
  53                 #echo "-> tag: '$tag'"
  54                 #echo "-> val: '$val'"
  55                 case $tag in
  56                 check-name:)            check_name="$val" ;;
  57                 check-command:)         check_command="$val" ;;
  58                 check-exit-value:)      check_exit_value="$val" ;;
  59                 check-known-to-fail)    check_known_to_fail=1 ;;
  60                 check-error-ignore)     check_error_ignore=1 ;;
  61                 check-output-ignore)    check_output_ignore=1 ;;
  62                 check-output-contains:) check_output_contains=1 ;;
  63                 check-output-excludes:) check_output_excludes=1 ;;
  64                 check-output-pattern-)  check_output_pattern=1 ;;
  65                 esac
  66         done << EOT
  67         $lines
  68 EOT
  69 }
  70
  71 ##
  72 # helper for has_(each|none)_patterns()
  73 has_patterns()
  74 {
  75         ifile="$1"
  76         patt="$2"
  77         ofile="$3"
  78         cmp="$4"
  79         grep "$patt:" "$ifile" | \
  80         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
  81         while read val; do
  82                 grep -s -q "$val" "$ofile"
  83                 if [ "$?" $cmp 0 ]; then
  84                         return 1
  85                 fi
  86         done
  87
  88         return $?
  89 }
  90
  91 ##
  92 # has_each_patterns(ifile tag ofile) - does ofile contains some
  93 #                        of the patterns given by ifile's tags?
  94 #
  95 # returns 0 if all present, 1 otherwise
  96 has_each_patterns()
  97 {
  98         has_patterns "$1" "$2" "$3" -ne
  99 }
 100
 101 ##
 102 # has_none_patterns(ifile tag ofile) - does ofile contains some
 103 #                        of the patterns given by ifile's tags?
 104 #
 105 # returns 1 if any present, 0 otherwise
 106 has_none_patterns()
 107 {
 108         has_patterns "$1" "$2" "$3" -eq
 109 }
 110
 111 ##
 112 # nbr_patterns(ifile tag ofile) - does ofile contains the
 113 #                        the patterns given by ifile's tags
 114 #                        the right number of time?
 115 nbr_patterns()
 116 {
 117         ifile="$1"
 118         patt="$2"
 119         ofile="$3"
 120         grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
 121         sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
 122         while read nbr pat; do
 123                 n=$(grep -s "$pat" "$ofile" | wc -l)
 124                 if [ "$n" -ne "$nbr" ]; then
 125                         return 1
 126                 fi
 127         done
 128
 129         return $?
 130 }
 131
 132 ##
 133 # verbose(string) - prints string if we are in verbose mode
 134 verbose()
 135 {
 136         [ "$V" -eq "1" ] && echo "        $1"
 137         return 0
 138 }
 139
 140 ##
 141 # error(string[, die]) - prints an error and exits with value die if given
 142 error()
 143 {
 144         [ "$quiet" -ne 1 ] && echo "error: $1"
 145         [ -n "$2" ] && exit $2
 146         return 0
 147 }
 148
 149 do_usage()
 150 {
 151 echo "$prog_name - a tiny automatic testing script"
 152 echo "Usage: $prog_name [command] [command arguments]"
 153 echo
 154 echo "commands:"
 155 echo "    none                       runs the whole test suite"
 156 echo "    single file                runs the test in 'file'"
 157 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
 158 echo
 159 echo "    help                       prints usage"
 160 }
 161
 162 ##
 163 # do_test(file) - tries to validate a test case
 164 #
 165 # it "parses" file, looking for check-* tags and tries to validate
 166 # the test against an expected result
 167 # returns:
 168 #       - 0 if the test passed,
 169 #       - 1 if it failed,
 170 #       - 2 if it is not a "test-suite" test.
 171 #       - 3 if the test is disabled.
 172 do_test()
 173 {
 174         test_failed=0
 175         file="$1"
 176
 177         get_tag_value $file
 178
 179         # can this test be handled by test-suite ?
 180         # (it has to have a check-name key in it)
 181         if [ "$check_name" = "" ]; then
 182                 echo "warning: test '$file' unhandled"
 183                 unhandled_tests=$(($unhandled_tests + 1))
 184                 return 2
 185         fi
 186         test_name="$check_name"
 187
 188         # does the test provide a specific command ?
 189         if [ "$check_command" = "" ]; then
 190                 check_command="$defaut_command"
 191         fi
 192
 193         # check for disabled commands
 194         set -- $check_command
 195         base_cmd=$1
 196         for i in $disabled_cmds; do
 197                 if [ "$i" = "$base_cmd" ] ; then
 198                         disabled_tests=$(($disabled_tests + 1))
 199                         echo "     DISABLE $test_name ($file)"
 200                         return 3
 201                 fi
 202         done
 203
 204         cmd=`eval echo $default_path/$check_command`
 205
 206         echo "     TEST    $test_name ($file)"
 207
 208         verbose "Using command       : $cmd"
 209
 210         # grab the expected exit value
 211         expected_exit_value=$check_exit_value
 212         verbose "Expecting exit value: $expected_exit_value"
 213
 214
 215         # grab the actual output & exit value
 216         $cmd 1> $file.output.got 2> $file.error.got
 217         actual_exit_value=$?
 218
 219         must_fail=$check_known_to_fail
 220         quiet=0
 221         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 222         known_ko_tests=$(($known_ko_tests + $must_fail))
 223
 224         for stream in output error; do
 225                 eval ignore=\$check_${stream}_ignore
 226                 [ $ignore -eq 1 ] && continue
 227
 228                 # grab the expected output
 229                 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
 230                 | grep -v check-$stream > "$file".$stream.expected
 231
 232                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 233                 if [ "$?" -ne "0" ]; then
 234                         error "actual $stream text does not match expected $stream text."
 235                         error  "see $file.$stream.* for further investigation."
 236                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 237                         test_failed=1
 238                 fi
 239         done
 240
 241         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 242                 error "Actual exit value does not match the expected one."
 243                 error "expected $expected_exit_value, got $actual_exit_value."
 244                 test_failed=1
 245         fi
 246
 247         # verify the 'check-output-contains/excludes' tags
 248         if [ $check_output_contains -eq 1 ]; then
 249                 has_each_patterns "$file" 'check-output-contains' $file.output.got
 250                 if [ "$?" -ne "0" ]; then
 251                         error "Actual output doesn't contain some of the expected patterns."
 252                         test_failed=1
 253                 fi
 254         fi
 255         if [ $check_output_excludes -eq 1 ]; then
 256                 has_none_patterns "$file" 'check-output-excludes' $file.output.got
 257                 if [ "$?" -ne "0" ]; then
 258                         error "Actual output contains some patterns which are not expected."
 259                         test_failed=1
 260                 fi
 261         fi
 262         if [ $check_output_pattern -eq 1 ]; then
 263                 # verify the 'check-output-pattern-X-times' tags
 264                 nbr_patterns "$file" 'check-output-pattern' $file.output.got
 265                 if [ "$?" -ne "0" ]; then
 266                         error "Actual output doesn't contain the pattern the expected number."
 267                         test_failed=1
 268                 fi
 269         fi
 270
 271         [ "$test_failed" -eq "$must_fail" ] || failed=1
 272
 273         if [ "$must_fail" -eq "1" ]; then
 274                 if [ "$test_failed" -eq "1" ]; then
 275                         echo "info: test '$file' is known to fail"
 276                 else
 277                         echo "error: test '$file' is known to fail but succeed!"
 278                         test_failed=1
 279                 fi
 280         fi
 281
 282         if [ "$test_failed" -eq "1" ]; then
 283                 ko_tests=$(($ko_tests + 1))
 284         else
 285                 ok_tests=$(($ok_tests + 1))
 286                 rm -f $file.{error,output}.{expected,got,diff}
 287         fi
 288         return $test_failed
 289 }
 290
 291 do_test_suite()
 292 {
 293         for i in $tests_list; do
 294                 do_test "$i"
 295         done
 296
 297         # prints some numbers
 298         tests_nr=$(($ok_tests + $ko_tests))
 299         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 300         echo " ($known_ko_tests of them are known to fail)"
 301         if [ "$unhandled_tests" -ne "0" ]; then
 302                 echo "$unhandled_tests tests could not be handled by $prog_name"
 303         fi
 304         if [ "$disabled_tests" -ne "0" ]; then
 305                 echo "$disabled_tests tests were disabled"
 306         fi
 307 }
 308
 309 ##
 310 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 311 do_format()
 312 {
 313         if [ -z "$2" ]; then
 314                 fname="$1"
 315                 fcmd=$default_cmd
 316         elif [ -z "$3" ]; then
 317                 fname="$2"
 318                 fcmd=$default_cmd
 319         else
 320                 fname="$2"
 321                 fcmd="$3"
 322         fi
 323         file="$1"
 324         cmd=`eval echo $default_path/$fcmd`
 325         $cmd 1> $file.output.got 2> $file.error.got
 326         fexit_value=$?
 327         cat <<_EOF
 328 /*
 329  * check-name: $fname
 330 _EOF
 331         if [ "$fcmd" != "$default_cmd" ]; then
 332                 echo " * check-command: $fcmd"
 333         fi
 334         if [ "$fexit_value" -ne "0" ]; then
 335                 echo " * check-exit-value: $fexit_value"
 336         fi
 337         for stream in output error; do
 338                 if [ -s "$file.$stream.got" ]; then
 339                         echo " *"
 340                         echo " * check-$stream-start"
 341                         cat "$file.$stream.got"
 342                         echo " * check-$stream-end"
 343                 fi
 344         done
 345         echo " */"
 346         return 0
 347 }
 348
 349 ##
 350 # arg_file(filename) - checks if filename exists
 351 arg_file()
 352 {
 353         [ -z "$1" ] && {
 354                 do_usage
 355                 exit 1
 356         }
 357         [ -e "$1" ] || {
 358                 error "Can't open file $1"
 359                 exit 1
 360         }
 361         return 0
 362 }
 363
 364 case "$1" in
 365         '')
 366                 do_test_suite
 367                 ;;
 368         single)
 369                 arg_file "$2"
 370                 do_test "$2"
 371                 case "$?" in
 372                         0) echo "$2 passed !";;
 373                         1) echo "$2 failed !";;
 374                         2) echo "$2 can't be handled by $prog_name";;
 375                 esac
 376                 ;;
 377         format)
 378                 arg_file "$2"
 379                 do_format "$2" "$3" "$4"
 380                 ;;
 381         help | *)
 382                 do_usage
 383                 exit 1
 384                 ;;
 385 esac
 386
 387 exit $failed
 388