validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
  10 prog_name=`basename $0`
  11
  12 if [ ! -x "$default_path/sparse-llvm" ]; then
  13         disabled_cmds="sparsec sparsei sparse-llvm"
  14 fi
  15
  16 # flags:
  17 #       - some tests gave an unexpected result
  18 failed=0
  19
  20 # counts:
  21 #       - tests that have not been converted to test-suite format
  22 #       - tests that are disabled
  23 #       - tests that passed
  24 #       - tests that failed
  25 #       - tests that failed but are known to fail
  26 unhandled_tests=0
  27 disabled_tests=0
  28 ok_tests=0
  29 ko_tests=0
  30 known_ko_tests=0
  31
  32 # defaults to not verbose
  33 [ -z "$V" ] && V=0
  34
  35 ##
  36 # get_tag_value(file) - get the 'check-<...>' tags & values
  37 get_tag_value()
  38 {
  39         check_name=""
  40         check_command="$default_cmd"
  41         check_exit_value=0
  42         check_known_to_fail=0
  43         check_error_ignore=0
  44         check_output_ignore=0
  45         check_output_contains=0
  46         check_output_excludes=0
  47         check_output_pattern=0
  48
  49         lines=$(grep 'check-[a-z-]*' $1 | \
  50                 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
  51
  52         while read tag val; do
  53                 #echo "-> tag: '$tag'"
  54                 #echo "-> val: '$val'"
  55                 case $tag in
  56                 check-name:)            check_name="$val" ;;
  57                 check-command:)         check_command="$val" ;;
  58                 check-exit-value:)      check_exit_value="$val" ;;
  59                 check-known-to-fail)    check_known_to_fail=1 ;;
  60                 check-error-ignore)     check_error_ignore=1 ;;
  61                 check-output-ignore)    check_output_ignore=1 ;;
  62                 check-output-contains:) check_output_contains=1 ;;
  63                 check-output-excludes:) check_output_excludes=1 ;;
  64                 check-output-pattern-)  check_output_pattern=1 ;;
  65                 esac
  66         done << EOT
  67         $lines
  68 EOT
  69 }
  70
  71 ##
  72 # helper for has_(each|none)_patterns()
  73 has_patterns()
  74 {
  75         ifile="$1"
  76         patt="$2"
  77         ofile="$3"
  78         cmp="$4"
  79         grep "$patt:" "$ifile" | \
  80         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
  81         while read val; do
  82                 grep -s -q "$val" "$ofile"
  83                 if [ "$?" $cmp 0 ]; then
  84                         return 1
  85                 fi
  86         done
  87
  88         return $?
  89 }
  90
  91 ##
  92 # has_each_patterns(ifile tag ofile) - does ofile contains some
  93 #                        of the patterns given by ifile's tags?
  94 #
  95 # returns 0 if all present, 1 otherwise
  96 has_each_patterns()
  97 {
  98         has_patterns "$1" "$2" "$3" -ne
  99 }
 100
 101 ##
 102 # has_none_patterns(ifile tag ofile) - does ofile contains some
 103 #                        of the patterns given by ifile's tags?
 104 #
 105 # returns 1 if any present, 0 otherwise
 106 has_none_patterns()
 107 {
 108         has_patterns "$1" "$2" "$3" -eq
 109 }
 110
 111 ##
 112 # nbr_patterns(ifile tag ofile) - does ofile contains the
 113 #                        the patterns given by ifile's tags
 114 #                        the right number of time?
 115 nbr_patterns()
 116 {
 117         ifile="$1"
 118         patt="$2"
 119         ofile="$3"
 120         grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
 121         sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
 122         while read nbr pat; do
 123                 n=$(grep -s "$pat" "$ofile" | wc -l)
 124                 if [ "$n" -ne "$nbr" ]; then
 125                         return 1
 126                 fi
 127         done
 128
 129         return $?
 130 }
 131
 132 ##
 133 # verbose(string) - prints string if we are in verbose mode
 134 verbose()
 135 {
 136         [ "$V" -eq "1" ] && echo "        $1"
 137         return 0
 138 }
 139
 140 ##
 141 # error(string[, die]) - prints an error and exits with value die if given
 142 error()
 143 {
 144         [ "$quiet" -ne 1 ] && echo "error: $1"
 145         [ -n "$2" ] && exit $2
 146         return 0
 147 }
 148
 149 do_usage()
 150 {
 151 echo "$prog_name - a tiny automatic testing script"
 152 echo "Usage: $prog_name [command] [command arguments]"
 153 echo
 154 echo "commands:"
 155 echo "    none                       runs the whole test suite"
 156 echo "    single file                runs the test in 'file'"
 157 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
 158 echo
 159 echo "    help                       prints usage"
 160 }
 161
 162 ##
 163 # do_test(file) - tries to validate a test case
 164 #
 165 # it "parses" file, looking for check-* tags and tries to validate
 166 # the test against an expected result
 167 # returns:
 168 #       - 0 if the test passed,
 169 #       - 1 if it failed,
 170 #       - 2 if it is not a "test-suite" test.
 171 #       - 3 if the test is disabled.
 172 do_test()
 173 {
 174         test_failed=0
 175         file="$1"
 176
 177         get_tag_value $file
 178
 179         # can this test be handled by test-suite ?
 180         # (it has to have a check-name key in it)
 181         if [ "$check_name" = "" ]; then
 182                 echo "warning: test '$file' unhandled"
 183                 unhandled_tests=`expr $unhandled_tests + 1`
 184                 return 2
 185         fi
 186         test_name="$check_name"
 187
 188         # does the test provide a specific command ?
 189         if [ "$check_command" = "" ]; then
 190                 check_command="$defaut_command"
 191         fi
 192         cmd=`eval echo $default_path/$check_command`
 193
 194         # check for disabled commands
 195         set -- $cmd
 196         base_cmd=`basename $1`
 197         for i in $disabled_cmds; do
 198                 if [ "$i" = "$base_cmd" ] ; then
 199                         disabled_tests=`expr $disabled_tests + 1`
 200                         echo "     DISABLE $test_name ($file)"
 201                         return 3
 202                 fi
 203         done
 204
 205         echo "     TEST    $test_name ($file)"
 206
 207         verbose "Using command       : $cmd"
 208
 209         # grab the expected output
 210         sed -n '/check-output-start/,/check-output-end/p' $file \
 211                 | grep -v check-output > "$file".output.expected
 212         sed -n '/check-error-start/,/check-error-end/p' $file \
 213                 | grep -v check-error > "$file".error.expected
 214
 215         # grab the expected exit value
 216         expected_exit_value=$check_exit_value
 217         verbose "Expecting exit value: $expected_exit_value"
 218
 219
 220         # grab the actual output & exit value
 221         $cmd 1> $file.output.got 2> $file.error.got
 222         actual_exit_value=$?
 223
 224         must_fail=$check_known_to_fail
 225         quiet=0
 226         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 227         known_ko_tests=`expr $known_ko_tests + $must_fail`
 228
 229         for stream in output error; do
 230                 eval ignore=\$check_${stream}_ignore
 231                 [ $ignore -eq 1 ] && continue
 232
 233                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 234                 if [ "$?" -ne "0" ]; then
 235                         error "actual $stream text does not match expected $stream text."
 236                         error  "see $file.$stream.* for further investigation."
 237                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 238                         test_failed=1
 239                 fi
 240         done
 241
 242         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 243                 error "Actual exit value does not match the expected one."
 244                 error "expected $expected_exit_value, got $actual_exit_value."
 245                 test_failed=1
 246         fi
 247
 248         # verify the 'check-output-contains/excludes' tags
 249         has_each_patterns "$file" 'check-output-contains' $file.output.got
 250         if [ "$?" -ne "0" ]; then
 251                 error "Actual output doesn't contain some of the expected patterns."
 252                 test_failed=1
 253         fi
 254         has_none_patterns "$file" 'check-output-excludes' $file.output.got
 255         if [ "$?" -ne "0" ]; then
 256                 error "Actual output contains some patterns which are not expected."
 257                 test_failed=1
 258         fi
 259
 260         # verify the 'check-output-pattern-X-times' tags
 261         nbr_patterns "$file" 'check-output-pattern' $file.output.got
 262         if [ "$?" -ne "0" ]; then
 263                 error "Actual output doesn't contain the pattern the expected number."
 264                 test_failed=1
 265         fi
 266
 267         [ "$test_failed" -eq "$must_fail" ] || failed=1
 268
 269         if [ "$must_fail" -eq "1" ]; then
 270                 if [ "$test_failed" -eq "1" ]; then
 271                         echo "info: test '$file' is known to fail"
 272                 else
 273                         echo "error: test '$file' is known to fail but succeed!"
 274                         test_failed=1
 275                 fi
 276         fi
 277
 278         if [ "$test_failed" -eq "1" ]; then
 279                 ko_tests=`expr $ko_tests + 1`
 280         else
 281                 ok_tests=`expr $ok_tests + 1`
 282                 rm -f $file.{error,output}.{expected,got,diff}
 283         fi
 284         return $test_failed
 285 }
 286
 287 do_test_suite()
 288 {
 289         for i in $tests_list; do
 290                 do_test "$i"
 291         done
 292
 293         # prints some numbers
 294         tests_nr=`expr $ok_tests + $ko_tests`
 295         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 296         echo " ($known_ko_tests of them are known to fail)"
 297         if [ "$unhandled_tests" -ne "0" ]; then
 298                 echo "$unhandled_tests tests could not be handled by $prog_name"
 299         fi
 300         if [ "$disabled_tests" -ne "0" ]; then
 301                 echo "$disabled_tests tests were disabled"
 302         fi
 303 }
 304
 305 ##
 306 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 307 do_format()
 308 {
 309         if [ -z "$2" ]; then
 310                 fname="$1"
 311                 fcmd=$default_cmd
 312         elif [ -z "$3" ]; then
 313                 fname="$2"
 314                 fcmd=$default_cmd
 315         else
 316                 fname="$2"
 317                 fcmd="$3"
 318         fi
 319         file="$1"
 320         cmd=`eval echo $default_path/$fcmd`
 321         $cmd 1> $file.output.got 2> $file.error.got
 322         fexit_value=$?
 323         cat <<_EOF
 324 /*
 325  * check-name: $fname
 326 _EOF
 327         if [ "$fcmd" != "$default_cmd" ]; then
 328                 echo " * check-command: $fcmd"
 329         fi
 330         if [ "$fexit_value" -ne "0" ]; then
 331                 echo " * check-exit-value: $fexit_value"
 332         fi
 333         for stream in output error; do
 334                 if [ -s "$file.$stream.got" ]; then
 335                         echo " *"
 336                         echo " * check-$stream-start"
 337                         cat "$file.$stream.got"
 338                         echo " * check-$stream-end"
 339                 fi
 340         done
 341         echo " */"
 342         return 0
 343 }
 344
 345 ##
 346 # arg_file(filename) - checks if filename exists
 347 arg_file()
 348 {
 349         [ -z "$1" ] && {
 350                 do_usage
 351                 exit 1
 352         }
 353         [ -e "$1" ] || {
 354                 error "Can't open file $1"
 355                 exit 1
 356         }
 357         return 0
 358 }
 359
 360 case "$1" in
 361         '')
 362                 do_test_suite
 363                 ;;
 364         single)
 365                 arg_file "$2"
 366                 do_test "$2"
 367                 case "$?" in
 368                         0) echo "$2 passed !";;
 369                         1) echo "$2 failed !";;
 370                         2) echo "$2 can't be handled by $prog_name";;
 371                 esac
 372                 ;;
 373         format)
 374                 arg_file "$2"
 375                 do_format "$2" "$3" "$4"
 376                 ;;
 377         help | *)
 378                 do_usage
 379                 exit 1
 380                 ;;
 381 esac
 382
 383 exit $failed
 384