validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
  10 prog_name=`basename $0`
  11
  12 if [ ! -x "$default_path/sparse-llvm" ]; then
  13         disabled_cmds="sparsec sparsei sparse-llvm"
  14 fi
  15
  16 # flags:
  17 #       - some tests gave an unexpected result
  18 failed=0
  19
  20 # counts:
  21 #       - tests that have not been converted to test-suite format
  22 #       - tests that are disabled
  23 #       - tests that passed
  24 #       - tests that failed
  25 #       - tests that failed but are known to fail
  26 unhandled_tests=0
  27 disabled_tests=0
  28 ok_tests=0
  29 ko_tests=0
  30 known_ko_tests=0
  31
  32 # defaults to not verbose
  33 [ -z "$V" ] && V=0
  34
  35 ##
  36 # get_value(key, file) - gets the value of a (key, value) pair in file.
  37 #
  38 # returns 0 on success, 1 if the file does not have the key
  39 get_value()
  40 {
  41         last_result=`grep $1: $2 | sed -e "s/^.*$1:\(.*\)$/\1/"`
  42         [ -z "$last_result" ] && return 1
  43         return 0
  44 }
  45
  46 ##
  47 # get_tag(key, file) - does file has the tag key in it ?
  48 #
  49 # returns 0 if present, 1 otherwise
  50 get_tag()
  51 {
  52         last_result=`grep $1 $2`
  53         return $?
  54 }
  55
  56 ##
  57 # helper for has_(each|none)_patterns()
  58 has_patterns()
  59 {
  60         ifile="$1"
  61         patt="$2"
  62         ofile="$3"
  63         cmp="$4"
  64         grep "$patt:" "$ifile" | \
  65         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
  66         while read val; do
  67                 grep -s -q "$val" "$ofile"
  68                 if [ "$?" $cmp 0 ]; then
  69                         return 1
  70                 fi
  71         done
  72
  73         return $?
  74 }
  75
  76 ##
  77 # has_each_patterns(ifile tag ofile) - does ofile contains some
  78 #                        of the patterns given by ifile's tags?
  79 #
  80 # returns 0 if all present, 1 otherwise
  81 has_each_patterns()
  82 {
  83         has_patterns "$1" "$2" "$3" -ne
  84 }
  85
  86 ##
  87 # has_none_patterns(ifile tag ofile) - does ofile contains some
  88 #                        of the patterns given by ifile's tags?
  89 #
  90 # returns 1 if any present, 0 otherwise
  91 has_none_patterns()
  92 {
  93         has_patterns "$1" "$2" "$3" -eq
  94 }
  95
  96 ##
  97 # nbr_patterns(ifile tag ofile) - does ofile contains the
  98 #                        the patterns given by ifile's tags
  99 #                        the right number of time?
 100 nbr_patterns()
 101 {
 102         ifile="$1"
 103         patt="$2"
 104         ofile="$3"
 105         grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
 106         sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
 107         while read nbr pat; do
 108                 n=$(grep -s "$pat" "$ofile" | wc -l)
 109                 if [ "$n" -ne "$nbr" ]; then
 110                         return 1
 111                 fi
 112         done
 113
 114         return $?
 115 }
 116
 117 ##
 118 # verbose(string) - prints string if we are in verbose mode
 119 verbose()
 120 {
 121         [ "$V" -eq "1" ] && echo "        $1"
 122         return 0
 123 }
 124
 125 ##
 126 # error(string[, die]) - prints an error and exits with value die if given
 127 error()
 128 {
 129         [ "$quiet" -ne 1 ] && echo "error: $1"
 130         [ -n "$2" ] && exit $2
 131         return 0
 132 }
 133
 134 do_usage()
 135 {
 136 echo "$prog_name - a tiny automatic testing script"
 137 echo "Usage: $prog_name [command] [command arguments]"
 138 echo
 139 echo "commands:"
 140 echo "    none                       runs the whole test suite"
 141 echo "    single file                runs the test in 'file'"
 142 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
 143 echo
 144 echo "    help                       prints usage"
 145 }
 146
 147 ##
 148 # do_test(file) - tries to validate a test case
 149 #
 150 # it "parses" file, looking for check-* tags and tries to validate
 151 # the test against an expected result
 152 # returns:
 153 #       - 0 if the test passed,
 154 #       - 1 if it failed,
 155 #       - 2 if it is not a "test-suite" test.
 156 #       - 3 if the test is disabled.
 157 do_test()
 158 {
 159         test_failed=0
 160         file="$1"
 161
 162         # can this test be handled by test-suite ?
 163         # (it has to have a check-name key in it)
 164         get_value "check-name" $file
 165         if [ "$?" -eq 1 ]; then
 166                 echo "warning: test '$file' unhandled"
 167                 unhandled_tests=`expr $unhandled_tests + 1`
 168                 return 2
 169         fi
 170         test_name=$last_result
 171
 172         # does the test provide a specific command ?
 173         cmd=`eval echo $default_path/$default_cmd`
 174         get_value "check-command" $file
 175         if [ "$?" -eq "0" ]; then
 176                 last_result=`echo $last_result | sed -e 's/^ *//'`
 177                 cmd=`eval echo $default_path/$last_result`
 178         fi
 179
 180         # check for disabled commands
 181         set -- $cmd
 182         base_cmd=`basename $1`
 183         for i in $disabled_cmds; do
 184                 if [ "$i" = "$base_cmd" ] ; then
 185                         disabled_tests=`expr $disabled_tests + 1`
 186                         echo "     DISABLE $test_name ($file)"
 187                         return 3
 188                 fi
 189         done
 190
 191         echo "     TEST    $test_name ($file)"
 192
 193         verbose "Using command       : $cmd"
 194
 195         # grab the expected output
 196         sed -n '/check-output-start/,/check-output-end/p' $file \
 197                 | grep -v check-output > "$file".output.expected
 198         sed -n '/check-error-start/,/check-error-end/p' $file \
 199                 | grep -v check-error > "$file".error.expected
 200
 201         # grab the expected exit value
 202         get_value "check-exit-value" $file
 203         if [ "$?" -eq "0" ]; then
 204                 expected_exit_value=`echo $last_result | tr -d ' '`
 205         else
 206                 expected_exit_value=0
 207         fi
 208         verbose "Expecting exit value: $expected_exit_value"
 209
 210
 211         # grab the actual output & exit value
 212         $cmd 1> $file.output.got 2> $file.error.got
 213         actual_exit_value=$?
 214
 215         get_tag "check-known-to-fail" $file
 216         must_fail=`expr "$?" = 0`
 217         quiet=0
 218         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 219         known_ko_tests=`expr $known_ko_tests + $must_fail`
 220
 221         for stream in output error; do
 222                 grep -s -q "check-$stream-ignore" $file && continue
 223
 224                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 225                 if [ "$?" -ne "0" ]; then
 226                         error "actual $stream text does not match expected $stream text."
 227                         error  "see $file.$stream.* for further investigation."
 228                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 229                         test_failed=1
 230                 fi
 231         done
 232
 233         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 234                 error "Actual exit value does not match the expected one."
 235                 error "expected $expected_exit_value, got $actual_exit_value."
 236                 test_failed=1
 237         fi
 238
 239         # verify the 'check-output-contains/excludes' tags
 240         has_each_patterns "$file" 'check-output-contains' $file.output.got
 241         if [ "$?" -ne "0" ]; then
 242                 error "Actual output doesn't contain some of the expected patterns."
 243                 test_failed=1
 244         fi
 245         has_none_patterns "$file" 'check-output-excludes' $file.output.got
 246         if [ "$?" -ne "0" ]; then
 247                 error "Actual output contains some patterns which are not expected."
 248                 test_failed=1
 249         fi
 250
 251         # verify the 'check-output-pattern-X-times' tags
 252         nbr_patterns "$file" 'check-output-pattern' $file.output.got
 253         if [ "$?" -ne "0" ]; then
 254                 error "Actual output doesn't contain the pattern the expected number."
 255                 test_failed=1
 256         fi
 257
 258         [ "$test_failed" -eq "$must_fail" ] || failed=1
 259
 260         if [ "$must_fail" -eq "1" ]; then
 261                 if [ "$test_failed" -eq "1" ]; then
 262                         echo "info: test '$file' is known to fail"
 263                 else
 264                         echo "error: test '$file' is known to fail but succeed!"
 265                         test_failed=1
 266                 fi
 267         fi
 268
 269         if [ "$test_failed" -eq "1" ]; then
 270                 ko_tests=`expr $ko_tests + 1`
 271         else
 272                 ok_tests=`expr $ok_tests + 1`
 273                 rm -f $file.{error,output}.{expected,got,diff}
 274         fi
 275         return $test_failed
 276 }
 277
 278 do_test_suite()
 279 {
 280         for i in $tests_list; do
 281                 do_test "$i"
 282         done
 283
 284         # prints some numbers
 285         tests_nr=`expr $ok_tests + $ko_tests`
 286         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 287         echo " ($known_ko_tests of them are known to fail)"
 288         if [ "$unhandled_tests" -ne "0" ]; then
 289                 echo "$unhandled_tests tests could not be handled by $prog_name"
 290         fi
 291         if [ "$disabled_tests" -ne "0" ]; then
 292                 echo "$disabled_tests tests were disabled"
 293         fi
 294 }
 295
 296 ##
 297 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 298 do_format()
 299 {
 300         if [ -z "$2" ]; then
 301                 fname="$1"
 302                 fcmd=$default_cmd
 303         elif [ -z "$3" ]; then
 304                 fname="$2"
 305                 fcmd=$default_cmd
 306         else
 307                 fname="$2"
 308                 fcmd="$3"
 309         fi
 310         file="$1"
 311         cmd=`eval echo $default_path/$fcmd`
 312         $cmd 1> $file.output.got 2> $file.error.got
 313         fexit_value=$?
 314         cat <<_EOF
 315 /*
 316  * check-name: $fname
 317 _EOF
 318         if [ "$fcmd" != "$default_cmd" ]; then
 319                 echo " * check-command: $fcmd"
 320         fi
 321         if [ "$fexit_value" -ne "0" ]; then
 322                 echo " * check-exit-value: $fexit_value"
 323         fi
 324         for stream in output error; do
 325                 if [ -s "$file.$stream.got" ]; then
 326                         echo " *"
 327                         echo " * check-$stream-start"
 328                         cat "$file.$stream.got"
 329                         echo " * check-$stream-end"
 330                 fi
 331         done
 332         echo " */"
 333         return 0
 334 }
 335
 336 ##
 337 # arg_file(filename) - checks if filename exists
 338 arg_file()
 339 {
 340         [ -z "$1" ] && {
 341                 do_usage
 342                 exit 1
 343         }
 344         [ -e "$1" ] || {
 345                 error "Can't open file $1"
 346                 exit 1
 347         }
 348         return 0
 349 }
 350
 351 case "$1" in
 352         '')
 353                 do_test_suite
 354                 ;;
 355         single)
 356                 arg_file "$2"
 357                 do_test "$2"
 358                 case "$?" in
 359                         0) echo "$2 passed !";;
 360                         1) echo "$2 failed !";;
 361                         2) echo "$2 can't be handled by $prog_name";;
 362                 esac
 363                 ;;
 364         format)
 365                 arg_file "$2"
 366                 do_format "$2" "$3" "$4"
 367                 ;;
 368         help | *)
 369                 do_usage
 370                 exit 1
 371                 ;;
 372 esac
 373
 374 exit $failed
 375