validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 default_path=".."
   6 default_cmd="sparse \$file"
   7 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
   8 prog_name=`basename $0`
   9
  10 if [ ! -x "$default_path/sparse-llvm" ]; then
  11         disabled_cmds="sparsec sparsei sparse-llvm"
  12 fi
  13
  14 # counts:
  15 #       - tests that have not been converted to test-suite format
  16 #       - tests that are disabled
  17 #       - tests that passed
  18 #       - tests that failed
  19 #       - tests that failed but are known to fail
  20 unhandled_tests=0
  21 disabled_tests=0
  22 ok_tests=0
  23 ko_tests=0
  24 known_ko_tests=0
  25
  26 # defaults to not verbose
  27 [ -z "$V" ] && V=0
  28
  29 ##
  30 # get_value(key, file) - gets the value of a (key, value) pair in file.
  31 #
  32 # returns 0 on success, 1 if the file does not have the key
  33 get_value()
  34 {
  35         last_result=`grep $1: $2 | sed -e "s/^.*$1:\(.*\)$/\1/"`
  36         [ -z "$last_result" ] && return 1
  37         return 0
  38 }
  39
  40 ##
  41 # get_tag(key, file) - does file has the tag key in it ?
  42 #
  43 # returns 0 if present, 1 otherwise
  44 get_tag()
  45 {
  46         last_result=`grep $1 $2`
  47         return $?
  48 }
  49
  50 ##
  51 # verbose(string) - prints string if we are in verbose mode
  52 verbose()
  53 {
  54         [ "$V" -eq "1" ] && echo "        $1"
  55         return 0
  56 }
  57
  58 ##
  59 # error(string[, die]) - prints an error and exits with value die if given
  60 error()
  61 {
  62         echo "error: $1"
  63         [ -n "$2" ] && exit $2
  64         return 0
  65 }
  66
  67 do_usage()
  68 {
  69 echo "$prog_name - a tiny automatic testing script"
  70 echo "Usage: $prog_name [command] [command arguments]"
  71 echo
  72 echo "commands:"
  73 echo "    none                       runs the whole test suite"
  74 echo "    single file                runs the test in 'file'"
  75 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
  76 echo
  77 echo "    help                       prints usage"
  78 }
  79
  80 ##
  81 # do_test(file) - tries to validate a test case
  82 #
  83 # it "parses" file, looking for check-* tags and tries to validate
  84 # the test against an expected result
  85 # returns:
  86 #       - 0 if the test passed,
  87 #       - 1 if it failed,
  88 #       - 2 if it is not a "test-suite" test.
  89 #       - 3 if the test is disabled.
  90 do_test()
  91 {
  92         test_failed=0
  93         file="$1"
  94
  95         # can this test be handled by test-suite ?
  96         # (it has to have a check-name key in it)
  97         get_value "check-name" $file
  98         if [ "$?" -eq 1 ]; then
  99                 echo "warning: test '$file' unhandled"
 100                 unhandled_tests=`expr $unhandled_tests + 1`
 101                 return 2
 102         fi
 103         test_name=$last_result
 104
 105         # does the test provide a specific command ?
 106         cmd=`eval echo $default_path/$default_cmd`
 107         get_value "check-command" $file
 108         if [ "$?" -eq "0" ]; then
 109                 last_result=`echo $last_result | sed -e 's/^ *//'`
 110                 cmd=`eval echo $default_path/$last_result`
 111         fi
 112
 113         # check for disabled commands
 114         set -- $cmd
 115         base_cmd=`basename $1`
 116         for i in $disabled_cmds; do
 117                 if [ "$i" = "$base_cmd" ] ; then
 118                         disabled_tests=`expr $disabled_tests + 1`
 119                         echo "     DISABLE $test_name ($file)"
 120                         return 3
 121                 fi
 122         done
 123
 124         echo "     TEST    $test_name ($file)"
 125
 126         verbose "Using command       : $cmd"
 127
 128         # grab the expected output
 129         sed -n '/check-output-start/,/check-output-end/p' $file \
 130                 | grep -v check-output > "$file".output.expected
 131         sed -n '/check-error-start/,/check-error-end/p' $file \
 132                 | grep -v check-error > "$file".error.expected
 133
 134         # grab the expected exit value
 135         get_value "check-exit-value" $file
 136         if [ "$?" -eq "0" ]; then
 137                 expected_exit_value=`echo $last_result | tr -d ' '`
 138         else
 139                 expected_exit_value=0
 140         fi
 141         verbose "Expecting exit value: $expected_exit_value"
 142
 143
 144         # grab the actual output & exit value
 145         $cmd 1> $file.output.got 2> $file.error.got
 146         actual_exit_value=$?
 147
 148         for stream in output error; do
 149                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 150                 if [ "$?" -ne "0" ]; then
 151                         error "actual $stream text does not match expected $stream text."
 152                         error  "see $file.$stream.* for further investigation."
 153                         cat "$file".$stream.diff
 154                         test_failed=1
 155                 fi
 156         done
 157
 158         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 159                 error "Actual exit value does not match the expected one."
 160                 error "expected $expected_exit_value, got $actual_exit_value."
 161                 test_failed=1
 162         fi
 163
 164         if [ "$test_failed" -eq "1" ]; then
 165                 ko_tests=`expr $ko_tests + 1`
 166                 get_tag "check-known-to-fail" $file
 167                 if [ "$?" -eq "0" ]; then
 168                         echo "info: test '$file' is known to fail"
 169                         known_ko_tests=`expr $known_ko_tests + 1`
 170                 fi
 171                 return 1
 172         else
 173                 ok_tests=`expr $ok_tests + 1`
 174                 return 0
 175         fi
 176 }
 177
 178 do_test_suite()
 179 {
 180         for i in $tests_list; do
 181                 do_test "$i"
 182         done
 183
 184         # prints some numbers
 185         tests_nr=`expr $ok_tests + $ko_tests`
 186         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 187         echo " ($known_ko_tests of them are known to fail)"
 188         if [ "$unhandled_tests" -ne "0" ]; then
 189                 echo "$unhandled_tests tests could not be handled by $prog_name"
 190         fi
 191         if [ "$disabled_tests" -ne "0" ]; then
 192                 echo "$disabled_tests tests were disabled"
 193         fi
 194 }
 195
 196 ##
 197 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 198 do_format()
 199 {
 200         if [ -z "$2" ]; then
 201                 fname="$1"
 202                 fcmd=$default_cmd
 203         elif [ -z "$3" ]; then
 204                 fname="$2"
 205                 fcmd=$default_cmd
 206         else
 207                 fname="$2"
 208                 fcmd="$3"
 209         fi
 210         file="$1"
 211         cmd=`eval echo $default_path/$fcmd`
 212         $cmd 1> $file.output.got 2> $file.error.got
 213         fexit_value=$?
 214         cat <<_EOF
 215 /*
 216  * check-name: $fname
 217 _EOF
 218         if [ "$fcmd" != "$default_cmd" ]; then
 219                 echo " * check-command: $fcmd"
 220         fi
 221         if [ "$fexit_value" -ne "0" ]; then
 222                 echo " * check-exit-value: $fexit_value"
 223         fi
 224         for stream in output error; do
 225                 if [ -s "$file.$stream.got" ]; then
 226                         echo " *"
 227                         echo " * check-$stream-start"
 228                         cat "$file.$stream.got"
 229                         echo " * check-$stream-end"
 230                 fi
 231         done
 232         echo " */"
 233         return 0
 234 }
 235
 236 ##
 237 # arg_file(filename) - checks if filename exists
 238 arg_file()
 239 {
 240         [ -z "$1" ] && {
 241                 do_usage
 242                 exit 1
 243         }
 244         [ -e "$1" ] || {
 245                 error "Can't open file $1"
 246                 exit 1
 247         }
 248         return 0
 249 }
 250
 251 case "$1" in
 252         '')
 253                 do_test_suite
 254                 ;;
 255         single)
 256                 arg_file "$2"
 257                 do_test "$2"
 258                 case "$?" in
 259                         0) echo "$2 passed !";;
 260                         1) echo "$2 failed !";;
 261                         2) echo "$2 can't be handled by $prog_name";;
 262                 esac
 263                 ;;
 264         format)
 265                 arg_file "$2"
 266                 do_format "$2" "$3" "$4"
 267                 ;;
 268         help | *)
 269                 do_usage
 270                 exit 1
 271                 ;;
 272 esac
 273
 274 exit 0
 275