validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 default_path=".."
   6 default_cmd="sparse \$file"
   7 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
   8 prog_name=`basename $0`
   9
  10 # counts:
  11 #       - tests that have not been converted to test-suite format
  12 #       - tests that passed
  13 #       - tests that failed
  14 #       - tests that failed but are known to fail
  15 unhandled_tests=0
  16 ok_tests=0
  17 ko_tests=0
  18 known_ko_tests=0
  19
  20 # defaults to not verbose
  21 [ -z "$V" ] && V=0
  22
  23 ##
  24 # get_value(key, file) - gets the value of a (key, value) pair in file.
  25 #
  26 # returns 0 on success, 1 if the file does not have the key
  27 get_value()
  28 {
  29         last_result=`grep $1: $2 | sed -e "s/^.*$1:\(.*\)$/\1/"`
  30         [ -z "$last_result" ] && return 1
  31         return 0
  32 }
  33
  34 ##
  35 # get_tag(key, file) - does file has the tag key in it ?
  36 #
  37 # returns 0 if present, 1 otherwise
  38 get_tag()
  39 {
  40         last_result=`grep $1 $2`
  41         return $?
  42 }
  43
  44 ##
  45 # verbose(string) - prints string if we are in verbose mode
  46 verbose()
  47 {
  48         [ "$V" -eq "1" ] && echo "        $1"
  49         return 0
  50 }
  51
  52 ##
  53 # error(string[, die]) - prints an error and exits with value die if given
  54 error()
  55 {
  56         echo "error: $1"
  57         [ -n "$2" ] && exit $2
  58         return 0
  59 }
  60
  61 do_usage()
  62 {
  63 echo "$prog_name - a tiny automatic testing script"
  64 echo "Usage: $prog_name [command] [command arguments]"
  65 echo
  66 echo "commands:"
  67 echo "    none                       runs the whole test suite"
  68 echo "    single file                runs the test in 'file'"
  69 echo "    format file [name [cmd]]   helps writing a new test case using cmd"
  70 echo
  71 echo "    help                       prints usage"
  72 }
  73
  74 ##
  75 # do_test(file) - tries to validate a test case
  76 #
  77 # it "parses" file, looking for check-* tags and tries to validate
  78 # the test against an expected result
  79 # returns:
  80 #       - 0 if the test passed,
  81 #       - 1 if it failed,
  82 #       - 2 if it is not a "test-suite" test.
  83 do_test()
  84 {
  85         test_failed=0
  86         file="$1"
  87
  88         # can this test be handled by test-suite ?
  89         # (it has to have a check-name key in it)
  90         get_value "check-name" $file
  91         if [ "$?" -eq 1 ]; then
  92                 echo "warning: test '$file' unhandled"
  93                 unhandled_tests=`expr $unhandled_tests + 1`
  94                 return 2
  95         fi
  96         test_name=$last_result
  97
  98         echo "     TEST    $test_name ($file)"
  99
 100         # does the test provide a specific command ?
 101         cmd=`eval echo $default_path/$default_cmd`
 102         get_value "check-command" $file
 103         if [ "$?" -eq "0" ]; then
 104                 last_result=`echo $last_result | sed -e 's/^ *//'`
 105                 cmd=`eval echo $default_path/$last_result`
 106         fi
 107         verbose "Using command       : $cmd"
 108
 109         # grab the expected output
 110         sed -n '/check-output-start/,/check-output-end/p' $file \
 111                 | grep -v check-output > "$file".output.expected
 112         sed -n '/check-error-start/,/check-error-end/p' $file \
 113                 | grep -v check-error > "$file".error.expected
 114
 115         # grab the expected exit value
 116         get_value "check-exit-value" $file
 117         if [ "$?" -eq "0" ]; then
 118                 expected_exit_value=`echo $last_result | tr -d ' '`
 119         else
 120                 grep -q -E "^[^:]+:[[:digit:]]+:[[:digit:]]+: error:" "$file".error.expected
 121                 if  [ "$?" -eq "0" ]; then
 122                         expected_exit_value=1
 123                 else
 124                         expected_exit_value=0
 125                 fi
 126         fi
 127         verbose "Expecting exit value: $expected_exit_value"
 128
 129
 130         # grab the actual output & exit value
 131         $cmd 1> $file.output.got 2> $file.error.got
 132         actual_exit_value=$?
 133
 134         for stream in output error; do
 135                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 136                 if [ "$?" -ne "0" ]; then
 137                         error "actual $stream text does not match expected $stream text."
 138                         error  "see $file.$stream.* for further investigation."
 139                         cat "$file".$stream.diff
 140                         test_failed=1
 141                 fi
 142         done
 143
 144         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 145                 error "Actual exit value does not match the expected one."
 146                 error "expected $expected_exit_value, got $actual_exit_value."
 147                 test_failed=1
 148         fi
 149
 150         if [ "$test_failed" -eq "1" ]; then
 151                 ko_tests=`expr $ko_tests + 1`
 152                 get_tag "check-known-to-fail" $file
 153                 if [ "$?" -eq "0" ]; then
 154                         echo "info: test '$file' is known to fail"
 155                         known_ko_tests=`expr $known_ko_tests + 1`
 156                 fi
 157                 return 1
 158         else
 159                 ok_tests=`expr $ok_tests + 1`
 160                 return 0
 161         fi
 162 }
 163
 164 do_test_suite()
 165 {
 166         for i in $tests_list; do
 167                 do_test "$i"
 168         done
 169
 170         # prints some numbers
 171         tests_nr=`expr $ok_tests + $ko_tests`
 172         echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 173         echo " ($known_ko_tests of them are known to fail)"
 174         if [ "$unhandled_tests" -ne "0" ]; then
 175                 echo "$unhandled_tests tests could not be handled by $prog_name"
 176         fi
 177 }
 178
 179 ##
 180 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
 181 do_format()
 182 {
 183         if [ -z "$2" ]; then
 184                 fname="$1"
 185                 fcmd=$default_cmd
 186         elif [ -z "$3" ]; then
 187                 fname="$2"
 188                 fcmd=$default_cmd
 189         else
 190                 fname="$2"
 191                 fcmd="$3"
 192         fi
 193         file="$1"
 194         cmd=`eval echo $default_path/$fcmd`
 195         $cmd 1> $file.output.got 2> $file.error.got
 196         fexit_value=$?
 197         cat <<_EOF
 198 /*
 199  * check-name: $fname
 200 _EOF
 201         if [ "$fcmd" != "$default_cmd" ]; then
 202                 echo " * check-command: $fcmd"
 203         fi
 204         if [ "$fexit_value" -ne "0" ]; then
 205                 echo " * check-exit-value: $fexit_value"
 206         fi
 207         for stream in output error; do
 208                 if [ -s "$file.$stream.got" ]; then
 209                         echo " *"
 210                         echo " * check-$stream-start"
 211                         cat "$file.$stream.got"
 212                         echo " * check-$stream-end"
 213                 fi
 214         done
 215         echo " */"
 216         return 0
 217 }
 218
 219 ##
 220 # arg_file(filename) - checks if filename exists
 221 arg_file()
 222 {
 223         [ -z "$1" ] && {
 224                 do_usage
 225                 exit 1
 226         }
 227         [ -e "$1" ] || {
 228                 error "Can't open file $1"
 229                 exit 1
 230         }
 231         return 0
 232 }
 233
 234 case "$1" in
 235         '')
 236                 do_test_suite
 237                 ;;
 238         single)
 239                 arg_file "$2"
 240                 do_test "$2"
 241                 case "$?" in
 242                         0) echo "$2 passed !";;
 243                         1) echo "$2 failed !";;
 244                         2) echo "$2 can't be handled by $prog_name";;
 245                 esac
 246                 ;;
 247         format)
 248                 arg_file "$2"
 249                 do_format "$2" "$3" "$4"
 250                 ;;
 251         help | *)
 252                 do_usage
 253                 exit 1
 254                 ;;
 255 esac
 256
 257 exit 0
 258