validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 default_args="$SPARSE_TEST_ARGS"
  10 tests_list=""
  11 prog_name=`basename $0`
  12
  13 if [ ! -x "$default_path/sparse-llvm" ]; then
  14         disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
  15 fi
  16 if [ ! -x "$default_path/scheck" ]; then
  17         disabled_cmds="$disabled_cmds scheck"
  18 fi
  19
  20 # flags:
  21 #       - some tests gave an unexpected result
  22 failed=0
  23
  24 # counts:
  25 #       - tests that have not been converted to test-suite format
  26 #       - tests that are disabled
  27 #       - tests that passed
  28 #       - tests that failed
  29 #       - tests that failed but are known to fail
  30 unhandled_tests=0
  31 disabled_tests=0
  32 ok_tests=0
  33 ko_tests=0
  34 known_ko_tests=0
  35
  36 # defaults to not verbose
  37 [ -z "$V" ] && V=0
  38 vquiet=""
  39 quiet=0
  40 abort=0
  41
  42
  43 ##
  44 # verbose(string) - prints string if we are in verbose mode
  45 verbose()
  46 {
  47         [ "$V" -eq "1" ] && echo "        $1"
  48         return 0
  49 }
  50
  51 ##
  52 # warning(string) - prints a warning
  53 warning()
  54 {
  55         [ "$quiet" -ne 1 ] && echo "warning: $1"
  56         return 0
  57 }
  58
  59 ##
  60 # error(string[, die]) - prints an error and exits with value die if given
  61 error()
  62 {
  63         [ "$quiet" -ne 1 ] && echo "error: $1"
  64         [ -n "$2" ] && exit $2
  65         return 0
  66 }
  67
  68
  69 ##
  70 # get_tag_value(file) - get the 'check-<...>' tags & values
  71 get_tag_value()
  72 {
  73         check_name=""
  74         check_command="$default_cmd"
  75         check_exit_value=0
  76         check_timeout=0
  77         check_known_to_fail=0
  78         check_error_ignore=0
  79         check_output_ignore=0
  80         check_output_contains=0
  81         check_output_excludes=0
  82         check_output_pattern=0
  83         check_output_match=0
  84         check_output_returns=0
  85         check_arch_ignore=""
  86         check_arch_only=""
  87         check_assert=""
  88         check_cpp_if=""
  89
  90         lines=$(grep '^ \* check-[a-z-]*' $1 | \
  91                 sed -e 's/^ \* \(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
  92
  93         while read tag val; do
  94                 #echo "-> tag: '$tag'"
  95                 #echo "-> val: '$val'"
  96                 case $tag in
  97                 check-name:)            check_name="$val" ;;
  98                 check-command:)         check_command="$val" ;;
  99                 check-exit-value:)      check_exit_value="$val" ;;
 100                 check-timeout:)         [ -z "$val" ] && val=1
 101                                         check_timeout="$val" ;;
 102                 check-known-to-fail)    check_known_to_fail=1 ;;
 103                 check-error-ignore)     check_error_ignore=1 ;;
 104                 check-output-ignore)    check_output_ignore=1 ;;
 105                 check-output-contains:) check_output_contains=1 ;;
 106                 check-output-excludes:) check_output_excludes=1 ;;
 107                 check-output-pattern)   check_output_pattern=1 ;;
 108                 check-output-match)     check_output_match=1 ;;
 109                 check-output-returns:)  check_output_returns=1 ;;
 110                 check-arch-ignore:)     arch=$(uname -m)
 111                                         check_arch_ignore="$val" ;;
 112                 check-arch-only:)       arch=$(uname -m)
 113                                         check_arch_only="$val" ;;
 114                 check-assert:)          check_assert="$val" ;;
 115                 check-cpp-if:)          check_cpp_if="$val" ;;
 116
 117                 check-description:)     ;;      # ignore
 118                 check-note:)            ;;      # ignore
 119                 check-warning:)         ;;      # ignore
 120                 check-error-start)      ;;      # ignore
 121                 check-error-end)        ;;      # ignore
 122                 check-output-start)     ;;      # ignore
 123                 check-output-end)       ;;      # ignore
 124                 check-should-pass)      ;;      # ignore, unused annotation
 125                 check-should-fail)      ;;      # ignore, unused annotation
 126                 check-should-warn)      ;;      # ignore, unused annotation
 127                 check-*)                error "$1: unknown tag '$tag'" 1 ;;
 128                 esac
 129         done << EOT
 130         $lines
 131 EOT
 132 }
 133
 134 ##
 135 # helper for has_(each|none)_patterns()
 136 has_patterns()
 137 {
 138         ifile="$1"
 139         patt="$2"
 140         ofile="$3"
 141         cmp="$4"
 142         msg="$5"
 143         grep "$patt:" "$ifile" | \
 144         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
 145         while read val; do
 146                 grep -s -q "$val" "$ofile"
 147                 if [ "$?" $cmp 0 ]; then
 148                         error " Pattern '$val' unexpectedly $msg"
 149                         return 1
 150                 fi
 151         done
 152
 153         return $?
 154 }
 155
 156 ##
 157 # has_each_patterns(ifile tag ofile) - does ofile contains some
 158 #                        of the patterns given by ifile's tags?
 159 #
 160 # returns 0 if all present, 1 otherwise
 161 has_each_patterns()
 162 {
 163         has_patterns "$1" "$2" "$4" -ne "$3"
 164 }
 165
 166 ##
 167 # has_none_patterns(ifile tag ofile) - does ofile contains some
 168 #                        of the patterns given by ifile's tags?
 169 #
 170 # returns 1 if any present, 0 otherwise
 171 has_none_patterns()
 172 {
 173         has_patterns "$1" "$2" "$4" -eq "$3"
 174 }
 175
 176 ##
 177 # minmax_patterns(ifile tag ofile) - does ofile contains the
 178 #                        the patterns given by ifile's tags
 179 #                        the right number of time?
 180 minmax_patterns()
 181 {
 182         ifile="$1"
 183         patt="$2"
 184         ofile="$3"
 185         grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
 186         sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
 187             -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
 188         while read min max pat; do
 189                 n=$(grep -s "$pat" "$ofile" | wc -l)
 190                 if [ "$max" = "eq" ]; then
 191                     if [ "$n" -ne "$min" ]; then
 192                         error " Pattern '$pat' expected $min times but got $n times"
 193                         return 1
 194                     fi
 195                     continue
 196                 fi
 197                 if [ "$min" != '-' ]; then
 198                     if [ "$n" -lt "$min" ]; then
 199                         error " Pattern '$pat' expected min $min times but got $n times"
 200                         return 1
 201                     fi
 202                 fi
 203                 if [ "$max" != '-' ]; then
 204                     if [ "$n" -gt "$max" ]; then
 205                         error " Pattern '$pat' expected max $max times but got $n times"
 206                         return 1
 207                     fi
 208                 fi
 209         done
 210
 211         return $?
 212 }
 213
 214 ##
 215 match_patterns()
 216 {
 217         ifile="$1"
 218         patt="$2"
 219         ofile="$3"
 220         grep "$patt" "$ifile" | sed -e "s/^.*$patt(\(.*\)): *\(.*\)$/\1 \2/" | \
 221         while read ins pat; do
 222                 grep -s "^      $ins\\.*[0-9]* " "$ofile" | grep -v -s -q "$pat"
 223                 if [ "$?" -ne 1 ]; then
 224                         error " IR doesn't match '$pat'"
 225                         return 1
 226                 fi
 227         done
 228
 229         return $?
 230 }
 231
 232 ##
 233 return_patterns()
 234 {
 235         ifile="$1"
 236         patt="$2"
 237         ofile="$3"
 238         grep "$patt:" "$ifile" | sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
 239         while read ret; do
 240                 grep -s "^      ret\\.[0-9]" "$ofile" | grep -v -s -q "[ \$]${ret}\$"
 241                 if [ "$?" -ne 1 ]; then
 242                         error " Return doesn't match '$ret'"
 243                         return 1
 244                 fi
 245         done
 246
 247         return $?
 248 }
 249
 250 ##
 251 # arg_file(filename) - checks if filename exists
 252 arg_file()
 253 {
 254         [ -z "$1" ] && {
 255                 do_usage
 256                 exit 1
 257         }
 258         [ -e "$1" ] || {
 259                 error "Can't open file $1"
 260                 exit 1
 261         }
 262         return 0
 263 }
 264
 265
 266 ##
 267 do_usage()
 268 {
 269 echo "$prog_name - a tiny automatic testing script"
 270 echo "Usage: $prog_name [option(s)] [command] [arguments]"
 271 echo
 272 echo "options:"
 273 echo "    -a|--abort                 Abort the tests as soon as one fails."
 274 echo "    -q|--quiet                 Be extra quiet while running the tests."
 275 echo "    --args='...'               Add these options to the test command."
 276 echo
 277 echo "commands:"
 278 echo "    [file ...]                 Runs the test suite on the given file(s)."
 279 echo "                               If a directory is given, run only those files."
 280 echo "                               If no file is given, run the whole testsuite."
 281 echo "    single file                Run the test in 'file'."
 282 echo "    format file [name [cmd]]   Help writing a new test case using cmd."
 283 echo
 284 echo "    [command] help             Print usage."
 285 }
 286
 287 disable()
 288 {
 289         disabled_tests=$(($disabled_tests + 1))
 290         if [ -z "$vquiet" ]; then
 291                 echo "  SKIP    $1 ($2)"
 292         fi
 293 }
 294
 295 ##
 296 # do_test(file) - tries to validate a test case
 297 #
 298 # it "parses" file, looking for check-* tags and tries to validate
 299 # the test against an expected result
 300 # returns:
 301 #       - 0 if the test passed,
 302 #       - 1 if it failed,
 303 #       - 2 if it is not a "test-suite" test.
 304 #       - 3 if the test is disabled.
 305 do_test()
 306 {
 307         test_failed=0
 308         file="$1"
 309         quiet=0
 310
 311         get_tag_value $file
 312
 313         # can this test be handled by test-suite ?
 314         # (it has to have a check-name key in it)
 315         if [ "$check_name" = "" ]; then
 316                 warning "$file: test unhandled"
 317                 unhandled_tests=$(($unhandled_tests + 1))
 318                 return 2
 319         fi
 320         test_name="$check_name"
 321
 322         # does the test provide a specific command ?
 323         if [ "$check_command" = "" ]; then
 324                 check_command="$defaut_command"
 325         fi
 326
 327         # check for disabled commands
 328         set -- $check_command
 329         base_cmd=$1
 330         for i in $disabled_cmds; do
 331                 if [ "$i" = "$base_cmd" ] ; then
 332                         disable "$test_name" "$file"
 333                         return 3
 334                 fi
 335         done
 336         if [ "$check_arch_ignore" != "" ]; then
 337                 if echo $arch | egrep -q -w "$check_arch_ignore"; then
 338                         disable "$test_name" "$file"
 339                         return 3
 340                 fi
 341         fi
 342         if [ "$check_arch_only" != "" ]; then
 343                 if ! (echo $arch | egrep -q -w "$check_arch_only"); then
 344                         disable "$test_name" "$file"
 345                         return 3
 346                 fi
 347         fi
 348         if [ "$check_assert" != "" ]; then
 349                 res=$(../sparse - 2>&1 >/dev/null <<- EOF
 350                         _Static_assert($check_assert, "$check_assert");
 351                         EOF
 352                 )
 353                 if [ "$res" != "" ]; then
 354                         disable "$test_name" "$file"
 355                         return 3
 356                 fi
 357         fi
 358         if [ "$check_cpp_if" != "" ]; then
 359                 res=$(../sparse -E - 2>/dev/null <<- EOF
 360                         #if !($check_cpp_if)
 361                         fail
 362                         #endif
 363                         EOF
 364                 )
 365                 if [ "$res" != "" ]; then
 366                         disable "$test_name" "$file"
 367                         return 3
 368                 fi
 369         fi
 370
 371         if [ -z "$vquiet" ]; then
 372                 echo "  TEST    $test_name ($file)"
 373         fi
 374
 375         verbose "Using command       : $(echo "$@")"
 376
 377         # grab the expected exit value
 378         expected_exit_value=$check_exit_value
 379         verbose "Expecting exit value: $expected_exit_value"
 380
 381         # do we want a timeout?
 382         pre_cmd=""
 383         if [ $check_timeout -ne 0 ]; then
 384                 pre_cmd="timeout $check_timeout"
 385         fi
 386
 387         shift
 388         # launch the test command and
 389         # grab the actual output & exit value
 390         eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
 391                 1> $file.output.got 2> $file.error.got
 392         actual_exit_value=$?
 393
 394         must_fail=$check_known_to_fail
 395         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 396         known_ko_tests=$(($known_ko_tests + $must_fail))
 397
 398         for stream in error output; do
 399                 eval ignore=\$check_${stream}_ignore
 400                 [ $ignore -eq 1 ] && continue
 401
 402                 # grab the expected output
 403                 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
 404                 | grep -v check-$stream > "$file".$stream.expected
 405
 406                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 407                 if [ "$?" -ne "0" ]; then
 408                         error "actual $stream text does not match expected $stream text."
 409                         error  "see $file.$stream.* for further investigation."
 410                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 411                         test_failed=1
 412                 fi
 413         done
 414
 415         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 416                 error "Actual exit value does not match the expected one."
 417                 error "expected $expected_exit_value, got $actual_exit_value."
 418                 test_failed=1
 419         fi
 420
 421         # verify the 'check-output-contains/excludes' tags
 422         if [ $check_output_contains -eq 1 ]; then
 423                 has_each_patterns "$file" 'check-output-contains' absent $file.output.got
 424                 if [ "$?" -ne "0" ]; then
 425                         test_failed=1
 426                 fi
 427         fi
 428         if [ $check_output_excludes -eq 1 ]; then
 429                 has_none_patterns "$file" 'check-output-excludes' present $file.output.got
 430                 if [ "$?" -ne "0" ]; then
 431                         test_failed=1
 432                 fi
 433         fi
 434         if [ $check_output_pattern -eq 1 ]; then
 435                 # verify the 'check-output-pattern(...)' tags
 436                 minmax_patterns "$file" 'check-output-pattern' $file.output.got
 437                 if [ "$?" -ne "0" ]; then
 438                         test_failed=1
 439                 fi
 440         fi
 441         if [ $check_output_match -eq 1 ]; then
 442                 # verify the 'check-output-match($insn): $patt' tags
 443                 match_patterns "$file" 'check-output-match' $file.output.got
 444                 if [ "$?" -ne "0" ]; then
 445                         test_failed=1
 446                 fi
 447         fi
 448         if [ $check_output_returns -eq 1 ]; then
 449                 # verify the 'check-output-return: $value' tags
 450                 return_patterns "$file" 'check-output-returns' $file.output.got
 451                 if [ "$?" -ne "0" ]; then
 452                         test_failed=1
 453                 fi
 454         fi
 455
 456         if [ "$must_fail" -eq "1" ]; then
 457                 if [ "$test_failed" -eq "1" ]; then
 458                         [ -z "$vquiet" ] && \
 459                         echo "info: XFAIL: test '$file' is known to fail"
 460                 else
 461                         echo "error: XPASS: test '$file' is known to fail but succeed!"
 462                 fi
 463         else
 464                 if [ "$test_failed" -eq "1" ]; then
 465                         echo "error: FAIL: test '$file' failed"
 466                 else
 467                         [ "$V" -ne "0" ] && \
 468                         echo "info: PASS: test '$file' passed"
 469                 fi
 470         fi
 471
 472         if [ "$test_failed" -ne "$must_fail" ]; then
 473                 [ $abort -eq 1 ] && exit 1
 474                 test_failed=1
 475                 failed=1
 476         fi
 477
 478         if [ "$test_failed" -eq "1" ]; then
 479                 ko_tests=$(($ko_tests + 1))
 480         else
 481                 ok_tests=$(($ok_tests + 1))
 482                 rm -f $file.{error,output}.{expected,got,diff}
 483         fi
 484         return $test_failed
 485 }
 486
 487 do_test_suite()
 488 {
 489         for i in $tests_list; do
 490                 do_test "$i"
 491         done
 492
 493         OK=OK
 494         [ $failed -eq 0 ] || OK=KO
 495
 496         # prints some numbers
 497         tests_nr=$(($ok_tests + $ko_tests))
 498         echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 499         if [ "$known_ko_tests" -ne 0 ]; then
 500                 echo "  $known_ko_tests of them are known to fail"
 501         fi
 502         if [ "$unhandled_tests" -ne "0" ]; then
 503                 echo "  $unhandled_tests tests could not be handled by $prog_name"
 504         fi
 505         if [ "$disabled_tests" -ne "0" ]; then
 506                 echo "  $disabled_tests tests were disabled"
 507         fi
 508 }
 509
 510 ##
 511 do_format_help() {
 512 echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
 513 echo
 514 echo "options:"
 515 echo "    -a                         append the created test to the input file"
 516 echo "    -f                         write a test known to fail"
 517 echo "    -l                         write a test for linearized code"
 518 echo "    -r                         write a test for linearized code returning 1"
 519 echo "    -p                         write a test for pre-processing"
 520 echo "    -s                         write a test for symbolic checking"
 521 echo
 522 echo "argument(s):"
 523 echo "    file                       file containing the test case(s)"
 524 echo "    name                       name for the test case (defaults to file)"
 525 echo "    cmd                        command to be used (defaults to 'sparse \$file')"
 526 }
 527
 528 ##
 529 # do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
 530 do_format()
 531 {
 532         def_cmd="$default_cmd"
 533         append=0
 534         linear=0
 535         fail=0
 536         ret=''
 537
 538         while [ $# -gt 0 ] ; do
 539                 case "$1" in
 540                 -a)
 541                         append=1 ;;
 542                 -f)
 543                         fail=1 ;;
 544                 -l)
 545                         def_cmd='test-linearize -Wno-decl $file'
 546                         linear=1 ;;
 547                 -r)
 548                         def_cmd='test-linearize -Wno-decl $file'
 549                         ret=1 ;;
 550                 -p)
 551                         def_cmd='sparse -E $file' ;;
 552                 -s)
 553                         def_cmd='scheck $file' ;;
 554
 555                 help|-*)
 556                         do_format_help
 557                         return 0
 558                         ;;
 559                 *)      break ;;
 560                 esac
 561                 shift
 562                 continue
 563         done
 564
 565         if [ $# -lt 1 -o $# -gt 3 ]; then
 566                 do_format_help
 567                 return 0
 568         fi
 569
 570         arg_file "$1" || return 1
 571
 572         file="$1"
 573         fname="$2"
 574         [ -z "$fname" ] && fname="$(basename "$1" .c)"
 575         fcmd="$3"
 576         [ -z "$fcmd" ] && fcmd="$def_cmd"
 577
 578         cmd=`eval echo $default_path/$fcmd`
 579         $cmd 1> $file.output.got 2> $file.error.got
 580         fexit_value=$?
 581         [ $append != 0 ] && exec >> $file
 582         cat <<_EOF
 583
 584 /*
 585  * check-name: $fname
 586 _EOF
 587         if [ "$fcmd" != "$default_cmd" ]; then
 588                 echo " * check-command: $fcmd"
 589         fi
 590         if [ "$fexit_value" -ne "0" ]; then
 591                 echo " * check-exit-value: $fexit_value"
 592         fi
 593         if [ $fail != 0 ]; then
 594                 echo " * check-known-to-fail"
 595         fi
 596         if [ "$ret" != '' ]; then
 597                 echo ' *'
 598                 echo ' * check-output-ignore'
 599                 echo " * check-output-returns: $ret"
 600                 rm -f "$file.output.got"
 601         fi
 602         if [ $linear != 0 ]; then
 603                 echo ' *'
 604                 echo ' * check-output-ignore'
 605                 echo ' * check-output-contains: xyz\\\\.'
 606                 echo ' * check-output-excludes: \\\\.'
 607         fi
 608         for stream in output error; do
 609                 if [ -s "$file.$stream.got" ]; then
 610                         echo " *"
 611                         echo " * check-$stream-start"
 612                         cat "$file.$stream.got"
 613                         echo " * check-$stream-end"
 614                 fi
 615         done
 616         echo " */"
 617         return 0
 618 }
 619
 620 ## allow flags from environment
 621 set -- $SPARSE_TEST_FLAGS "$@"
 622
 623 ## process the flags
 624 while [ "$#" -gt "0" ]; do
 625         case "$1" in
 626         -a|--abort)
 627                 abort=1
 628                 ;;
 629         -q|--quiet)
 630                 vquiet=1
 631                 ;;
 632         --args=*)
 633                 default_args="${1#--args=}";
 634                 ;;
 635
 636         single|--single)
 637                 arg_file "$2"
 638                 do_test "$2"
 639                 case "$?" in
 640                         0) echo "$2 passed !";;
 641                         1) echo "$2 failed !";;
 642                         2) echo "$2 can't be handled by $prog_name";;
 643                 esac
 644                 exit $failed
 645                 ;;
 646         format|--format)
 647                 shift
 648                 do_format "$@"
 649                 exit 0
 650                 ;;
 651         help)
 652                 do_usage
 653                 exit 1
 654                 ;;
 655
 656         *.c|*.cdoc)
 657                 tests_list="$tests_list $1"
 658                 ;;
 659         *)
 660                 if [ ! -d "$1" ]; then
 661                         do_usage
 662                         exit 1
 663                 fi
 664                 tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
 665                 ;;
 666         esac
 667         shift
 668 done
 669
 670 if [ -z "$tests_list" ]; then
 671         tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
 672 fi
 673
 674 do_test_suite
 675 exit $failed
 676