validation/test-suite

   1 #!/bin/sh
   2
   3 #set -x
   4
   5 cd $(dirname "$0")
   6
   7 default_path=".."
   8 default_cmd="sparse \$file"
   9 default_args="$SPARSE_TEST_ARGS"
  10 tests_list=""
  11 prog_name=`basename $0`
  12
  13 if [ ! -x "$default_path/sparse-llvm" ]; then
  14         disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
  15 fi
  16
  17 # flags:
  18 #       - some tests gave an unexpected result
  19 failed=0
  20
  21 # counts:
  22 #       - tests that have not been converted to test-suite format
  23 #       - tests that are disabled
  24 #       - tests that passed
  25 #       - tests that failed
  26 #       - tests that failed but are known to fail
  27 unhandled_tests=0
  28 disabled_tests=0
  29 ok_tests=0
  30 ko_tests=0
  31 known_ko_tests=0
  32
  33 # defaults to not verbose
  34 [ -z "$V" ] && V=0
  35 vquiet=""
  36 quiet=0
  37 abort=0
  38
  39
  40 ##
  41 # verbose(string) - prints string if we are in verbose mode
  42 verbose()
  43 {
  44         [ "$V" -eq "1" ] && echo "        $1"
  45         return 0
  46 }
  47
  48 ##
  49 # warning(string) - prints a warning
  50 warning()
  51 {
  52         [ "$quiet" -ne 1 ] && echo "warning: $1"
  53         return 0
  54 }
  55
  56 ##
  57 # error(string[, die]) - prints an error and exits with value die if given
  58 error()
  59 {
  60         [ "$quiet" -ne 1 ] && echo "error: $1"
  61         [ -n "$2" ] && exit $2
  62         return 0
  63 }
  64
  65
  66 ##
  67 # get_tag_value(file) - get the 'check-<...>' tags & values
  68 get_tag_value()
  69 {
  70         check_name=""
  71         check_command="$default_cmd"
  72         check_exit_value=0
  73         check_timeout=0
  74         check_known_to_fail=0
  75         check_error_ignore=0
  76         check_output_ignore=0
  77         check_output_contains=0
  78         check_output_excludes=0
  79         check_output_pattern=0
  80         check_output_match=0
  81         check_output_returns=0
  82         check_arch_ignore=""
  83         check_arch_only=""
  84         check_assert=""
  85         check_cpp_if=""
  86
  87         lines=$(grep '^ \* check-[a-z-]*' $1 | \
  88                 sed -e 's/^ \* \(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
  89
  90         while read tag val; do
  91                 #echo "-> tag: '$tag'"
  92                 #echo "-> val: '$val'"
  93                 case $tag in
  94                 check-name:)            check_name="$val" ;;
  95                 check-command:)         check_command="$val" ;;
  96                 check-exit-value:)      check_exit_value="$val" ;;
  97                 check-timeout:)         [ -z "$val" ] && val=1
  98                                         check_timeout="$val" ;;
  99                 check-known-to-fail)    check_known_to_fail=1 ;;
 100                 check-error-ignore)     check_error_ignore=1 ;;
 101                 check-output-ignore)    check_output_ignore=1 ;;
 102                 check-output-contains:) check_output_contains=1 ;;
 103                 check-output-excludes:) check_output_excludes=1 ;;
 104                 check-output-pattern)   check_output_pattern=1 ;;
 105                 check-output-match)     check_output_match=1 ;;
 106                 check-output-returns:)  check_output_returns=1 ;;
 107                 check-arch-ignore:)     arch=$(uname -m)
 108                                         check_arch_ignore="$val" ;;
 109                 check-arch-only:)       arch=$(uname -m)
 110                                         check_arch_only="$val" ;;
 111                 check-assert:)          check_assert="$val" ;;
 112                 check-cpp-if:)          check_cpp_if="$val" ;;
 113
 114                 check-description:)     ;;      # ignore
 115                 check-note:)            ;;      # ignore
 116                 check-warning:)         ;;      # ignore
 117                 check-error-start)      ;;      # ignore
 118                 check-error-end)        ;;      # ignore
 119                 check-output-start)     ;;      # ignore
 120                 check-output-end)       ;;      # ignore
 121                 check-should-pass)      ;;      # ignore, unused annotation
 122                 check-should-fail)      ;;      # ignore, unused annotation
 123                 check-should-warn)      ;;      # ignore, unused annotation
 124                 check-*)                error "$1: unknown tag '$tag'" 1 ;;
 125                 esac
 126         done << EOT
 127         $lines
 128 EOT
 129 }
 130
 131 ##
 132 # helper for has_(each|none)_patterns()
 133 has_patterns()
 134 {
 135         ifile="$1"
 136         patt="$2"
 137         ofile="$3"
 138         cmp="$4"
 139         msg="$5"
 140         grep "$patt:" "$ifile" | \
 141         sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
 142         while read val; do
 143                 grep -s -q "$val" "$ofile"
 144                 if [ "$?" $cmp 0 ]; then
 145                         error " Pattern '$val' unexpectedly $msg"
 146                         return 1
 147                 fi
 148         done
 149
 150         return $?
 151 }
 152
 153 ##
 154 # has_each_patterns(ifile tag ofile) - does ofile contains some
 155 #                        of the patterns given by ifile's tags?
 156 #
 157 # returns 0 if all present, 1 otherwise
 158 has_each_patterns()
 159 {
 160         has_patterns "$1" "$2" "$4" -ne "$3"
 161 }
 162
 163 ##
 164 # has_none_patterns(ifile tag ofile) - does ofile contains some
 165 #                        of the patterns given by ifile's tags?
 166 #
 167 # returns 1 if any present, 0 otherwise
 168 has_none_patterns()
 169 {
 170         has_patterns "$1" "$2" "$4" -eq "$3"
 171 }
 172
 173 ##
 174 # minmax_patterns(ifile tag ofile) - does ofile contains the
 175 #                        the patterns given by ifile's tags
 176 #                        the right number of time?
 177 minmax_patterns()
 178 {
 179         ifile="$1"
 180         patt="$2"
 181         ofile="$3"
 182         grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
 183         sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
 184             -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
 185         while read min max pat; do
 186                 n=$(grep -s "$pat" "$ofile" | wc -l)
 187                 if [ "$max" = "eq" ]; then
 188                     if [ "$n" -ne "$min" ]; then
 189                         error " Pattern '$pat' expected $min times but got $n times"
 190                         return 1
 191                     fi
 192                     continue
 193                 fi
 194                 if [ "$min" != '-' ]; then
 195                     if [ "$n" -lt "$min" ]; then
 196                         error " Pattern '$pat' expected min $min times but got $n times"
 197                         return 1
 198                     fi
 199                 fi
 200                 if [ "$max" != '-' ]; then
 201                     if [ "$n" -gt "$max" ]; then
 202                         error " Pattern '$pat' expected max $max times but got $n times"
 203                         return 1
 204                     fi
 205                 fi
 206         done
 207
 208         return $?
 209 }
 210
 211 ##
 212 match_patterns()
 213 {
 214         ifile="$1"
 215         patt="$2"
 216         ofile="$3"
 217         grep "$patt" "$ifile" | sed -e "s/^.*$patt(\(.*\)): *\(.*\)$/\1 \2/" | \
 218         while read ins pat; do
 219                 grep -s "^      $ins\\.*[0-9]* " "$ofile" | grep -v -s -q "$pat"
 220                 if [ "$?" -ne 1 ]; then
 221                         error " IR doesn't match '$pat'"
 222                         return 1
 223                 fi
 224         done
 225
 226         return $?
 227 }
 228
 229 ##
 230 return_patterns()
 231 {
 232         ifile="$1"
 233         patt="$2"
 234         ofile="$3"
 235         grep "$patt:" "$ifile" | sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
 236         while read ret; do
 237                 grep -s "^      ret\\.[0-9]" "$ofile" | grep -v -s -q "[ \$]${ret}\$"
 238                 if [ "$?" -ne 1 ]; then
 239                         error " Return doesn't match '$ret'"
 240                         return 1
 241                 fi
 242         done
 243
 244         return $?
 245 }
 246
 247 ##
 248 # arg_file(filename) - checks if filename exists
 249 arg_file()
 250 {
 251         [ -z "$1" ] && {
 252                 do_usage
 253                 exit 1
 254         }
 255         [ -e "$1" ] || {
 256                 error "Can't open file $1"
 257                 exit 1
 258         }
 259         return 0
 260 }
 261
 262
 263 ##
 264 do_usage()
 265 {
 266 echo "$prog_name - a tiny automatic testing script"
 267 echo "Usage: $prog_name [option(s)] [command] [arguments]"
 268 echo
 269 echo "options:"
 270 echo "    -a|--abort                 Abort the tests as soon as one fails."
 271 echo "    -q|--quiet                 Be extra quiet while running the tests."
 272 echo "    --args='...'               Add these options to the test command."
 273 echo
 274 echo "commands:"
 275 echo "    [file ...]                 Runs the test suite on the given file(s)."
 276 echo "                               If a directory is given, run only those files."
 277 echo "                               If no file is given, run the whole testsuite."
 278 echo "    single file                Run the test in 'file'."
 279 echo "    format file [name [cmd]]   Help writing a new test case using cmd."
 280 echo
 281 echo "    [command] help             Print usage."
 282 }
 283
 284 disable()
 285 {
 286         disabled_tests=$(($disabled_tests + 1))
 287         if [ -z "$vquiet" ]; then
 288                 echo "  SKIP    $1 ($2)"
 289         fi
 290 }
 291
 292 ##
 293 # do_test(file) - tries to validate a test case
 294 #
 295 # it "parses" file, looking for check-* tags and tries to validate
 296 # the test against an expected result
 297 # returns:
 298 #       - 0 if the test passed,
 299 #       - 1 if it failed,
 300 #       - 2 if it is not a "test-suite" test.
 301 #       - 3 if the test is disabled.
 302 do_test()
 303 {
 304         test_failed=0
 305         file="$1"
 306         quiet=0
 307
 308         get_tag_value $file
 309
 310         # can this test be handled by test-suite ?
 311         # (it has to have a check-name key in it)
 312         if [ "$check_name" = "" ]; then
 313                 warning "$file: test unhandled"
 314                 unhandled_tests=$(($unhandled_tests + 1))
 315                 return 2
 316         fi
 317         test_name="$check_name"
 318
 319         # does the test provide a specific command ?
 320         if [ "$check_command" = "" ]; then
 321                 check_command="$defaut_command"
 322         fi
 323
 324         # check for disabled commands
 325         set -- $check_command
 326         base_cmd=$1
 327         for i in $disabled_cmds; do
 328                 if [ "$i" = "$base_cmd" ] ; then
 329                         disable "$test_name" "$file"
 330                         return 3
 331                 fi
 332         done
 333         if [ "$check_arch_ignore" != "" ]; then
 334                 if echo $arch | egrep -q -w "$check_arch_ignore"; then
 335                         disable "$test_name" "$file"
 336                         return 3
 337                 fi
 338         fi
 339         if [ "$check_arch_only" != "" ]; then
 340                 if ! (echo $arch | egrep -q -w "$check_arch_only"); then
 341                         disable "$test_name" "$file"
 342                         return 3
 343                 fi
 344         fi
 345         if [ "$check_assert" != "" ]; then
 346                 res=$(../sparse - 2>&1 >/dev/null <<- EOF
 347                         _Static_assert($check_assert, "$check_assert");
 348                         EOF
 349                 )
 350                 if [ "$res" != "" ]; then
 351                         disable "$test_name" "$file"
 352                         return 3
 353                 fi
 354         fi
 355         if [ "$check_cpp_if" != "" ]; then
 356                 res=$(../sparse -E - 2>/dev/null <<- EOF
 357                         #if !($check_cpp_if)
 358                         fail
 359                         #endif
 360                         EOF
 361                 )
 362                 if [ "$res" != "" ]; then
 363                         disable "$test_name" "$file"
 364                         return 3
 365                 fi
 366         fi
 367
 368         if [ -z "$vquiet" ]; then
 369                 echo "  TEST    $test_name ($file)"
 370         fi
 371
 372         verbose "Using command       : $(echo "$@")"
 373
 374         # grab the expected exit value
 375         expected_exit_value=$check_exit_value
 376         verbose "Expecting exit value: $expected_exit_value"
 377
 378         # do we want a timeout?
 379         pre_cmd=""
 380         if [ $check_timeout -ne 0 ]; then
 381                 pre_cmd="timeout $check_timeout"
 382         fi
 383
 384         shift
 385         # launch the test command and
 386         # grab the actual output & exit value
 387         eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
 388                 1> $file.output.got 2> $file.error.got
 389         actual_exit_value=$?
 390
 391         must_fail=$check_known_to_fail
 392         [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
 393         known_ko_tests=$(($known_ko_tests + $must_fail))
 394
 395         for stream in error output; do
 396                 eval ignore=\$check_${stream}_ignore
 397                 [ $ignore -eq 1 ] && continue
 398
 399                 # grab the expected output
 400                 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
 401                 | grep -v check-$stream > "$file".$stream.expected
 402
 403                 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
 404                 if [ "$?" -ne "0" ]; then
 405                         error "actual $stream text does not match expected $stream text."
 406                         error  "see $file.$stream.* for further investigation."
 407                         [ $quiet -ne 1 ] && cat "$file".$stream.diff
 408                         test_failed=1
 409                 fi
 410         done
 411
 412         if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 413                 error "Actual exit value does not match the expected one."
 414                 error "expected $expected_exit_value, got $actual_exit_value."
 415                 test_failed=1
 416         fi
 417
 418         # verify the 'check-output-contains/excludes' tags
 419         if [ $check_output_contains -eq 1 ]; then
 420                 has_each_patterns "$file" 'check-output-contains' absent $file.output.got
 421                 if [ "$?" -ne "0" ]; then
 422                         test_failed=1
 423                 fi
 424         fi
 425         if [ $check_output_excludes -eq 1 ]; then
 426                 has_none_patterns "$file" 'check-output-excludes' present $file.output.got
 427                 if [ "$?" -ne "0" ]; then
 428                         test_failed=1
 429                 fi
 430         fi
 431         if [ $check_output_pattern -eq 1 ]; then
 432                 # verify the 'check-output-pattern(...)' tags
 433                 minmax_patterns "$file" 'check-output-pattern' $file.output.got
 434                 if [ "$?" -ne "0" ]; then
 435                         test_failed=1
 436                 fi
 437         fi
 438         if [ $check_output_match -eq 1 ]; then
 439                 # verify the 'check-output-match($insn): $patt' tags
 440                 match_patterns "$file" 'check-output-match' $file.output.got
 441                 if [ "$?" -ne "0" ]; then
 442                         test_failed=1
 443                 fi
 444         fi
 445         if [ $check_output_returns -eq 1 ]; then
 446                 # verify the 'check-output-return: $value' tags
 447                 return_patterns "$file" 'check-output-returns' $file.output.got
 448                 if [ "$?" -ne "0" ]; then
 449                         test_failed=1
 450                 fi
 451         fi
 452
 453         if [ "$must_fail" -eq "1" ]; then
 454                 if [ "$test_failed" -eq "1" ]; then
 455                         [ -z "$vquiet" ] && \
 456                         echo "info: XFAIL: test '$file' is known to fail"
 457                 else
 458                         echo "error: XPASS: test '$file' is known to fail but succeed!"
 459                 fi
 460         else
 461                 if [ "$test_failed" -eq "1" ]; then
 462                         echo "error: FAIL: test '$file' failed"
 463                 else
 464                         [ "$V" -ne "0" ] && \
 465                         echo "info: PASS: test '$file' passed"
 466                 fi
 467         fi
 468
 469         if [ "$test_failed" -ne "$must_fail" ]; then
 470                 [ $abort -eq 1 ] && exit 1
 471                 test_failed=1
 472                 failed=1
 473         fi
 474
 475         if [ "$test_failed" -eq "1" ]; then
 476                 ko_tests=$(($ko_tests + 1))
 477         else
 478                 ok_tests=$(($ok_tests + 1))
 479                 rm -f $file.{error,output}.{expected,got,diff}
 480         fi
 481         return $test_failed
 482 }
 483
 484 do_test_suite()
 485 {
 486         for i in $tests_list; do
 487                 do_test "$i"
 488         done
 489
 490         OK=OK
 491         [ $failed -eq 0 ] || OK=KO
 492
 493         # prints some numbers
 494         tests_nr=$(($ok_tests + $ko_tests))
 495         echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
 496         if [ "$known_ko_tests" -ne 0 ]; then
 497                 echo "  $known_ko_tests of them are known to fail"
 498         fi
 499         if [ "$unhandled_tests" -ne "0" ]; then
 500                 echo "  $unhandled_tests tests could not be handled by $prog_name"
 501         fi
 502         if [ "$disabled_tests" -ne "0" ]; then
 503                 echo "  $disabled_tests tests were disabled"
 504         fi
 505 }
 506
 507 ##
 508 do_format_help() {
 509 echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
 510 echo
 511 echo "options:"
 512 echo "    -a                         append the created test to the input file"
 513 echo "    -f                         write a test known to fail"
 514 echo "    -l                         write a test for linearized code"
 515 echo "    -p                         write a test for pre-processing"
 516 echo
 517 echo "argument(s):"
 518 echo "    file                       file containing the test case(s)"
 519 echo "    name                       name for the test case (defaults to file)"
 520 echo "    cmd                        command to be used (defaults to 'sparse \$file')"
 521 }
 522
 523 ##
 524 # do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
 525 do_format()
 526 {
 527         def_cmd="$default_cmd"
 528         append=0
 529         linear=0
 530         fail=0
 531
 532         while [ $# -gt 0 ] ; do
 533                 case "$1" in
 534                 -a)
 535                         append=1 ;;
 536                 -f)
 537                         fail=1 ;;
 538                 -l)
 539                         def_cmd='test-linearize -Wno-decl $file'
 540                         linear=1 ;;
 541                 -p)
 542                         def_cmd='sparse -E $file' ;;
 543
 544                 help|-*)
 545                         do_format_help
 546                         return 0
 547                         ;;
 548                 *)      break ;;
 549                 esac
 550                 shift
 551                 continue
 552         done
 553
 554         if [ $# -lt 1 -o $# -gt 3 ]; then
 555                 do_format_help
 556                 return 0
 557         fi
 558
 559         arg_file "$1" || return 1
 560
 561         file="$1"
 562         fname="$2"
 563         [ -z "$fname" ] && fname="$(basename "$1" .c)"
 564         fcmd="$3"
 565         [ -z "$fcmd" ] && fcmd="$def_cmd"
 566
 567         cmd=`eval echo $default_path/$fcmd`
 568         $cmd 1> $file.output.got 2> $file.error.got
 569         fexit_value=$?
 570         [ $append != 0 ] && exec >> $file
 571         cat <<_EOF
 572
 573 /*
 574  * check-name: $fname
 575 _EOF
 576         if [ "$fcmd" != "$default_cmd" ]; then
 577                 echo " * check-command: $fcmd"
 578         fi
 579         if [ "$fexit_value" -ne "0" ]; then
 580                 echo " * check-exit-value: $fexit_value"
 581         fi
 582         if [ $fail != 0 ]; then
 583                 echo " * check-known-to-fail"
 584         fi
 585         if [ $linear != 0 ]; then
 586                 echo ' *'
 587                 echo ' * check-output-ignore'
 588                 echo ' * check-output-contains: xyz\\\\.'
 589                 echo ' * check-output-excludes: \\\\.'
 590         fi
 591         for stream in output error; do
 592                 if [ -s "$file.$stream.got" ]; then
 593                         echo " *"
 594                         echo " * check-$stream-start"
 595                         cat "$file.$stream.got"
 596                         echo " * check-$stream-end"
 597                 fi
 598         done
 599         echo " */"
 600         return 0
 601 }
 602
 603 ## allow flags from environment
 604 set -- $SPARSE_TEST_FLAGS "$@"
 605
 606 ## process the flags
 607 while [ "$#" -gt "0" ]; do
 608         case "$1" in
 609         -a|--abort)
 610                 abort=1
 611                 ;;
 612         -q|--quiet)
 613                 vquiet=1
 614                 ;;
 615         --args=*)
 616                 default_args="${1#--args=}";
 617                 ;;
 618
 619         single|--single)
 620                 arg_file "$2"
 621                 do_test "$2"
 622                 case "$?" in
 623                         0) echo "$2 passed !";;
 624                         1) echo "$2 failed !";;
 625                         2) echo "$2 can't be handled by $prog_name";;
 626                 esac
 627                 exit $failed
 628                 ;;
 629         format|--format)
 630                 shift
 631                 do_format "$@"
 632                 exit 0
 633                 ;;
 634         help)
 635                 do_usage
 636                 exit 1
 637                 ;;
 638
 639         *.c|*.cdoc)
 640                 tests_list="$tests_list $1"
 641                 ;;
 642         *)
 643                 if [ ! -d "$1" ]; then
 644                         do_usage
 645                         exit 1
 646                 fi
 647                 tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
 648                 ;;
 649         esac
 650         shift
 651 done
 652
 653 if [ -z "$tests_list" ]; then
 654         tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
 655 fi
 656
 657 do_test_suite
 658 exit $failed
 659