tests/run_tests.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2003, 2004, 2005, 2006, 2007, 2009, 2010 Dan McMahill
   4
   5 #  This program is free software; you can redistribute it and/or modify
   6 #  it under the terms of version 2 of the GNU General Public License as
   7 #  published by the Free Software Foundation
   8 #
   9 #  This program is distributed in the hope that it will be useful,
  10 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #  GNU General Public License for more details.
  13 #
  14 #  You should have received a copy of the GNU General Public License
  15 #  along with this program; if not, write to the Free Software
  16 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111 USA
  17 # All rights reserved.
  18 #
  19 # This code was derived from code written by Dan McMahill as part of the
  20 # latex-mk testsuite.  The original code was covered by a BSD license
  21 # but the copyright holder (Dan McMahill) has re-released the code under
  22 # the GPL.
  23
  24 magic_test_skip=${PCB_MAGIC_TEST_SKIP:-no}
  25
  26 if test "x${magic_test_skip}" = "xyes" ; then
  27         cat << EOF
  28
  29 The environment variable PCB_MAGIC_TEST_SKIP is set to yes.
  30 This causes the testsuite to skip all tests and report no errors.
  31 This is used for certain debugging *only*.  The primary use is to
  32 allow testing of the 'distcheck' target without including the effects
  33 of the testsuite. The reason this is useful is that due to minor differences
  34 in library versions and perhaps roundoff in different CPU's, the testsuite
  35 may falsely report failures on some systems.  These reported failures
  36 prevent using 'distcheck' for verifying the rest of the build system.
  37 These comments only apply to the tests which try to compare image files
  38 like PNG files.
  39
  40 EOF
  41
  42         exit 0
  43 fi
  44
  45 regen=no
  46
  47 usage() {
  48 cat <<EOF
  49
  50 $0 -- Run pcb regression tests
  51
  52 $0 -h|--help
  53 $0 [-d | --debug] [-g | --golden dir] [-r|--regen] [testname1 [testname2[ ...]]]
  54
  55 OVERVIEW
  56
  57 The $0 script is used both for running the pcb regression testsuite
  58 as well as maintaining it.  The way the test suite works is a number
  59 of different layouts are exported using the various export HID's.
  60
  61 The resulting output files are examined in various ways to make sure
  62 they are correct.  The exact details of how they are compared varies.
  63 For example, the PNG outputs are compared using tools from the ImageMagick
  64 suite while the ASCII centroid and bill of materials files are normalized
  65 with awk and then compared with the standard diff utility.  The normalization
  66 removes things like a comment line which contains the creation date.
  67
  68 OPTIONS
  69
  70 -d | --debug              Enables extra debug output
  71
  72 -g | --golden <dir>    :  Specifies that <dir> should be used for the
  73                           reference files.
  74
  75 -r | --regen           :  Specifies that the results should be taken as new
  76                           reference files instead of comparing them to the
  77                           existing ones.
  78
  79 LIMITATIONS
  80
  81 - The GUI interface is not checked via the regression testsuite.
  82
  83 EOF
  84 }
  85
  86 show_sep() {
  87     echo "----------------------------------------------------------------------"
  88 }
  89
  90 ##########################################################################
  91 #
  92 # debug print out
  93 #
  94
  95 do_debug=no
  96 debug() {
  97     if test $do_debug = yes ; then
  98         cat <<EOF
  99 $*
 100 EOF
 101     fi
 102 }
 103
 104 ##########################################################################
 105 #
 106 # command line processing
 107 #
 108
 109
 110 all_tests=""
 111 while test -n "$1"
 112   do
 113   case "$1"
 114       in
 115
 116       -d|--debug)
 117           do_debug=yes
 118           shift
 119           ;;
 120
 121       -h|--help)
 122           usage
 123           exit 0
 124           ;;
 125
 126       -g|--golden)
 127         # set the 'golden' directory.
 128           REFDIR="$2"
 129           shift 2
 130           ;;
 131
 132       -r|--regen)
 133         # regenerate the 'golden' output files.  Use this with caution.
 134         # In particular, all differences should be noted and understood.
 135           regen=yes
 136           shift
 137           ;;
 138
 139       -*)
 140           echo "unknown option: $1"
 141           exit 1
 142           ;;
 143
 144       *)
 145           break
 146           ;;
 147
 148   esac
 149 done
 150
 151 if test "X$regen" = "Xyes" && test $# -ne 1; then
 152     echo "Please regenerate only one test at a time."
 153     echo "This limitation is a safety measure."
 154     exit 1
 155 fi
 156
 157 ##########################################################################
 158 #
 159 # set up various tools
 160 #
 161
 162
 163 all_tests="$*"
 164
 165 # Source directory
 166 srcdir=${srcdir:-.}
 167 top_srcdir=${top_srcdir:-.}
 168
 169 # The pcb wrapper script we want to test
 170 #
 171 # by default we will be running it from $(srcdir)/runs/testname
 172 # so we need to look 3 levels up and then down to src
 173 PCB=${PCB:-../../../src/pcbtest.sh}
 174
 175 # The gerbv executible
 176 GERBV=${GERBV:-gerbv}
 177 GERBV_DEFAULT_FLAGS=${GERBV_DEFAULT_FLAGS:---export=png --window=640x480}
 178
 179 # various ImageMagick tools
 180 IM_ANIMATE=${IM_ANIMATE:-animate}
 181 IM_COMPARE=${IM_COMPARE:-compare}
 182 IM_COMPOSITE=${IM_COMPOSITE:-composite}
 183 IM_CONVERT=${IM_CONVERT:-convert}
 184 IM_DISPLAY=${IM_DISPLAY:-display}
 185 IM_MONTAGE=${IM_MONTAGE:-montage}
 186
 187 # golden directories
 188 INDIR=${INDIR:-${srcdir}/inputs}
 189 OUTDIR=outputs
 190 REFDIR=${REFDIR:-${srcdir}/golden}
 191
 192 # some system tools
 193 AWK=${AWK:-awk}
 194
 195 # the list of tests to run
 196 TESTLIST=${srcdir}/tests.list
 197
 198 if test "X$regen" = "Xyes" ; then
 199     OUTDIR="${REFDIR}"
 200 fi
 201
 202 # create output directory
 203 if test ! -d $OUTDIR ; then
 204     mkdir -p $OUTDIR
 205     if test $? -ne 0 ; then
 206         echo "Failed to create output directory ${OUTDIR}"
 207         exit 1
 208     fi
 209 fi
 210
 211 if test -z "$all_tests" ; then
 212     if test ! -f ${TESTLIST} ; then
 213         echo "ERROR: ($0)  Test list $TESTLIST does not exist"
 214         exit 1
 215     fi
 216     all_tests=`${AWK} 'BEGIN{FS="|"} /^#/{next} {print $1}' ${TESTLIST} | sed 's; ;;g'`
 217 fi
 218
 219 if test -z "${all_tests}" ; then
 220     echo "$0:  No tests specified"
 221     exit 0
 222 fi
 223
 224
 225 # fail/pass/total counts
 226 fail=0
 227 pass=0
 228 skip=0
 229 tot=0
 230
 231 ##########################################################################
 232 #
 233 # config summary
 234 #
 235
 236
 237 cat << EOF
 238
 239 srcdir                ${srcdir}
 240 top_srcdir            ${top_srcdir}
 241
 242 AWK                   ${AWK}
 243 PCB                   ${PCB}
 244 INDIR                 ${INDIR}
 245 OUTDIR                ${OUTDIR}
 246 REFDIR                ${REFDIR}
 247 TESTLIST              ${TESTLIST}
 248
 249 Gerbv:
 250
 251 GERBV                 ${GERBV}
 252 GERBV_DEFAULT_FLAGS   ${GERBV_DEFAULT_FLAGS}
 253
 254 ImageMagick Tools:
 255
 256 IM_ANIMATE               ${IM_ANIMATE}
 257 IM_COMPARE               ${IM_COMPARE}
 258 IM_COMPOSITE             ${IM_COMPOSITE}
 259 IM_CONVERT               ${IM_CONVERT}
 260 IM_DISPLAY               ${IM_DISPLAY}
 261 IM_MONTAGE               ${IM_MONTAGE}
 262
 263 EOF
 264
 265 tmpd=/tmp/pcb_tests.$$
 266 mkdir -p -m 0700 $tmpd
 267 rc=$?
 268 if test $rc -ne 0 ; then
 269     echo "$0:  ERROR:  could not create $tmpd"
 270     exit 1
 271 fi
 272
 273 ##########################################################################
 274 #
 275 # utility functions for comparison
 276 #
 277
 278 # Usage:
 279 #  compare_check "test_name" "file1" "file2"
 280 #
 281 # Makes sure that file1 and file2 both exist.  If not, mark the current
 282 # test as skipped and give an error message
 283 #
 284 compare_check() {
 285     local fn="$1"
 286     local f1="$2"
 287     local f2="$3"
 288
 289     if test ! -f "$f1" ; then
 290         echo "$0:  ${fn}(): $f1 does not exist"
 291         test_skipped=yes
 292         return 1
 293     fi
 294
 295     if test ! -f "$f2" ; then
 296         echo "$0:  ${fn}(): $f2 does not exist"
 297         test_skipped=yes
 298         return 1
 299     fi
 300     return 0
 301 }
 302
 303 ##########################################################################
 304 #
 305 # ASCII file comparison routines
 306 #
 307
 308 # Usage:
 309 #   run_diff "file1" "file2"
 310 #
 311 run_diff() {
 312     local f1="$1"
 313     local f2="$2"
 314     diff -U 2 $f1 $f2
 315     if test $? -ne 0 ; then return 1 ; fi
 316     return 0
 317 }
 318
 319 ##########################################################################
 320 #
 321 # BOM comparison routines
 322 #
 323
 324 # used to remove things like creation date from BOM files
 325 normalize_bom() {
 326     local f1="$1"
 327     local f2="$2"
 328     $AWK '
 329         /^# Date:/ {print "# Date: today"; next}
 330         /^# Author:/ {print "# Author: PCB"; next}
 331         {print}' \
 332         $f1 > $f2
 333 }
 334
 335 # top level function to compare BOM output
 336 compare_bom() {
 337     local f1="$1"
 338     local f2="$2"
 339     compare_check "compare_bom" "$f1" "$f2" || return 1
 340
 341     # an example BOM file is:
 342
 343     #  # PcbBOM Version 1.0
 344     #  # Date: Wed Jun 17 14:41:43 2009 UTC
 345     #  # Author: Dan McMahill
 346     #  # Title: Basic BOM/XY Test - PCB BOM
 347     #  # Quantity, Description, Value, RefDes
 348     #  # --------------------------------------------
 349     #  8,"Standard SMT resistor, capacitor etc","RESC3216N",R90_TOP R180_TOP R270_TOP R0_TOP R270_BOT R180_BOT R90_BOT R0_BOT
 350     #  8,"Dual in-line package, narrow (300 mil)","DIP8",UDIP90_TOP UDIP180_TOP UDIP270_TOP UDIP0_TOP UDIP270_BOT UDIP180_BOT UDIP90_BOT UDIP0_BOT
 351     #  8,"Small outline package, narrow (150mil)","SO8",USO90_TOP USO180_TOP USO270_TOP USO0_TOP USO270_BOT USO180_BOT USO90_BOT USO0_BOT
 352
 353     #  For comparison, we need to ignore changes in the Date and Author lines.
 354     local cf1=${tmpd}/`basename $f1`-ref
 355     local cf2=${tmpd}/`basename $f2`-out
 356
 357     normalize_bom $f1 $cf1
 358     normalize_bom $f2 $cf2
 359     run_diff $cf1 $cf2 || test_failed=yes
 360 }
 361
 362 ##########################################################################
 363 #
 364 # X-Y (centroid) comparison routines
 365 #
 366
 367 # used to remove things like creation date from BOM files
 368 normalize_xy() {
 369     local f1="$1"
 370     local f2="$2"
 371     $AWK '
 372         /^# Date:/ {print "# Date: today"; next}
 373         /^# Author:/ {print "# Author: PCB"; next}
 374         {print}' \
 375         $f1 > $f2
 376 }
 377
 378 compare_xy() {
 379     local f1="$1"
 380     local f2="$2"
 381     compare_check "compare_xy" "$f1" "$f2" || return 1
 382
 383     local cf1=${tmpd}/`basename $f1`-ref
 384     local cf2=${tmpd}/`basename $f2`-out
 385     normalize_xy "$f1" "$cf1"
 386     normalize_xy "$f2" "$cf2"
 387     run_diff "$cf1" "$cf2" || test_failed=yes
 388 }
 389
 390 ##########################################################################
 391 #
 392 # GCODE comparison routines
 393 #
 394
 395 # used to remove things like creation date from gcode files
 396 normalize_gcode() {
 397     local f1="$1"
 398     local f2="$2"
 399     # matches string such as '( Tue Mar  9 17:45:43 2010 )'
 400     $AWK --posix '!/^\( *[A-Z][a-z]{2} [A-Z][a-z]{2} [0123 ][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4} *\)$/' \
 401         $f1 > $f2
 402 }
 403
 404 compare_gcode() {
 405     local f1="$1"
 406     local f2="$2"
 407     compare_check "compare_gcode" "$f1" "$f2" || return 1
 408
 409     #  For comparison, we need to ignore changes in the Date and Author lines.
 410     local cf1=${tmpd}/`basename $f1`-ref
 411     local cf2=${tmpd}/`basename $f2`-out
 412
 413     normalize_gcode $f1 $cf1
 414     normalize_gcode $f2 $cf2
 415
 416     run_diff "$cf1" "$cf2" || test_failed=yes
 417 }
 418
 419 ##########################################################################
 420 #
 421 # RS274-X and Excellon comparison
 422 #
 423
 424 compare_rs274x() {
 425     local f1="$1"
 426     local f2="$2"
 427     compare_check "compare_rs274x" "$f1" "$f2" || return 1
 428
 429     # use gerbv to export our reference RS274-X file and our generated
 430     # RS274-X file to png.  Then we'll use ImageMagick to see
 431     # if there are any differences in the resulting files
 432     pngdir=${rundir}/png
 433     mkdir -p ${pngdir}
 434     nb=`basename $f1`
 435     png1=${pngdir}/${nb}-ref.png
 436     png2=${pngdir}/${nb}-out.png
 437
 438     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}"
 439     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}
 440
 441     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}"
 442     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}
 443
 444     compare_image ${png1} ${png2}
 445
 446 }
 447
 448 compare_cnc() {
 449     compare_rs274x $*
 450 }
 451
 452 ##########################################################################
 453 #
 454 # PostScript comparison
 455 #
 456
 457 compare_ps() {
 458     local f1="$1"
 459     local f2="$2"
 460     compare_check "compare_ps" "$f1" "$f2" || return 1
 461
 462     # PostScript output is difficult to compare, because the last page
 463     # ( = fab page) contains a date stamp written not as text, but drawn
 464     # with lines. For now we only check wether the file contains valid
 465     # PostScript and wether the page count matches.
 466     TEMP_FILE=`mktemp`
 467     echo "%!"                                      > ${TEMP_FILE}
 468     echo "currentpagedevice /PageCount get"       >> ${TEMP_FILE}
 469     echo " == flush"                              >> ${TEMP_FILE}
 470
 471     ORG_COUNT=`gs -q -dNODISPLAY -dBATCH -dNOPAUSE "$f1" ${TEMP_FILE}`
 472     NEW_COUNT=`gs -q -dNODISPLAY -dBATCH -dNOPAUSE "$f2" ${TEMP_FILE}`
 473     RESULT=$?
 474
 475     rm -f ${TEMP_FILE}
 476
 477     if test ${RESULT} -ne 0; then
 478         echo "Invalid PostScript generated."
 479         test_failed=yes
 480     else if test "${ORG_COUNT}" != "${NEW_COUNT}"; then
 481         echo -n "Page count of generated PostScript is ${NEW_COUNT} "
 482         echo    "instead of expected ${OLD_COUNT}."
 483         test_failed=yes
 484     fi; fi
 485
 486     unset ORG_COUNT NEW_COUNT RESULT
 487 }
 488
 489 ##########################################################################
 490 #
 491 # GIF/JPEG/PNG comparison routines
 492 #
 493
 494 compare_image() {
 495     local f1="$1"
 496     local f2="$2"
 497     compare_check "compare_image" "$f1" "$f2" || return 1
 498
 499     # now see if the image files are the same
 500     debug "${IM_COMPARE} -metric MAE ${f1} ${f2}  null:"
 501     same=`${IM_COMPARE} -metric MAE ${f1} ${f2}  null: 2>&1 | \
 502           ${AWK} '{if($1 == 0){print "yes"} else {print "no"}}'`
 503     debug "compare_image():  same = $same"
 504
 505     if test "$same" != yes ; then
 506         test_failed=yes
 507         echo "FAILED:  See ${errdir}"
 508         mkdir -p ${errdir}
 509         ${IM_COMPARE} ${f1} ${f2} ${errdir}/compare.png
 510         ${IM_COMPOSITE} ${f1} ${f2} -compose difference ${errdir}/composite.png
 511         ${IM_CONVERT} ${f1} ${f2} -compose difference -composite  -colorspace gray   ${errdir}/gray.png
 512         cat > ${errdir}/animate.sh << EOF
 513 #!/bin/sh
 514 ${IM_CONVERT} -label "%f" ${f1} ${f2} miff:- | \
 515 ${IM_MONTAGE} - -geometry +0+0 -tile 1x1 miff:- | \
 516 ${IM_ANIMATE} -delay 0.5 -loop 0 -
 517 EOF
 518         chmod a+x ${errdir}/animate.sh
 519     fi
 520 }
 521
 522 ##########################################################################
 523 #
 524 # PCB (layout) comparison routines
 525 #
 526
 527 compare_pcb() {
 528     local f1="$1"
 529     local f2="$2"
 530     compare_check "compare_pcb" "$f1" "$f2" || return 1
 531
 532     run_diff "$f1" "$f2" || test_failed=yes
 533 }
 534
 535 ##########################################################################
 536 #
 537 # The main program loop
 538 #
 539
 540 for t in $all_tests ; do
 541     show_sep
 542     echo "Test:  $t"
 543
 544     tot=`expr $tot + 1`
 545
 546     ######################################################################
 547     #
 548     # extract the details for the test
 549     #
 550
 551     pcb_flags="${PCB_DEFAULT_FLAGS}"
 552
 553     rundir="${OUTDIR}/${t}"
 554     refdir="${REFDIR}/${t}"
 555     errdir=${rundir}/mismatch
 556
 557     # test_name | layout file(s) | [export hid name] | [optional arguments to pcb] |  [mismatch]
 558     # | output files
 559     name=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $1}'`
 560     files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $2}'`
 561     hid=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {gsub(/[ \t]*/, ""); print $3}'`
 562     args=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $4}'`
 563     mismatch=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {if($5 == "mismatch"){print "yes"}else{print "no"}}'`
 564     out_files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $6}'`
 565
 566     # strip whitespace from single file names
 567     while test "X${files# }" != "X${files#}" ; do
 568         files="${files# }"
 569     done
 570     while test "X${files% }" != "X${files%}" ; do
 571         files="${files% }"
 572     done
 573
 574     if test "X${name}" = "X" ; then
 575         echo "ERROR:  Specified test ${t} does not appear to exist"
 576         skip=`expr $skip + 1`
 577         continue
 578     fi
 579
 580     if test "X${args}" != "X" ; then
 581         pcb_flags="${args}"
 582     fi
 583
 584     if test "X${hid}" = "Xgerber" ; then
 585         pcb_flags="--fab-author Fab_Author ${pcb_flags}"
 586     fi
 587
 588     if test "X${hid}" = "Xaction" ; then
 589         script_file=${files%.pcb}.script
 590         pcb_flags="--action-script ../../${INDIR}/${script_file}"
 591     else
 592         # hidden here, still in effect for all HID tests
 593         pcb_flags="-x ${hid} ${pcb_flags}"
 594     fi
 595
 596     ######################################################################
 597     #
 598     # Set up the run directory
 599     #
 600
 601     test -d ${rundir} && rm -fr ${rundir}
 602     mkdir -p ${rundir}
 603     if test $? -ne 0 ; then
 604         echo "$0:  Could not create run directory ${rundir}"
 605         skip=`expr $skip + 1`
 606         continue
 607     fi
 608
 609
 610     ######################################################################
 611     #
 612     # check to see if the files we need exist and copy them to the run
 613     # directory
 614     #
 615
 616     missing_files=no
 617     path_files=""
 618     for f in $files ; do
 619         if test ! -f ${INDIR}/${f} ; then
 620             echo "ERROR:  File $f specified as part of the $t test does not exist"
 621             missing_files=yes
 622         else
 623             path_files="${path_files} ${INDIR}/${f}"
 624             cp "${INDIR}/${f}" "${rundir}"
 625             chmod u+w "${rundir}/${f}"
 626         fi
 627     done
 628
 629     if test "$missing_files" = "yes" ; then
 630         echo "${t} had missing input files.  Skipping test"
 631         skip=`expr $skip + 1`
 632         continue
 633     fi
 634
 635     ######################################################################
 636     #
 637     # run PCB
 638     #
 639
 640     echo "(cd ${rundir} && ${PCB} ${pcb_flags} ${files})"
 641     (cd ${rundir} && ${PCB} ${pcb_flags} ${files})
 642     pcb_rc=$?
 643
 644     if test $pcb_rc -ne 0 ; then
 645         echo "${PCB} returned ${pcb_rc}.  This is a failure."
 646         fail=`expr $fail + 1`
 647         continue
 648     fi
 649
 650     ######################################################################
 651     #
 652     # check the result
 653     #
 654
 655     if test "X$regen" = "Xyes" ; then
 656         echo "Regenerated ${t}"
 657     else
 658         # compare the result to our reference file
 659         test_failed=no
 660         test_skipped=no
 661         for f in $out_files ; do
 662             debug "processing $f"
 663             # break apart type:fn into the type and file name
 664             type=`echo $f | sed 's;:.*;;g'`
 665             fn=`echo $f | sed 's;.*:;;g'`
 666
 667             case $type in
 668                 # BOM HID
 669                 bom)
 670                     compare_bom ${refdir}/${fn} ${rundir}/${fn}
 671                     ;;
 672
 673                 xy)
 674                     compare_xy ${refdir}/${fn} ${rundir}/${fn}
 675                     ;;
 676
 677                 # GCODE HID
 678                 gcode)
 679                     compare_gcode ${refdir}/${fn} ${rundir}/${fn}
 680                     ;;
 681
 682                 # GERBER HID
 683                 cnc)
 684                     compare_cnc ${refdir}/${fn} ${rundir}/${fn}
 685                     ;;
 686
 687                 gbx)
 688                     compare_rs274x ${refdir}/${fn} ${rundir}/${fn}
 689                     ;;
 690
 691                 # PS HID
 692                 ps)
 693                     compare_ps ${refdir}/${fn} ${rundir}/${fn}
 694                     ;;
 695
 696                 # PNG HID
 697                 gif)
 698                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 699                     ;;
 700
 701                 jpg)
 702                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 703                     ;;
 704
 705                 png)
 706                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 707                     ;;
 708
 709                 # actions
 710                 pcb)
 711                     compare_pcb ${refdir}/${fn} ${rundir}/${fn}
 712                     ;;
 713
 714                 # unknown
 715                 *)
 716                     echo "internal error:  $type is not a known file type"
 717                     exit 1
 718                     ;;
 719             esac
 720
 721         done
 722
 723         if test "X${hid}" != "Xaction" ; then
 724             # clean up the input files we'd copied over
 725             for f in $files ; do
 726                 rm -f ${rundir}/${f}
 727             done
 728         fi
 729
 730         if test $test_failed = yes ; then
 731             echo "FAIL"
 732             fail=`expr $fail + 1`
 733         elif test $test_skipped = yes ; then
 734             echo "SKIPPED"
 735             skip=`expr $skip + 1`
 736         else
 737             echo "PASSED"
 738             pass=`expr $pass + 1`
 739         fi
 740
 741     fi
 742
 743 done
 744
 745 show_sep
 746 echo "Passed $pass, failed $fail, skipped $skip out of $tot tests."
 747
 748 rc=0
 749 if test $pass -ne $tot ; then
 750     rc=1
 751 fi
 752
 753 exit $rc
 754