tests/run_tests.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2003, 2004, 2005, 2006, 2007, 2009, 2010 Dan McMahill
   4
   5 #  This program is free software; you can redistribute it and/or modify
   6 #  it under the terms of version 2 of the GNU General Public License as
   7 #  published by the Free Software Foundation
   8 #
   9 #  This program is distributed in the hope that it will be useful,
  10 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #  GNU General Public License for more details.
  13 #
  14 #  You should have received a copy of the GNU General Public License
  15 #  along with this program; if not, write to the Free Software
  16 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111 USA
  17 # All rights reserved.
  18 #
  19 # This code was derived from code written by Dan McMahill as part of the
  20 # latex-mk testsuite.  The original code was covered by a BSD license
  21 # but the copyright holder (Dan McMahill) has re-released the code under
  22 # the GPL.
  23
  24 magic_test_skip=${PCB_MAGIC_TEST_SKIP:-no}
  25
  26 if test "x${magic_test_skip}" = "xyes" ; then
  27         cat << EOF
  28
  29 The environment variable PCB_MAGIC_TEST_SKIP is set to yes.
  30 This causes the testsuite to skip all tests and report no errors.
  31 This is used for certain debugging *only*.  The primary use is to
  32 allow testing of the 'distcheck' target without including the effects
  33 of the testsuite. The reason this is useful is that due to minor differences
  34 in library versions and perhaps roundoff in different CPU's, the testsuite
  35 may falsely report failures on some systems.  These reported failures
  36 prevent using 'distcheck' for verifying the rest of the build system.
  37 These comments only apply to the tests which try to compare image files
  38 like PNG files.
  39
  40 EOF
  41
  42         exit 0
  43 fi
  44
  45 regen=no
  46
  47 usage() {
  48 cat <<EOF
  49
  50 $0 -- Run pcb regression tests
  51
  52 $0 -h|--help
  53 $0 [-d | --debug] [-g | --golden dir] [-r|--regen] [testname1 [testname2[ ...]]]
  54
  55 OVERVIEW
  56
  57 The $0 script is used both for running the pcb regression testsuite
  58 as well as maintaining it.  The way the test suite works is a number
  59 of different layouts are exported using the various export HID's.
  60
  61 The resulting output files are examined in various ways to make sure
  62 they are correct.  The exact details of how they are compared varies.
  63 For example, the PNG outputs are compared using tools from the ImageMagick
  64 suite while the ASCII centroid and bill of materials files are normalized
  65 with awk and then compared with the standard diff utility.  The normalization
  66 removes things like a comment line which contains the creation date.
  67
  68 OPTIONS
  69
  70 -d | --debug              Enables extra debug output
  71
  72 -g | --golden <dir>    :  Specifies that <dir> should be used for the
  73                           reference files.
  74
  75 LIMITATIONS
  76
  77 - The GUI interface is not checked via the regression testsuite.
  78
  79 - Currently actions are also not exercised
  80
  81 EOF
  82 }
  83
  84 show_sep() {
  85     echo "----------------------------------------------------------------------"
  86 }
  87
  88 ##########################################################################
  89 #
  90 # debug print out
  91 #
  92
  93 do_debug=no
  94 debug() {
  95     if test $do_debug = yes ; then
  96         cat <<EOF
  97 $*
  98 EOF
  99     fi
 100 }
 101
 102 ##########################################################################
 103 #
 104 # command line processing
 105 #
 106
 107
 108 all_tests=""
 109 while test -n "$1"
 110   do
 111   case "$1"
 112       in
 113
 114       -d|--debug)
 115           do_debug=yes
 116           shift
 117           ;;
 118
 119       -h|--help)
 120           usage
 121           exit 0
 122           ;;
 123
 124       -g|--golden)
 125         # set the 'golden' directory.
 126           REFDIR="$2"
 127           shift 2
 128           ;;
 129
 130       -r|--regen)
 131         # regenerate the 'golden' output files.  Use this with caution.
 132         # In particular, all differences should be noted and understood.
 133           regen=yes
 134           shift
 135           ;;
 136
 137       -*)
 138           echo "unknown option: $1"
 139           exit 1
 140           ;;
 141
 142       *)
 143           break
 144           ;;
 145
 146   esac
 147 done
 148
 149 if test "X$regen" = "Xyes" && test $# -ne 1; then
 150     echo "Please regenerate only one test at a time."
 151     echo "This limitation is a safety measure."
 152     exit 1
 153 fi
 154
 155 ##########################################################################
 156 #
 157 # set up various tools
 158 #
 159
 160
 161 all_tests="$*"
 162
 163 # Source directory
 164 srcdir=${srcdir:-.}
 165 top_srcdir=${top_srcdir:-.}
 166
 167 # The pcb wrapper script we want to test
 168 #
 169 # by default we will be running it from $(top_builddir)/tests/outputs/testname
 170 # so we need to look 3 levels up and then down to src
 171 PCB=${PCB:-../../../src/pcbtest.sh}
 172
 173 # The gerbv executible
 174 GERBV=${GERBV:-gerbv}
 175 GERBV_DEFAULT_FLAGS=${GERBV_DEFAULT_FLAGS:---export=png --window=640x480}
 176
 177 # various ImageMagick tools
 178 IM_ANIMATE=${IM_ANIMATE:-animate}
 179 IM_COMPARE=${IM_COMPARE:-compare}
 180 IM_COMPOSITE=${IM_COMPOSITE:-composite}
 181 IM_CONVERT=${IM_CONVERT:-convert}
 182 IM_DISPLAY=${IM_DISPLAY:-display}
 183 IM_MONTAGE=${IM_MONTAGE:-montage}
 184
 185 # golden directories
 186 INDIR=${INDIR:-${srcdir}/inputs}
 187 OUTDIR=outputs
 188 REFDIR=${REFDIR:-${srcdir}/golden}
 189
 190 # some system tools
 191 AWK=${AWK:-awk}
 192
 193 # the list of tests to run
 194 TESTLIST=${srcdir}/tests.list
 195
 196 if test "X$regen" = "Xyes" ; then
 197     OUTDIR="${REFDIR}"
 198 fi
 199
 200 # create output directory
 201 if test ! -d $OUTDIR ; then
 202     mkdir -p $OUTDIR
 203     if test $? -ne 0 ; then
 204         echo "Failed to create output directory ${OUTDIR}"
 205         exit 1
 206     fi
 207 fi
 208
 209 if test -z "$all_tests" ; then
 210     if test ! -f ${TESTLIST} ; then
 211         echo "ERROR: ($0)  Test list $TESTLIST does not exist"
 212         exit 1
 213     fi
 214     all_tests=`${AWK} 'BEGIN{FS="|"} /^#/{next} {print $1}' ${TESTLIST} | sed 's; ;;g'`
 215 fi
 216
 217 if test -z "${all_tests}" ; then
 218     echo "$0:  No tests specified"
 219     exit 0
 220 fi
 221
 222
 223 # fail/pass/total counts
 224 fail=0
 225 pass=0
 226 skip=0
 227 tot=0
 228
 229 ##########################################################################
 230 #
 231 # config summary
 232 #
 233
 234
 235 cat << EOF
 236
 237 srcdir                ${srcdir}
 238 top_srcdir            ${top_srcdir}
 239
 240 AWK                   ${AWK}
 241 PCB                   ${PCB}
 242 INDIR                 ${INDIR}
 243 OUTDIR                ${OUTDIR}
 244 REFDIR                ${REFDIR}
 245 TESTLIST              ${TESTLIST}
 246
 247 Gerbv:
 248
 249 GERBV                 ${GERBV}
 250 GERBV_DEFAULT_FLAGS   ${GERBV_DEFAULT_FLAGS}
 251
 252 ImageMagick Tools:
 253
 254 IM_ANIMATE               ${IM_ANIMATE}
 255 IM_COMPARE               ${IM_COMPARE}
 256 IM_COMPOSITE             ${IM_COMPOSITE}
 257 IM_CONVERT               ${IM_CONVERT}
 258 IM_DISPLAY               ${IM_DISPLAY}
 259 IM_MONTAGE               ${IM_MONTAGE}
 260
 261 EOF
 262
 263 tmpd=/tmp/pcb_tests.$$
 264 mkdir -p -m 0700 $tmpd
 265 rc=$?
 266 if test $rc -ne 0 ; then
 267     echo "$0:  ERROR:  could not create $tmpd"
 268     exit 1
 269 fi
 270
 271 ##########################################################################
 272 #
 273 # utility functions for comparison
 274 #
 275
 276 # Usage:
 277 #  compare_check "test_name" "file1" "file2"
 278 #
 279 # Makes sure that file1 and file2 both exist.  If not, mark the current
 280 # test as skipped and give an error message
 281 #
 282 compare_check() {
 283     local fn="$1"
 284     local f1="$2"
 285     local f2="$3"
 286
 287     if test ! -f "$f1" ; then
 288         echo "$0:  ${fn}(): $f1 does not exist"
 289         test_skipped=yes
 290         return 1
 291     fi
 292
 293     if test ! -f "$f2" ; then
 294         echo "$0:  ${fn}(): $f2 does not exist"
 295         test_skipped=yes
 296         return 1
 297     fi
 298     return 0
 299 }
 300
 301 ##########################################################################
 302 #
 303 # ASCII file comparison routines
 304 #
 305
 306 # Usage:
 307 #   run_diff "file1" "file2"
 308 #
 309 run_diff() {
 310     local f1="$1"
 311     local f2="$2"
 312     diff -U 2 $f1 $f2
 313     if test $? -ne 0 ; then return 1 ; fi
 314     return 0
 315 }
 316
 317 ##########################################################################
 318 #
 319 # BOM comparison routines
 320 #
 321
 322 # used to remove things like creation date from BOM files
 323 normalize_bom() {
 324     local f1="$1"
 325     local f2="$2"
 326     $AWK '
 327         /^# Date:/ {print "# Date: today"; next}
 328         /^# Author:/ {print "# Author: PCB"; next}
 329         {print}' \
 330         $f1 > $f2
 331 }
 332
 333 # top level function to compare BOM output
 334 compare_bom() {
 335     local f1="$1"
 336     local f2="$2"
 337     compare_check "compare_bom" "$f1" "$f2" || return 1
 338
 339     # an example BOM file is:
 340
 341     #  # PcbBOM Version 1.0
 342     #  # Date: Wed Jun 17 14:41:43 2009 UTC
 343     #  # Author: Dan McMahill
 344     #  # Title: Basic BOM/XY Test - PCB BOM
 345     #  # Quantity, Description, Value, RefDes
 346     #  # --------------------------------------------
 347     #  8,"Standard SMT resistor, capacitor etc","RESC3216N",R90_TOP R180_TOP R270_TOP R0_TOP R270_BOT R180_BOT R90_BOT R0_BOT
 348     #  8,"Dual in-line package, narrow (300 mil)","DIP8",UDIP90_TOP UDIP180_TOP UDIP270_TOP UDIP0_TOP UDIP270_BOT UDIP180_BOT UDIP90_BOT UDIP0_BOT
 349     #  8,"Small outline package, narrow (150mil)","SO8",USO90_TOP USO180_TOP USO270_TOP USO0_TOP USO270_BOT USO180_BOT USO90_BOT USO0_BOT
 350
 351     #  For comparison, we need to ignore changes in the Date and Author lines.
 352     local cf1=${tmpd}/`basename $f1`-ref
 353     local cf2=${tmpd}/`basename $f2`-out
 354
 355     normalize_bom $f1 $cf1
 356     normalize_bom $f2 $cf2
 357     run_diff $cf1 $cf2 || test_failed=yes
 358 }
 359
 360 ##########################################################################
 361 #
 362 # X-Y (centroid) comparison routines
 363 #
 364
 365 # used to remove things like creation date from BOM files
 366 normalize_xy() {
 367     local f1="$1"
 368     local f2="$2"
 369     $AWK '
 370         /^# Date:/ {print "# Date: today"; next}
 371         /^# Author:/ {print "# Author: PCB"; next}
 372         {print}' \
 373         $f1 > $f2
 374 }
 375
 376 compare_xy() {
 377     local f1="$1"
 378     local f2="$2"
 379     compare_check "compare_xy" "$f1" "$f2" || return 1
 380
 381     local cf1=${tmpd}/`basename $f1`-ref
 382     local cf2=${tmpd}/`basename $f2`-out
 383     normalize_xy "$f1" "$cf1"
 384     normalize_xy "$f2" "$cf2"
 385     run_diff "$cf1" "$cf2" || test_failed=yes
 386 }
 387
 388 ##########################################################################
 389 #
 390 # GCODE comparison routines
 391 #
 392
 393 # used to remove things like creation date from gcode files
 394 normalize_gcode() {
 395     local f1="$1"
 396     local f2="$2"
 397     # matches string such as '( Tue Mar  9 17:45:43 2010 )'
 398     $AWK --posix '!/^\( *[A-Z][a-z]{2} [A-Z][a-z]{2} [0123 ][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4} *\)$/' \
 399         $f1 > $f2
 400 }
 401
 402 compare_gcode() {
 403     local f1="$1"
 404     local f2="$2"
 405     compare_check "compare_gcode" "$f1" "$f2" || return 1
 406
 407     #  For comparison, we need to ignore changes in the Date and Author lines.
 408     local cf1=${tmpd}/`basename $f1`-ref
 409     local cf2=${tmpd}/`basename $f2`-out
 410
 411     normalize_gcode $f1 $cf1
 412     normalize_gcode $f2 $cf2
 413
 414     run_diff "$cf1" "$cf2" || test_failed=yes
 415 }
 416
 417 ##########################################################################
 418 #
 419 # RS274-X and Excellon comparison
 420 #
 421
 422 compare_rs274x() {
 423     local f1="$1"
 424     local f2="$2"
 425     compare_check "compare_rs274x" "$f1" "$f2" || return 1
 426
 427     # use gerbv to export our reference RS274-X file and our generated
 428     # RS274-X file to png.  Then we'll use ImageMagick to see
 429     # if there are any differences in the resulting files
 430     pngdir=${rundir}/png
 431     mkdir -p ${pngdir}
 432     nb=`basename $f1`
 433     png1=${pngdir}/${nb}-ref.png
 434     png2=${pngdir}/${nb}-out.png
 435
 436     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}"
 437     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}
 438
 439     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}"
 440     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}
 441
 442     compare_image ${png1} ${png2}
 443
 444 }
 445
 446 compare_cnc() {
 447     compare_rs274x $*
 448 }
 449
 450 ##########################################################################
 451 #
 452 # GIF/JPEG/PNG comparison routines
 453 #
 454
 455 compare_image() {
 456     local f1="$1"
 457     local f2="$2"
 458     compare_check "compare_image" "$f1" "$f2" || return 1
 459
 460     # now see if the image files are the same
 461     debug "${IM_COMPARE} -metric MAE ${f1} ${f2}  null:"
 462     same=`${IM_COMPARE} -metric MAE ${f1} ${f2}  null: 2>&1 | \
 463           ${AWK} '{if($1 == 0){print "yes"} else {print "no"}}'`
 464     debug "compare_image():  same = $same"
 465
 466     if test "$same" != yes ; then
 467         test_failed=yes
 468         echo "FAILED:  See ${errdir}"
 469         mkdir -p ${errdir}
 470         ${IM_COMPARE} ${f1} ${f2} ${errdir}/compare.png
 471         ${IM_COMPOSITE} ${f1} ${f2} -compose difference ${errdir}/composite.png
 472         ${IM_CONVERT} ${f1} ${f2} -compose difference -composite  -colorspace gray   ${errdir}/gray.png
 473         cat > ${errdir}/animate.sh << EOF
 474 #!/bin/sh
 475 ${IM_CONVERT} -label "%f" ${f1} ${f2} miff:- | \
 476 ${IM_MONTAGE} - -geometry +0+0 -tile 1x1 miff:- | \
 477 ${IM_ANIMATE} -delay 0.5 -loop 0 -
 478 EOF
 479         chmod a+x ${errdir}/animate.sh
 480     fi
 481 }
 482
 483 ##########################################################################
 484 #
 485 # The main program loop
 486 #
 487
 488 for t in $all_tests ; do
 489     show_sep
 490     echo "Test:  $t"
 491
 492     tot=`expr $tot + 1`
 493
 494     ######################################################################
 495     #
 496     # extract the details for the test
 497     #
 498
 499     pcb_flags="${PCB_DEFAULT_FLAGS}"
 500
 501     rundir="${OUTDIR}/${t}"
 502     refdir="${REFDIR}/${t}"
 503     errdir=${rundir}/mismatch
 504
 505     # test_name | layout file(s) | [export hid name] | [optional arguments to pcb] |  [mismatch]
 506     # | output files
 507     name=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $1}'`
 508     files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $2}'`
 509     hid=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {gsub(/[ \t]*/, ""); print $3}'`
 510     args=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $4}'`
 511     mismatch=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {if($5 == "mismatch"){print "yes"}else{print "no"}}'`
 512     out_files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $6}'`
 513
 514     if test "X${name}" = "X" ; then
 515         echo "ERROR:  Specified test ${t} does not appear to exist"
 516         skip=`expr $skip + 1`
 517         continue
 518     fi
 519
 520     if test "X${args}" != "X" ; then
 521         pcb_flags="${args}"
 522     fi
 523
 524     if test "X${hid}" = "Xgerber" ; then
 525         pcb_flags="--fab-author Fab_Author ${pcb_flags}"
 526     fi
 527
 528     ######################################################################
 529     #
 530     # Set up the run directory
 531     #
 532
 533     test -d ${rundir} && rm -fr ${rundir}
 534     mkdir -p ${rundir}
 535     if test $? -ne 0 ; then
 536         echo "$0:  Could not create run directory ${rundir}"
 537         skip=`expr $skip + 1`
 538         continue
 539     fi
 540
 541
 542     ######################################################################
 543     #
 544     # check to see if the files we need exist and copy them to the run
 545     # directory
 546     #
 547
 548     missing_files=no
 549     path_files=""
 550     for f in $files ; do
 551         if test ! -f ${INDIR}/${f} ; then
 552             echo "ERROR:  File $f specified as part of the $t test does not exist"
 553             missing_files=yes
 554         else
 555             path_files="${path_files} ${INDIR}/${f}"
 556             cp "${INDIR}/${f}" "${rundir}"
 557         fi
 558     done
 559
 560     if test "$missing_files" = "yes" ; then
 561         echo "${t} had missing input files.  Skipping test"
 562         skip=`expr $skip + 1`
 563         continue
 564     fi
 565
 566     ######################################################################
 567     #
 568     # run PCB
 569     #
 570
 571     echo "${PCB} -x ${hid} ${pcb_flags} ${path_files}"
 572     (cd ${rundir} && ${PCB} -x ${hid} ${pcb_flags} ${files})
 573     pcb_rc=$?
 574
 575     if test $pcb_rc -ne 0 ; then
 576         echo "${PCB} returned ${pcb_rc}.  This is a failure."
 577         fail=`expr $fail + 1`
 578         continue
 579     fi
 580
 581     # and clean up the input files we'd copied over
 582     for f in $files ; do
 583         rm -f ${rundir}/${f}
 584     done
 585
 586     ######################################################################
 587     #
 588     # check the result
 589     #
 590
 591     if test "X$regen" = "Xyes" ; then
 592         echo    "## -*- makefile -*-"   > ${rundir}/Makefile.am
 593         echo                           >> ${rundir}/Makefile.am
 594         echo -n "EXTRA_DIST="          >> ${rundir}/Makefile.am
 595         for f in $out_files ; do
 596             fn=`echo $f | sed 's;.*:;;g'`
 597             echo    " \\"              >> ${rundir}/Makefile.am
 598             echo -n "\t$fn"            >> ${rundir}/Makefile.am
 599         done
 600         echo                           >> ${rundir}/Makefile.am
 601
 602         echo "Regenerated ${t}"
 603     else
 604         # compare the result to our reference file
 605         test_failed=no
 606         test_skipped=no
 607         for f in $out_files ; do
 608             debug "processing $f"
 609             # break apart type:fn into the type and file name
 610             type=`echo $f | sed 's;:.*;;g'`
 611             fn=`echo $f | sed 's;.*:;;g'`
 612
 613             case $type in
 614                 # BOM HID
 615                 bom)
 616                     compare_bom ${refdir}/${fn} ${rundir}/${fn}
 617                     ;;
 618
 619                 xy)
 620                     compare_xy ${refdir}/${fn} ${rundir}/${fn}
 621                     ;;
 622
 623                 # GCODE HID
 624                 gcode)
 625                     compare_gcode ${refdir}/${fn} ${rundir}/${fn}
 626                     ;;
 627
 628                 # GERBER HID
 629                 cnc)
 630                     compare_cnc ${refdir}/${fn} ${rundir}/${fn}
 631                     ;;
 632
 633                 gbx)
 634                     compare_rs274x ${refdir}/${fn} ${rundir}/${fn}
 635                     ;;
 636
 637                 # PNG HID
 638                 gif)
 639                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 640                     ;;
 641
 642                 jpg)
 643                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 644                     ;;
 645
 646                 png)
 647                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 648                     ;;
 649
 650                 # unknown
 651                 *)
 652                     echo "internal error:  $type is not a known file type"
 653                     exit 1
 654                     ;;
 655             esac
 656
 657         done
 658
 659
 660         if test $test_failed = yes ; then
 661             echo "FAIL"
 662             fail=`expr $fail + 1`
 663         elif test $test_skipped = yes ; then
 664             echo "SKIPPED"
 665             skip=`expr $skip + 1`
 666         else
 667             echo "PASSED"
 668             pass=`expr $pass + 1`
 669         fi
 670
 671     fi
 672
 673 done
 674
 675 show_sep
 676 echo "Passed $pass, failed $fail, skipped $skip out of $tot tests."
 677
 678 rc=0
 679 if test $pass -ne $tot ; then
 680     rc=1
 681 fi
 682
 683 exit $rc
 684