tests/run_tests.sh

   1 #!/bin/sh
   2 #
   3 # $Id: run_tests.sh,v 1.8 2009/02/17 00:42:31 danmc Exp $
   4 #
   5 # Copyright (c) 2003, 2004, 2005, 2006, 2007, 2009, 2010 Dan McMahill
   6
   7 #  This program is free software; you can redistribute it and/or modify
   8 #  it under the terms of version 2 of the GNU General Public License as
   9 #  published by the Free Software Foundation
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU General Public License
  17 #  along with this program; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111 USA
  19 # All rights reserved.
  20 #
  21 # This code was derived from code written by Dan McMahill as part of the
  22 # latex-mk testsuite.  The original code was covered by a BSD license
  23 # but the copyright holder (Dan McMahill) has re-released the code under
  24 # the GPL.
  25
  26 magic_test_skip=${PCB_MAGIC_TEST_SKIP:-no}
  27
  28 if test "x${magic_test_skip}" = "xyes" ; then
  29         cat << EOF
  30
  31 The environment variable PCB_MAGIC_TEST_SKIP is set to yes.
  32 This causes the testsuite to skip all tests and report no errors.
  33 This is used for certain debugging *only*.  The primary use is to
  34 allow testing of the 'distcheck' target without including the effects
  35 of the testsuite. The reason this is useful is that due to minor differences
  36 in library versions and perhaps roundoff in different CPU's, the testsuite
  37 may falsely report failures on some systems.  These reported failures
  38 prevent using 'distcheck' for verifying the rest of the build system.
  39 These comments only apply to the tests which try to compare image files
  40 like PNG files.
  41
  42 EOF
  43
  44         exit 0
  45 fi
  46
  47 regen=no
  48
  49 usage() {
  50 cat <<EOF
  51
  52 $0 -- Run pcb regression tests
  53
  54 $0 -h|--help
  55 $0 [-d | --debug] [-g | --golden dir] [-r|--regen] [testname1 [testname2[ ...]]]
  56
  57 OVERVIEW
  58
  59 The $0 script is used both for running the pcb regression testsuite
  60 as well as maintaining it.  The way the test suite works is a number
  61 of different layouts are exported using the various export HID's.
  62
  63 The resulting output files are examined in various ways to make sure
  64 they are correct.  The exact details of how they are compared varies.
  65 For example, the PNG outputs are compared using tools from the ImageMagick
  66 suite while the ASCII centroid and bill of materials files are normalized
  67 with awk and then compared with the standard diff utility.  The normalization
  68 removes things like a comment line which contains the creation date.
  69
  70 OPTIONS
  71
  72 -d | --debug              Enables extra debug output
  73
  74 -g | --golden <dir>    :  Specifies that <dir> should be used for the
  75                           reference files.
  76
  77 LIMITATIONS
  78
  79 - The GUI interface is not checked via the regression testsuite.
  80
  81 - Currently actions are also not exercised
  82
  83 EOF
  84 }
  85
  86 show_sep() {
  87     echo "----------------------------------------------------------------------"
  88 }
  89
  90 ##########################################################################
  91 #
  92 # debug print out
  93 #
  94
  95 do_debug=no
  96 debug() {
  97     if test $do_debug = yes ; then
  98         cat <<EOF
  99 $*
 100 EOF
 101     fi
 102 }
 103
 104 ##########################################################################
 105 #
 106 # command line processing
 107 #
 108
 109
 110 all_tests=""
 111 while test -n "$1"
 112   do
 113   case "$1"
 114       in
 115
 116       -d|--debug)
 117           do_debug=yes
 118           shift
 119           ;;
 120
 121       -h|--help)
 122           usage
 123           exit 0
 124           ;;
 125
 126       -g|--golden)
 127         # set the 'golden' directory.
 128           REFDIR="$2"
 129           shift 2
 130           ;;
 131
 132       -r|--regen)
 133         # regenerate the 'golden' output files.  Use this with caution.
 134         # In particular, all differences should be noted and understood.
 135           regen=yes
 136           shift
 137           ;;
 138
 139       -*)
 140           echo "unknown option: $1"
 141           exit 1
 142           ;;
 143
 144       *)
 145           break
 146           ;;
 147
 148   esac
 149 done
 150
 151 ##########################################################################
 152 #
 153 # set up various tools
 154 #
 155
 156
 157 all_tests="$*"
 158
 159 # Source directory
 160 srcdir=${srcdir:-.}
 161 top_srcdir=${top_srcdir:-.}
 162
 163 # The pcb wrapper script we want to test
 164 #
 165 # by default we will be running it from $(top_builddir)/tests/outputs/testname
 166 # so we need to look 3 levels up and then down to src
 167 PCB=${PCB:-../../../src/pcbtest.sh}
 168
 169 # The gerbv executible
 170 GERBV=${GERBV:-gerbv}
 171 GERBV_DEFAULT_FLAGS=${GERBV_DEFAULT_FLAGS:---export=png --window=640x480}
 172
 173 # various ImageMagick tools
 174 IM_ANIMATE=${IM_ANIMATE:-animate}
 175 IM_COMPARE=${IM_COMPARE:-compare}
 176 IM_COMPOSITE=${IM_COMPOSITE:-composite}
 177 IM_CONVERT=${IM_CONVERT:-convert}
 178 IM_DISPLAY=${IM_DISPLAY:-display}
 179 IM_MONTAGE=${IM_MONTAGE:-montage}
 180
 181 # golden directories
 182 INDIR=${INDIR:-${srcdir}/inputs}
 183 OUTDIR=outputs
 184 REFDIR=${REFDIR:-${srcdir}/golden}
 185
 186 # some system tools
 187 AWK=${AWK:-awk}
 188
 189 # the list of tests to run
 190 TESTLIST=${srcdir}/tests.list
 191
 192 if test "X$regen" = "Xyes" ; then
 193     OUTDIR="${REFDIR}"
 194 fi
 195
 196 # create output directory
 197 if test ! -d $OUTDIR ; then
 198     mkdir -p $OUTDIR
 199     if test $? -ne 0 ; then
 200         echo "Failed to create output directory ${OUTDIR}"
 201         exit 1
 202     fi
 203 fi
 204
 205 if test -z "$all_tests" ; then
 206     if test ! -f ${TESTLIST} ; then
 207         echo "ERROR: ($0)  Test list $TESTLIST does not exist"
 208         exit 1
 209     fi
 210     all_tests=`${AWK} 'BEGIN{FS="|"} /^#/{next} {print $1}' ${TESTLIST} | sed 's; ;;g'`
 211 fi
 212
 213 if test -z "${all_tests}" ; then
 214     echo "$0:  No tests specified"
 215     exit 0
 216 fi
 217
 218
 219 # fail/pass/total counts
 220 fail=0
 221 pass=0
 222 skip=0
 223 tot=0
 224
 225 ##########################################################################
 226 #
 227 # config summary
 228 #
 229
 230
 231 cat << EOF
 232
 233 srcdir                ${srcdir}
 234 top_srcdir            ${top_srcdir}
 235
 236 AWK                   ${AWK}
 237 PCB                   ${PCB}
 238 INDIR                 ${INDIR}
 239 OUTDIR                ${OUTDIR}
 240 REFDIR                ${REFDIR}
 241 TESTLIST              ${TESTLIST}
 242
 243 Gerbv:
 244
 245 GERBV                 ${GERBV}
 246 GERBV_DEFAULT_FLAGS   ${GERBV_DEFAULT_FLAGS}
 247
 248 ImageMagick Tools:
 249
 250 IM_ANIMATE               ${IM_ANIMATE}
 251 IM_COMPARE               ${IM_COMPARE}
 252 IM_COMPOSITE             ${IM_COMPOSITE}
 253 IM_CONVERT               ${IM_CONVERT}
 254 IM_DISPLAY               ${IM_DISPLAY}
 255 IM_MONTAGE               ${IM_MONTAGE}
 256
 257 EOF
 258
 259 tmpd=/tmp/pcb_tests.$$
 260 mkdir -p -m 0700 $tmpd
 261 rc=$?
 262 if test $rc -ne 0 ; then
 263     echo "$0:  ERROR:  could not create $tmpd"
 264     exit 1
 265 fi
 266
 267 ##########################################################################
 268 #
 269 # utility functions for comparison
 270 #
 271
 272 # Usage:
 273 #  compare_check "test_name" "file1" "file2"
 274 #
 275 # Makes sure that file1 and file2 both exist.  If not, mark the current
 276 # test as skipped and give an error message
 277 #
 278 compare_check() {
 279     fn="$1"
 280     f1="$2"
 281     f2="$3"
 282
 283     if test ! -f "$f2" ; then
 284         echo "$0:  ${fn}(): $f1 does not exist"
 285         test_skipped=yes
 286         return 1
 287     fi
 288
 289     if test ! -f "$f2" ; then
 290         echo "$0:  ${fn}(): $f2 does not exist"
 291         test_skipped=yes
 292         return 1
 293     fi
 294     return 0
 295 }
 296
 297 ##########################################################################
 298 #
 299 # ASCII file comparison routines
 300 #
 301
 302 # Usage:
 303 #   run_diff "file1" "file2"
 304 #
 305 run_diff() {
 306     f1="$1"
 307     f2="$2"
 308     diff -U 2 $f1 $f2
 309     if test $? -ne 0 ; then return 1 ; fi
 310     return 0
 311 }
 312
 313 ##########################################################################
 314 #
 315 # BOM comparison routines
 316 #
 317
 318 # used to remove things like creation date from BOM files
 319 normalize_bom() {
 320     f1="$1"
 321     f2="$2"
 322     $AWK '
 323         /^# Date:/ {print "# Date: today"; next}
 324         /^# Author:/ {print "# Author: PCB"; next}
 325         {print}' \
 326         $f1 > $f2
 327 }
 328
 329 # top level function to compare BOM output
 330 compare_bom() {
 331     f1="$1"
 332     f2="$2"
 333     compare_check "compare_bom" "$f1" "$f2" || return 1
 334
 335     # an example BOM file is:
 336
 337     #  # $Id$
 338     #  # PcbBOM Version 1.0
 339     #  # Date: Wed Jun 17 14:41:43 2009 UTC
 340     #  # Author: Dan McMahill
 341     #  # Title: Basic BOM/XY Test - PCB BOM
 342     #  # Quantity, Description, Value, RefDes
 343     #  # --------------------------------------------
 344     #  8,"Standard SMT resistor, capacitor etc","RESC3216N",R90_TOP R180_TOP R270_TOP R0_TOP R270_BOT R180_BOT R90_BOT R0_BOT
 345     #  8,"Dual in-line package, narrow (300 mil)","DIP8",UDIP90_TOP UDIP180_TOP UDIP270_TOP UDIP0_TOP UDIP270_BOT UDIP180_BOT UDIP90_BOT UDIP0_BOT
 346     #  8,"Small outline package, narrow (150mil)","SO8",USO90_TOP USO180_TOP USO270_TOP USO0_TOP USO270_BOT USO180_BOT USO90_BOT USO0_BOT
 347
 348     #  For comparison, we need to ignore changes in the Date and Author lines.
 349     cf1=${tmpd}/`basename $f1`
 350     cf2=${tmpd}/`basename $f2`
 351
 352     normalize_bom $f1 $cf1
 353     normalize_bom $f2 $cf2
 354     run_diff $cf1 $cf2 || test_failed=yes
 355 }
 356
 357 ##########################################################################
 358 #
 359 # X-Y (centroid) comparison routines
 360 #
 361
 362 # used to remove things like creation date from BOM files
 363 normalize_xy() {
 364     f1="$1"
 365     f2="$2"
 366     $AWK '
 367         /^# Date:/ {print "# Date: today"; next}
 368         /^# Author:/ {print "# Author: PCB"; next}
 369         {print}' \
 370         $f1 > $f2
 371 }
 372
 373 compare_xy() {
 374     f1="$1"
 375     f2="$2"
 376     compare_check "compare_xy" "$f1" "$f2" || return 1
 377     normalize_xy "$f1" "$cf1"
 378     normalize_xy "$f2" "$cf2"
 379     run_diff "$cf1" "$cf2" || test_failed=yes
 380 }
 381
 382 ##########################################################################
 383 #
 384 # GCODE comparison routines
 385 #
 386
 387 # used to remove things like creation date from gcode files
 388 normalize_gcode() {
 389     f1="$1"
 390     f2="$2"
 391     $AWK '
 392         d == 1 {gsub(/ .* /, "Creation Date and Time"); d = 0;}
 393         /^\(Created by G-code exporter\)/ {d=1}
 394         {print}' \
 395         $f1 > $f2
 396 }
 397
 398 compare_gcode() {
 399     f1="$1"
 400     f2="$2"
 401     compare_check "compare_gcode" "$f1" "$f2" || return 1
 402
 403     #  For comparison, we need to ignore changes in the Date and Author lines.
 404     cf1=${tmpd}/`basename $f1`
 405     cf2=${tmpd}/`basename $f2`
 406
 407     normalize_gcode $f1 $cf1
 408     normalize_gcode $f2 $cf2
 409     run_diff "$cf1" "$cf2" || test_failed=yes
 410 }
 411
 412 ##########################################################################
 413 #
 414 # RS274-X and Excellon comparison
 415 #
 416
 417 compare_rs274x() {
 418     f1="$1"
 419     f2="$2"
 420     compare_check "compare_rs274x" "$f1" "$f2" || return 1
 421
 422     # use gerbv to export our reference RS274-X file and our generated
 423     # RS274-X file to png.  Then we'll use ImageMagick to see
 424     # if there are any differences in the resulting files
 425     pngdir=${rundir}/png
 426     mkdir -p ${pngdir}
 427     nb=`basename $f1`
 428     png1=${pngdir}/${nb}-ref.png
 429     png2=${pngdir}/${nb}-out.png
 430
 431     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}"
 432     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}
 433
 434     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}"
 435     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}
 436
 437     compare_image ${png1} ${png2}
 438
 439 }
 440
 441 compare_cnc() {
 442     compare_rs274x $*
 443 }
 444
 445 ##########################################################################
 446 #
 447 # GIF/JPEG/PNG comparison routines
 448 #
 449
 450 compare_image() {
 451     f1="$1"
 452     f2="$2"
 453     compare_check "compare_image" "$f1" "$f2" || return 1
 454
 455     # now see if the image files are the same
 456     debug "${IM_COMPARE} -metric MAE ${f1} ${f2}  null:"
 457     same=`${IM_COMPARE} -metric MAE ${f1} ${f2}  null: 2>&1 | \
 458           ${AWK} '{if($1 == 0){print "yes"} else {print "no"}}'`
 459     debug "compare_image():  same = $same"
 460
 461     if test "$same" != yes ; then
 462         test_failed=yes
 463         echo "FAILED:  See ${errdir}"
 464         mkdir -p ${errdir}
 465         ${IM_COMPARE} ${f1} ${f2} ${errdir}/compare.png
 466         ${IM_COMPOSITE} ${f1} ${f2} -compose difference ${errdir}/composite.png
 467         ${IM_CONVERT} ${f1} ${f2} -compose difference -composite  -colorspace gray   ${errdir}/gray.png
 468         cat > ${errdir}/animate.sh << EOF
 469 #!/bin/sh
 470 ${IM_CONVERT} -label "%f" ${f1} ${f2} miff:- | \
 471 ${IM_MONTAGE} - -geometry +0+0 -tile 1x1 miff:- | \
 472 ${IM_ANIMATE} -delay 0.5 -loop 0 -
 473 EOF
 474         chmod a+x ${errdir}/animate.sh
 475     fi
 476 }
 477
 478 ##########################################################################
 479 #
 480 # The main program loop
 481 #
 482
 483 for t in $all_tests ; do
 484     show_sep
 485     echo "Test:  $t"
 486
 487     tot=`expr $tot + 1`
 488
 489     ######################################################################
 490     #
 491     # extract the details for the test
 492     #
 493
 494     pcb_flags="${PCB_DEFAULT_FLAGS}"
 495
 496     rundir="${OUTDIR}/${t}"
 497     refdir="${REFDIR}/${t}"
 498     errdir=${rundir}/mismatch
 499
 500     # test_name | layout file(s) | [export hid name] | [optional arguments to pcb] |  [mismatch]
 501     # | output files
 502     name=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $1}'`
 503     files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $2}'`
 504     hid=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {gsub(/[ \t]*/, ""); print $3}'`
 505     args=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $4}'`
 506     mismatch=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {if($5 == "mismatch"){print "yes"}else{print "no"}}'`
 507     out_files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $6}'`
 508
 509     if test "X${name}" = "X" ; then
 510         echo "ERROR:  Specified test ${t} does not appear to exist"
 511         skip=`expr $skip + 1`
 512         continue
 513     fi
 514
 515     if test "X${args}" != "X" ; then
 516         pcb_flags="${args}"
 517     fi
 518
 519     if test "X${hid}" = "Xgerber" ; then
 520         pcb_flags="--fab-author Fab_Author ${pcb_flags}"
 521     fi
 522
 523     ######################################################################
 524     #
 525     # Set up the run directory
 526     #
 527
 528     test -d ${rundir} && rm -fr ${rundir}
 529     mkdir -p ${rundir}
 530     if test $? -ne 0 ; then
 531         echo "$0:  Could not create run directory ${rundir}"
 532         skip=`expr $skip + 1`
 533         continue
 534     fi
 535
 536
 537     ######################################################################
 538     #
 539     # check to see if the files we need exist and copy them to the run
 540     # directory
 541     #
 542
 543     missing_files=no
 544     path_files=""
 545     for f in $files ; do
 546         if test ! -f ${INDIR}/${f} ; then
 547             echo "ERROR:  File $f specified as part of the $t test does not exist"
 548             missing_files=yes
 549         else
 550             path_files="${path_files} ${INDIR}/${f}"
 551             cp "${INDIR}/${f}" "${rundir}"
 552         fi
 553     done
 554
 555     if test "$missing_files" = "yes" ; then
 556         echo "${t} had missing input files.  Skipping test"
 557         skip=`expr $skip + 1`
 558         continue
 559     fi
 560
 561     ######################################################################
 562     #
 563     # run PCB
 564     #
 565
 566     echo "${PCB} -x ${hid} ${pcb_flags} ${path_files}"
 567     (cd ${rundir} && ${PCB} -x ${hid} ${pcb_flags} ${files})
 568     pcb_rc=$?
 569
 570     if test $pcb_rc -ne 0 ; then
 571         echo "${PCB} returned ${pcb_rc}.  This is a failure."
 572         fail=`expr $fail + 1`
 573         continue
 574     fi
 575
 576     # and clean up the input files we'd copied over
 577     for f in $files ; do
 578         rm -f ${rundir}/${f}
 579     done
 580
 581     ######################################################################
 582     #
 583     # check the result
 584     #
 585
 586     if test "X$regen" = "Xyes" ; then
 587         echo "Regenerated ${t}"
 588     else
 589         # compare the result to our reference file
 590         test_failed=no
 591         test_skipped=no
 592         for f in $out_files ; do
 593             debug "processing $f"
 594             # break apart type:fn into the type and file name
 595             type=`echo $f | sed 's;:.*;;g'`
 596             fn=`echo $f | sed 's;.*:;;g'`
 597
 598             case $type in
 599                 # BOM HID
 600                 bom)
 601                     compare_bom ${refdir}/${fn} ${rundir}/${fn}
 602                     ;;
 603
 604                 xy)
 605                     compare_xy ${refdir}/${fn} ${rundir}/${fn}
 606                     ;;
 607
 608                 # GCODE HID
 609                 gcode)
 610                     compare_gcode ${refdir}/${fn} ${rundir}/${fn}
 611                     ;;
 612
 613                 # GERBER HID
 614                 cnc)
 615                     compare_cnc ${refdir}/${fn} ${rundir}/${fn}
 616                     ;;
 617
 618                 gbx)
 619                     compare_rs274x ${refdir}/${fn} ${rundir}/${fn}
 620                     ;;
 621
 622                 # PNG HID
 623                 gif)
 624                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 625                     ;;
 626
 627                 jpg)
 628                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 629                     ;;
 630
 631                 png)
 632                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 633                     ;;
 634
 635                 # unknown
 636                 *)
 637                     echo "internal error:  $type is not a known file type"
 638                     exit 1
 639                     ;;
 640             esac
 641
 642         done
 643
 644
 645         if test $test_failed = yes ; then
 646             echo "FAIL"
 647             fail=`expr $fail + 1`
 648         elif test $test_skipped = yes ; then
 649             echo "SKIPPED"
 650             skip=`expr $skip + 1`
 651         else
 652             echo "PASSED"
 653             pass=`expr $pass + 1`
 654         fi
 655
 656     fi
 657
 658 done
 659
 660 show_sep
 661 echo "Passed $pass, failed $fail, skipped $skip out of $tot tests."
 662
 663 rc=0
 664 if test $pass -ne $tot ; then
 665     rc=1
 666 fi
 667
 668 exit $rc
 669