tests/run_tests.sh

   1 #!/bin/sh
   2 #
   3 # $Id: run_tests.sh,v 1.8 2009/02/17 00:42:31 danmc Exp $
   4 #
   5 # Copyright (c) 2003, 2004, 2005, 2006, 2007, 2009, 2010 Dan McMahill
   6
   7 #  This program is free software; you can redistribute it and/or modify
   8 #  it under the terms of version 2 of the GNU General Public License as
   9 #  published by the Free Software Foundation
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU General Public License
  17 #  along with this program; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111 USA
  19 # All rights reserved.
  20 #
  21 # This code was derived from code written by Dan McMahill as part of the
  22 # latex-mk testsuite.  The original code was covered by a BSD license
  23 # but the copyright holder (Dan McMahill) has re-released the code under
  24 # the GPL.
  25
  26 magic_test_skip=${PCB_MAGIC_TEST_SKIP:-no}
  27
  28 if test "x${magic_test_skip}" = "xyes" ; then
  29         cat << EOF
  30
  31 The environment variable PCB_MAGIC_TEST_SKIP is set to yes.
  32 This causes the testsuite to skip all tests and report no errors.
  33 This is used for certain debugging *only*.  The primary use is to
  34 allow testing of the 'distcheck' target without including the effects
  35 of the testsuite. The reason this is useful is that due to minor differences
  36 in library versions and perhaps roundoff in different CPU's, the testsuite
  37 may falsely report failures on some systems.  These reported failures
  38 prevent using 'distcheck' for verifying the rest of the build system.
  39 These comments only apply to the tests which try to compare image files
  40 like PNG files.
  41
  42 EOF
  43
  44         exit 0
  45 fi
  46
  47 regen=no
  48
  49 usage() {
  50 cat <<EOF
  51
  52 $0 -- Run pcb regression tests
  53
  54 $0 -h|--help
  55 $0 [-d | --debug] [-g | --golden dir] [-r|--regen] [testname1 [testname2[ ...]]]
  56
  57 OVERVIEW
  58
  59 The $0 script is used both for running the pcb regression testsuite
  60 as well as maintaining it.  The way the test suite works is a number
  61 of different layouts are exported using the various export HID's.
  62
  63 The resulting output files are examined in various ways to make sure
  64 they are correct.  The exact details of how they are compared varies.
  65 For example, the PNG outputs are compared using tools from the ImageMagick
  66 suite while the ASCII centroid and bill of materials files are normalized
  67 with awk and then compared with the standard diff utility.  The normalization
  68 removes things like a comment line which contains the creation date.
  69
  70 OPTIONS
  71
  72 -d | --debug              Enables extra debug output
  73
  74 -g | --golden <dir>    :  Specifies that <dir> should be used for the
  75                           reference files.
  76
  77 LIMITATIONS
  78
  79 - The GUI interface is not checked via the regression testsuite.
  80
  81 - Currently actions are also not exercised
  82
  83 EOF
  84 }
  85
  86 show_sep() {
  87     echo "----------------------------------------------------------------------"
  88 }
  89
  90 ##########################################################################
  91 #
  92 # debug print out
  93 #
  94
  95 do_debug=no
  96 debug() {
  97     if test $do_debug = yes ; then
  98         cat <<EOF
  99 $*
 100 EOF
 101     fi
 102 }
 103
 104 ##########################################################################
 105 #
 106 # command line processing
 107 #
 108
 109
 110 all_tests=""
 111 while test -n "$1"
 112   do
 113   case "$1"
 114       in
 115
 116       -d|--debug)
 117           do_debug=yes
 118           shift
 119           ;;
 120
 121       -h|--help)
 122           usage
 123           exit 0
 124           ;;
 125
 126       -g|--golden)
 127         # set the 'golden' directory.
 128           REFDIR="$2"
 129           shift 2
 130           ;;
 131
 132       -r|--regen)
 133         # regenerate the 'golden' output files.  Use this with caution.
 134         # In particular, all differences should be noted and understood.
 135           regen=yes
 136           shift
 137           ;;
 138
 139       -*)
 140           echo "unknown option: $1"
 141           exit 1
 142           ;;
 143
 144       *)
 145           break
 146           ;;
 147
 148   esac
 149 done
 150
 151 ##########################################################################
 152 #
 153 # set up various tools
 154 #
 155
 156
 157 all_tests="$*"
 158
 159 # Source directory
 160 srcdir=${srcdir:-.}
 161 top_srcdir=${top_srcdir:-.}
 162
 163 # The pcb wrapper script we want to test
 164 #
 165 # by default we will be running it from $(top_builddir)/tests/outputs/testname
 166 # so we need to look 3 levels up and then down to src
 167 PCB=${PCB:-../../../src/pcbtest.sh}
 168
 169 # The gerbv executible
 170 GERBV=${GERBV:-gerbv}
 171 GERBV_DEFAULT_FLAGS=${GERBV_DEFAULT_FLAGS:---export=png --window=640x480}
 172
 173 # various ImageMagick tools
 174 IM_ANIMATE=${IM_ANIMATE:-animate}
 175 IM_COMPARE=${IM_COMPARE:-compare}
 176 IM_COMPOSITE=${IM_COMPOSITE:-composite}
 177 IM_CONVERT=${IM_CONVERT:-convert}
 178 IM_DISPLAY=${IM_DISPLAY:-display}
 179 IM_MONTAGE=${IM_MONTAGE:-montage}
 180
 181 # golden directories
 182 INDIR=${INDIR:-${srcdir}/inputs}
 183 OUTDIR=outputs
 184 REFDIR=${REFDIR:-${srcdir}/golden}
 185
 186 # some system tools
 187 AWK=${AWK:-awk}
 188
 189 # the list of tests to run
 190 TESTLIST=${srcdir}/tests.list
 191
 192 if test "X$regen" = "Xyes" ; then
 193     OUTDIR="${REFDIR}"
 194 fi
 195
 196 # create output directory
 197 if test ! -d $OUTDIR ; then
 198     mkdir -p $OUTDIR
 199     if test $? -ne 0 ; then
 200         echo "Failed to create output directory ${OUTDIR}"
 201         exit 1
 202     fi
 203 fi
 204
 205 if test -z "$all_tests" ; then
 206     if test ! -f ${TESTLIST} ; then
 207         echo "ERROR: ($0)  Test list $TESTLIST does not exist"
 208         exit 1
 209     fi
 210     all_tests=`${AWK} 'BEGIN{FS="|"} /^#/{next} {print $1}' ${TESTLIST} | sed 's; ;;g'`
 211 fi
 212
 213 if test -z "${all_tests}" ; then
 214     echo "$0:  No tests specified"
 215     exit 0
 216 fi
 217
 218
 219 # fail/pass/total counts
 220 fail=0
 221 pass=0
 222 skip=0
 223 tot=0
 224
 225 ##########################################################################
 226 #
 227 # config summary
 228 #
 229
 230
 231 cat << EOF
 232
 233 srcdir                ${srcdir}
 234 top_srcdir            ${top_srcdir}
 235
 236 AWK                   ${AWK}
 237 PCB                   ${PCB}
 238 INDIR                 ${INDIR}
 239 OUTDIR                ${OUTDIR}
 240 REFDIR                ${REFDIR}
 241 TESTLIST              ${TESTLIST}
 242
 243 Gerbv:
 244
 245 GERBV                 ${GERBV}
 246 GERBV_DEFAULT_FLAGS   ${GERBV_DEFAULT_FLAGS}
 247
 248 ImageMagick Tools:
 249
 250 IM_ANIMATE               ${IM_ANIMATE}
 251 IM_COMPARE               ${IM_COMPARE}
 252 IM_COMPOSITE             ${IM_COMPOSITE}
 253 IM_CONVERT               ${IM_CONVERT}
 254 IM_DISPLAY               ${IM_DISPLAY}
 255 IM_MONTAGE               ${IM_MONTAGE}
 256
 257 EOF
 258
 259 tmpd=/tmp/pcb_tests.$$
 260 mkdir -p -m 0700 $tmpd
 261 rc=$?
 262 if test $rc -ne 0 ; then
 263     echo "$0:  ERROR:  could not create $tmpd"
 264     exit 1
 265 fi
 266
 267 ##########################################################################
 268 #
 269 # utility functions for comparison
 270 #
 271
 272 # Usage:
 273 #  compare_check "test_name" "file1" "file2"
 274 #
 275 # Makes sure that file1 and file2 both exist.  If not, mark the current
 276 # test as skipped and give an error message
 277 #
 278 compare_check() {
 279     local fn="$1"
 280     local f1="$2"
 281     local f2="$3"
 282
 283     if test ! -f "$f1" ; then
 284         echo "$0:  ${fn}(): $f1 does not exist"
 285         test_skipped=yes
 286         return 1
 287     fi
 288
 289     if test ! -f "$f2" ; then
 290         echo "$0:  ${fn}(): $f2 does not exist"
 291         test_skipped=yes
 292         return 1
 293     fi
 294     return 0
 295 }
 296
 297 ##########################################################################
 298 #
 299 # ASCII file comparison routines
 300 #
 301
 302 # Usage:
 303 #   run_diff "file1" "file2"
 304 #
 305 run_diff() {
 306     local f1="$1"
 307     local f2="$2"
 308     diff -U 2 $f1 $f2
 309     if test $? -ne 0 ; then return 1 ; fi
 310     return 0
 311 }
 312
 313 ##########################################################################
 314 #
 315 # BOM comparison routines
 316 #
 317
 318 # used to remove things like creation date from BOM files
 319 normalize_bom() {
 320     local f1="$1"
 321     local f2="$2"
 322     $AWK '
 323         /^# Date:/ {print "# Date: today"; next}
 324         /^# Author:/ {print "# Author: PCB"; next}
 325         {print}' \
 326         $f1 > $f2
 327 }
 328
 329 # top level function to compare BOM output
 330 compare_bom() {
 331     local f1="$1"
 332     local f2="$2"
 333     compare_check "compare_bom" "$f1" "$f2" || return 1
 334
 335     # an example BOM file is:
 336
 337     #  # $Id$
 338     #  # PcbBOM Version 1.0
 339     #  # Date: Wed Jun 17 14:41:43 2009 UTC
 340     #  # Author: Dan McMahill
 341     #  # Title: Basic BOM/XY Test - PCB BOM
 342     #  # Quantity, Description, Value, RefDes
 343     #  # --------------------------------------------
 344     #  8,"Standard SMT resistor, capacitor etc","RESC3216N",R90_TOP R180_TOP R270_TOP R0_TOP R270_BOT R180_BOT R90_BOT R0_BOT
 345     #  8,"Dual in-line package, narrow (300 mil)","DIP8",UDIP90_TOP UDIP180_TOP UDIP270_TOP UDIP0_TOP UDIP270_BOT UDIP180_BOT UDIP90_BOT UDIP0_BOT
 346     #  8,"Small outline package, narrow (150mil)","SO8",USO90_TOP USO180_TOP USO270_TOP USO0_TOP USO270_BOT USO180_BOT USO90_BOT USO0_BOT
 347
 348     #  For comparison, we need to ignore changes in the Date and Author lines.
 349     local cf1=${tmpd}/`basename $f1`-ref
 350     local cf2=${tmpd}/`basename $f2`-out
 351
 352     normalize_bom $f1 $cf1
 353     normalize_bom $f2 $cf2
 354     run_diff $cf1 $cf2 || test_failed=yes
 355 }
 356
 357 ##########################################################################
 358 #
 359 # X-Y (centroid) comparison routines
 360 #
 361
 362 # used to remove things like creation date from BOM files
 363 normalize_xy() {
 364     local f1="$1"
 365     local f2="$2"
 366     $AWK '
 367         /^# Date:/ {print "# Date: today"; next}
 368         /^# Author:/ {print "# Author: PCB"; next}
 369         {print}' \
 370         $f1 > $f2
 371 }
 372
 373 compare_xy() {
 374     local f1="$1"
 375     local f2="$2"
 376     compare_check "compare_xy" "$f1" "$f2" || return 1
 377
 378     local cf1=${tmpd}/`basename $f1`-ref
 379     local cf2=${tmpd}/`basename $f2`-out
 380     normalize_xy "$f1" "$cf1"
 381     normalize_xy "$f2" "$cf2"
 382     run_diff "$cf1" "$cf2" || test_failed=yes
 383 }
 384
 385 ##########################################################################
 386 #
 387 # GCODE comparison routines
 388 #
 389
 390 # used to remove things like creation date from gcode files
 391 normalize_gcode() {
 392     local f1="$1"
 393     local f2="$2"
 394     # matches string such as '( Tue Mar  9 17:45:43 2010 )'
 395     $AWK --posix '!/^\( *[A-Z][a-z]{2} [A-Z][a-z]{2} [0123 ][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4} *\)$/' \
 396         $f1 > $f2
 397 }
 398
 399 compare_gcode() {
 400     local f1="$1"
 401     local f2="$2"
 402     compare_check "compare_gcode" "$f1" "$f2" || return 1
 403
 404     #  For comparison, we need to ignore changes in the Date and Author lines.
 405     local cf1=${tmpd}/`basename $f1`-ref
 406     local cf2=${tmpd}/`basename $f2`-out
 407
 408     normalize_gcode $f1 $cf1
 409     normalize_gcode $f2 $cf2
 410
 411     run_diff "$cf1" "$cf2" || test_failed=yes
 412 }
 413
 414 ##########################################################################
 415 #
 416 # RS274-X and Excellon comparison
 417 #
 418
 419 compare_rs274x() {
 420     local f1="$1"
 421     local f2="$2"
 422     compare_check "compare_rs274x" "$f1" "$f2" || return 1
 423
 424     # use gerbv to export our reference RS274-X file and our generated
 425     # RS274-X file to png.  Then we'll use ImageMagick to see
 426     # if there are any differences in the resulting files
 427     pngdir=${rundir}/png
 428     mkdir -p ${pngdir}
 429     nb=`basename $f1`
 430     png1=${pngdir}/${nb}-ref.png
 431     png2=${pngdir}/${nb}-out.png
 432
 433     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}"
 434     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png1} ${f1}
 435
 436     debug "${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}"
 437     ${GERBV} ${GERBV_DEFAULT_FLAGS} --output=${png2} ${f2}
 438
 439     compare_image ${png1} ${png2}
 440
 441 }
 442
 443 compare_cnc() {
 444     compare_rs274x $*
 445 }
 446
 447 ##########################################################################
 448 #
 449 # GIF/JPEG/PNG comparison routines
 450 #
 451
 452 compare_image() {
 453     local f1="$1"
 454     local f2="$2"
 455     compare_check "compare_image" "$f1" "$f2" || return 1
 456
 457     # now see if the image files are the same
 458     debug "${IM_COMPARE} -metric MAE ${f1} ${f2}  null:"
 459     same=`${IM_COMPARE} -metric MAE ${f1} ${f2}  null: 2>&1 | \
 460           ${AWK} '{if($1 == 0){print "yes"} else {print "no"}}'`
 461     debug "compare_image():  same = $same"
 462
 463     if test "$same" != yes ; then
 464         test_failed=yes
 465         echo "FAILED:  See ${errdir}"
 466         mkdir -p ${errdir}
 467         ${IM_COMPARE} ${f1} ${f2} ${errdir}/compare.png
 468         ${IM_COMPOSITE} ${f1} ${f2} -compose difference ${errdir}/composite.png
 469         ${IM_CONVERT} ${f1} ${f2} -compose difference -composite  -colorspace gray   ${errdir}/gray.png
 470         cat > ${errdir}/animate.sh << EOF
 471 #!/bin/sh
 472 ${IM_CONVERT} -label "%f" ${f1} ${f2} miff:- | \
 473 ${IM_MONTAGE} - -geometry +0+0 -tile 1x1 miff:- | \
 474 ${IM_ANIMATE} -delay 0.5 -loop 0 -
 475 EOF
 476         chmod a+x ${errdir}/animate.sh
 477     fi
 478 }
 479
 480 ##########################################################################
 481 #
 482 # The main program loop
 483 #
 484
 485 for t in $all_tests ; do
 486     show_sep
 487     echo "Test:  $t"
 488
 489     tot=`expr $tot + 1`
 490
 491     ######################################################################
 492     #
 493     # extract the details for the test
 494     #
 495
 496     pcb_flags="${PCB_DEFAULT_FLAGS}"
 497
 498     rundir="${OUTDIR}/${t}"
 499     refdir="${REFDIR}/${t}"
 500     errdir=${rundir}/mismatch
 501
 502     # test_name | layout file(s) | [export hid name] | [optional arguments to pcb] |  [mismatch]
 503     # | output files
 504     name=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $1}'`
 505     files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $2}'`
 506     hid=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {gsub(/[ \t]*/, ""); print $3}'`
 507     args=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $4}'`
 508     mismatch=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {if($5 == "mismatch"){print "yes"}else{print "no"}}'`
 509     out_files=`grep "^[ \t]*${t}[ \t]*|" $TESTLIST | $AWK 'BEGIN{FS="|"} {print $6}'`
 510
 511     if test "X${name}" = "X" ; then
 512         echo "ERROR:  Specified test ${t} does not appear to exist"
 513         skip=`expr $skip + 1`
 514         continue
 515     fi
 516
 517     if test "X${args}" != "X" ; then
 518         pcb_flags="${args}"
 519     fi
 520
 521     if test "X${hid}" = "Xgerber" ; then
 522         pcb_flags="--fab-author Fab_Author ${pcb_flags}"
 523     fi
 524
 525     ######################################################################
 526     #
 527     # Set up the run directory
 528     #
 529
 530     test -d ${rundir} && rm -fr ${rundir}
 531     mkdir -p ${rundir}
 532     if test $? -ne 0 ; then
 533         echo "$0:  Could not create run directory ${rundir}"
 534         skip=`expr $skip + 1`
 535         continue
 536     fi
 537
 538
 539     ######################################################################
 540     #
 541     # check to see if the files we need exist and copy them to the run
 542     # directory
 543     #
 544
 545     missing_files=no
 546     path_files=""
 547     for f in $files ; do
 548         if test ! -f ${INDIR}/${f} ; then
 549             echo "ERROR:  File $f specified as part of the $t test does not exist"
 550             missing_files=yes
 551         else
 552             path_files="${path_files} ${INDIR}/${f}"
 553             cp "${INDIR}/${f}" "${rundir}"
 554         fi
 555     done
 556
 557     if test "$missing_files" = "yes" ; then
 558         echo "${t} had missing input files.  Skipping test"
 559         skip=`expr $skip + 1`
 560         continue
 561     fi
 562
 563     ######################################################################
 564     #
 565     # run PCB
 566     #
 567
 568     echo "${PCB} -x ${hid} ${pcb_flags} ${path_files}"
 569     (cd ${rundir} && ${PCB} -x ${hid} ${pcb_flags} ${files})
 570     pcb_rc=$?
 571
 572     if test $pcb_rc -ne 0 ; then
 573         echo "${PCB} returned ${pcb_rc}.  This is a failure."
 574         fail=`expr $fail + 1`
 575         continue
 576     fi
 577
 578     # and clean up the input files we'd copied over
 579     for f in $files ; do
 580         rm -f ${rundir}/${f}
 581     done
 582
 583     ######################################################################
 584     #
 585     # check the result
 586     #
 587
 588     if test "X$regen" = "Xyes" ; then
 589         echo "Regenerated ${t}"
 590     else
 591         # compare the result to our reference file
 592         test_failed=no
 593         test_skipped=no
 594         for f in $out_files ; do
 595             debug "processing $f"
 596             # break apart type:fn into the type and file name
 597             type=`echo $f | sed 's;:.*;;g'`
 598             fn=`echo $f | sed 's;.*:;;g'`
 599
 600             case $type in
 601                 # BOM HID
 602                 bom)
 603                     compare_bom ${refdir}/${fn} ${rundir}/${fn}
 604                     ;;
 605
 606                 xy)
 607                     compare_xy ${refdir}/${fn} ${rundir}/${fn}
 608                     ;;
 609
 610                 # GCODE HID
 611                 gcode)
 612                     compare_gcode ${refdir}/${fn} ${rundir}/${fn}
 613                     ;;
 614
 615                 # GERBER HID
 616                 cnc)
 617                     compare_cnc ${refdir}/${fn} ${rundir}/${fn}
 618                     ;;
 619
 620                 gbx)
 621                     compare_rs274x ${refdir}/${fn} ${rundir}/${fn}
 622                     ;;
 623
 624                 # PNG HID
 625                 gif)
 626                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 627                     ;;
 628
 629                 jpg)
 630                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 631                     ;;
 632
 633                 png)
 634                     compare_image ${refdir}/${fn} ${rundir}/${fn}
 635                     ;;
 636
 637                 # unknown
 638                 *)
 639                     echo "internal error:  $type is not a known file type"
 640                     exit 1
 641                     ;;
 642             esac
 643
 644         done
 645
 646
 647         if test $test_failed = yes ; then
 648             echo "FAIL"
 649             fail=`expr $fail + 1`
 650         elif test $test_skipped = yes ; then
 651             echo "SKIPPED"
 652             skip=`expr $skip + 1`
 653         else
 654             echo "PASSED"
 655             pass=`expr $pass + 1`
 656         fi
 657
 658     fi
 659
 660 done
 661
 662 show_sep
 663 echo "Passed $pass, failed $fail, skipped $skip out of $tot tests."
 664
 665 rc=0
 666 if test $pass -ne $tot ; then
 667     rc=1
 668 fi
 669
 670 exit $rc
 671