tests/test.sh

   1 #! /bin/sh
   2 # Basic test script.
   3 # This file is part of AutoTroll.
   4 # Copyright (C) 2006  Benoit Sigoure <tsuna@lrde.epita.fr>
   5 #
   6 # AutoTroll is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License
   8 # as published by the Free Software Foundation; either version 2
   9 # of the License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
  19 # USA.
  20
  21   # ------------- #
  22   # Documentation #
  23   # ------------- #
  24
  25 # This test script provides a simple way to generate test suites using
  26 # GNU Automake. Note that once the test suite is generated and distributed,
  27 # only standard POSIX make is required.
  28 #
  29 # This script will check the output (on stdout and/or stderr) of a program
  30 # against a reference output (if available) whereas Automake's builtin test
  31 # feature only checks the return value (without using DejaGnu).
  32
  33 # Quick Setup
  34 # -----------
  35
  36 # Better than a long explanation, here is a sample test suite to put in your
  37 # tests/Makefile.am:
  38 #
  39 # check_PROGRAMS = foo bar
  40 # foo_SOURCES = foo.c
  41 # bar_SOURCES = bar.c aux.c
  42 # TESTS = foo.test bar.test
  43 #
  44 # SUFFIXES = .test
  45 # .c.test:
  46 #       $(LN_S) -f $(srcdir)/test.sh $@
  47                            # ^^^^^^^ where test.sh is this script.
  48 #
  49 # EXTRA_DIST = test.sh
  50 # CLEANFILES = *.my_stdout *.my_stderr
  51 # TESTS_ENVIRONMENT = SRCDIR=$(srcdir)
  52
  53 # Alternatively, you can generate the .test files like this:
  54 # $(TESTS): Makefile.am
  55 #       for i in $(TESTS); do $(LN_S) -f $(srcdir)/test.sh $$i; done
  56                   # where test.sh is this script.  ^^^^^^^
  57 # If all your tests use the script, you don't have to bother with TESTS:
  58 # TESTS = $(check_PROGRAMS:=.test)
  59
  60 # How to use
  61 # ----------
  62
  63 # Simply run `make check' or better, `make distcheck' :)
  64 # By default, only the return value will be checked -- which is just what
  65 # Automake does by default. If you want to check stderr/stdout, you need to
  66 # create (or generate) reference output files. Eg: foo.stdout, foo.stderr or
  67 # foo.ret (for the return value).
  68 #
  69 # You can do this by hand or use this script to generate the output:
  70 # $ make check GEN=stdout
  71 # will run all your tests and save their output in `test-name.stdout'. The
  72 # next time you run `make check', stdout will be checked against the saved
  73 # (reference) output. Running this again will simply update (overwrite)
  74 # reference files.
  75 #
  76 # You can also use `GEN=stderr' or `GEN=ret'.
  77 # If you encounter any problem, try `make check DEBUG=1'
  78 # You can also `make check VERBOSE=1' in order to get the output on stdout and
  79 # stderr as the tests run. Note that when doing this, stdout will always
  80 # appear first, then stderr. Messages on stdout/stderr will not be interleaved
  81 # as they might originally be. This is because stdout/stderr are buffered.
  82 #
  83 # When you generate reference output files, you might want to version them.
  84 # When you run `make check GEN=something', files are generated in the build
  85 # tree. You might want to move them to the source tree and `svn add' them. You
  86 # might also want to distribute them (using EXTRA_DIST = $(TESTS:.test=.stdout)
  87 # for instance, or list the .stdout/.stderr/.ret manually).
  88
  89 # new_reference_output_file [ret|stdout|stderr] <ref-file>
  90 new_reference_output_file()
  91 {
  92   test -s $bprog.my_$1 && return 0 # Skip empty files.
  93   # Simply update existing reference output file.
  94   if test x"$2" != x; then
  95     cp -f $bprog.my_$1 $2
  96   else # Create new reference output file (try first in SRCDIR).
  97     cp -f $bprog.my_$1 $SRCDIR/$bprog.$1 \
  98     || cp -f $bprog.my_$1 $bprog.$1 || {
  99       echo "$0: cannot generate $bprog.$1" >&2
 100       script_exit=1
 101     }
 102   fi
 103 }
 104
 105 test x"$DEBUG" != x && set -x
 106
 107 # Program to test
 108 prog=`echo "$0" | sed 's/\.test$//;y/ /_/'`
 109 # Basename of the program to test
 110 bprog=`basename $prog`
 111
 112 test -f $prog || {
 113   echo "$prog: No such file or directory." >&2
 114   exit 127
 115 }
 116
 117 test -x $prog || {
 118   echo "$prog: Not executable." >&2
 119   exit 126
 120 }
 121
 122 # We NEED the env var SRCDIR
 123 test x"$SRCDIR" = x && echo "$0: \$SRCDIR is empty" >&2 && exit 42
 124
 125 # default values
 126 ref_ret=0;     check_ret='no'
 127 ref_stdout=''; check_stdout='no'
 128 ref_stderr=''; check_stderr='no'
 129
 130 # Find reference output files (if they are provided).
 131 for i in ret stdout stderr; do
 132   # Search first in build dir, then where the prog is (should be build dir
 133   # too, but we never know) then finally in the SRCDIR.
 134   for f in ./$bprog.$i $prog.$i $SRCDIR/$bprog.$i; do
 135     if [ -f "$f" ]; then
 136       if [ -r "$f" ]; then
 137         # Meta-programming in Sh \o/
 138         eval ref_$i="$f"
 139         eval check_$i='yes'
 140         break
 141       else
 142         echo "$0: warning: "$f" isn't readable" >&2
 143       fi
 144     fi
 145   done
 146 done
 147
 148 # Run the program to check and save outputs.
 149 $prog >$bprog.my_stdout 2>$bprog.my_stderr
 150 my_ret=$?
 151
 152 # Return value of this script.
 153 script_exit=0
 154
 155 # Are we trying to generate reference output files?
 156 if test "x$GEN" = xret; then
 157   echo "$my_ret" >$bprog.my_ret
 158   new_reference_output_file ret "$ref_var"
 159   rm -f $bprog.my_ret
 160 fi
 161
 162 if [ $check_ret = 'no' ]; then # Check that the test returned 0 anyway.
 163   if [ $my_ret -ne 0 ]; then
 164     script_exit=1
 165     echo "$0: bad return value, got $my_ret, expected 0" >&2
 166   fi
 167 else
 168   ref_ret_val=`cat "$ref_ret"`
 169   ref_ret_without_digits=`echo "$ref_ret_val" | sed 's/[0-9]//g'`
 170   if [ x"$ref_ret_without_digits" != x ]; then
 171     script_exit=1
 172     echo "$0: invalid content for $ref_ret, maybe run \`make check GEN=ret'?"
 173   fi
 174   if [ $my_ret -ne "$ref_ret_val" ]; then
 175     script_exit=1
 176     echo "$0: bad return value, got $my_ret, expected $ref_ret_val" >&2
 177   fi
 178 fi
 179
 180 # Check stdout and stderr.
 181 for i in stdout stderr; do
 182   check_var=check_$i
 183   check_var=`eval echo \\\$$check_var`
 184   ref_var=ref_$i
 185   ref_var=`eval echo \\\$$ref_var`
 186
 187   # Display their output (a bit late...).
 188   test "x$VERBOSE" != x && cat $bprog.my_$i
 189
 190   # Are we trying to generate reference output files?
 191   if test "x$GEN" = x$i; then
 192     new_reference_output_file $i "$ref_var"
 193   fi
 194
 195   if [ $check_var = 'yes' ]; then
 196     if cmp -s $bprog.my_$i $ref_var; then
 197       rm -f $bprog.my_$i # Good output, remove temporary file.
 198     else
 199       script_exit=1
 200       echo "$0: wrong output on $i" >&2
 201       diff -u $ref_var $bprog.my_$i
 202     fi
 203   else
 204     rm -f $bprog.my_$i # No reference output => remove temporary file.
 205   fi
 206 done
 207
 208 exit $script_exit