tests/test.sh

   1 #! /bin/sh
   2 # Basic test script.
   3 # This file is part of AutoTroll.
   4 # Copyright (C) 2006  Benoit Sigoure <tsuna@lrde.epita.fr>
   5 #
   6 # AutoTroll is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License
   8 # as published by the Free Software Foundation; either version 2
   9 # of the License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
  19 # USA.
  20 #
  21 # In addition, as a special exception, the copyright holders of AutoTroll
  22 # give you unlimited permission to copy, distribute and modify the configure
  23 # scripts that are the output of Autoconf when processing the macros of
  24 # AutoTroll.  You need not follow the terms of the GNU General Public License
  25 # when using or distributing such scripts, even though portions of the text of
  26 # AutoTroll appear in them. The GNU General Public License (GPL) does govern
  27 # all other use of the material that constitutes AutoTroll.
  28 #
  29 # This special exception to the GPL applies to versions of AutoTroll
  30 # released by the copyright holders of AutoTroll.  Note that people who make
  31 # modified versions of AutoTroll are not obligated to grant this special
  32 # exception for their modified versions; it is their choice whether to do so.
  33 # The GNU General Public License gives permission to release a modified version
  34 # without this exception; this exception also makes it possible to release a
  35 # modified version which carries forward this exception.
  36
  37   # ------------- #
  38   # Documentation #
  39   # ------------- #
  40
  41 # This test script provides a simple way to generate test suites using
  42 # GNU Automake. Note that once the test suite is generated and distributed,
  43 # only standard POSIX make is required.
  44 #
  45 # This script will check the output (on stdout and/or stderr) of a program
  46 # against a reference output (if available) whereas Automake's builtin test
  47 # feature only checks the return value (without using DejaGnu).
  48
  49 # Quick Setup
  50 # -----------
  51
  52 # Better than a long explanation, here is a sample test suite to put in your
  53 # tests/Makefile.am:
  54 #
  55 # check_PROGRAMS = foo bar
  56 # foo_SOURCES = foo.c
  57 # bar_SOURCES = bar.c aux.c
  58 # TESTS = foo.test bar.test
  59 #
  60 # SUFFIXES = .test
  61 # .c.test:
  62 #       $(LN_S) -f $(srcdir)/test.sh $@
  63                            # ^^^^^^^ where test.sh is this script.
  64 #
  65 # EXTRA_DIST = test.sh
  66 # CLEANFILES = *.my_stdout *.my_stderr
  67 # TESTS_ENVIRONMENT = SRCDIR=$(srcdir)
  68
  69 # Alternatively, you can generate the .test files like this:
  70 # $(TESTS): Makefile.am
  71 #       for i in $(TESTS); do $(LN_S) -f $(srcdir)/test.sh $$i; done
  72                   # where test.sh is this script.  ^^^^^^^
  73 # If all your tests use the script, you don't have to bother with TESTS:
  74 # TESTS = $(check_PROGRAMS:=.test)
  75
  76 # How to use
  77 # ----------
  78
  79 # Simply run `make check' or better, `make distcheck' :)
  80 # By default, only the return value will be checked -- which is just what
  81 # Automake does by default. If you want to check stderr/stdout, you need to
  82 # create (or generate) reference output files. Eg: foo.stdout, foo.stderr or
  83 # foo.ret (for the return value).
  84 #
  85 # You can do this by hand or use this script to generate the output:
  86 # $ make check GEN=stdout
  87 # will run all your tests and save their output in `test-name.stdout'. The
  88 # next time you run `make check', stdout will be checked against the saved
  89 # (reference) output. Running this again will simply update (overwrite)
  90 # reference files.
  91 #
  92 # You can also use `GEN=stderr' or `GEN=ret'.
  93 # If you encounter any problem, try `make check DEBUG=1'
  94 # You can also `make check VERBOSE=1' in order to get the output on stdout and
  95 # stderr as the tests run. Note that when doing this, stdout will always
  96 # appear first, then stderr. Messages on stdout/stderr will not be interleaved
  97 # as they might originally be. This is because stdout/stderr are buffered.
  98 #
  99 # When you generate reference output files, you might want to version them.
 100 # When you run `make check GEN=something', files are generated in the build
 101 # tree. You might want to move them to the source tree and `svn add' them. You
 102 # might also want to distribute them (using EXTRA_DIST = $(TESTS:.test=.stdout)
 103 # for instance, or list the .stdout/.stderr/.ret manually).
 104
 105 # new_reference_output_file [ret|stdout|stderr] <ref-file>
 106 new_reference_output_file()
 107 {
 108   test -s $bprog.my_$1 && return 0 # Skip empty files.
 109   # Simply update existing reference output file.
 110   if test x"$2" != x; then
 111     cp -f $bprog.my_$1 $2
 112   else # Create new reference output file (try first in SRCDIR).
 113     cp -f $bprog.my_$1 $SRCDIR/$bprog.$1 \
 114     || cp -f $bprog.my_$1 $bprog.$1 || {
 115       echo "$0: cannot generate $bprog.$1" >&2
 116       script_exit=1
 117     }
 118   fi
 119 }
 120
 121 test x"$DEBUG" != x && set -x
 122
 123 # Program to test
 124 prog=`echo "$0" | sed 's/\.test$//;y/ /_/'`
 125 # Basename of the program to test
 126 bprog=`basename $prog`
 127
 128 test -f $prog || {
 129   echo "$prog: No such file or directory." >&2
 130   exit 127
 131 }
 132
 133 test -x $prog || {
 134   echo "$prog: Not executable." >&2
 135   exit 126
 136 }
 137
 138 # We NEED the env var SRCDIR
 139 test x"$SRCDIR" = x && echo "$0: \$SRCDIR is empty" >&2 && exit 42
 140
 141 # default values
 142 ref_ret=0;     check_ret='no'
 143 ref_stdout=''; check_stdout='no'
 144 ref_stderr=''; check_stderr='no'
 145
 146 # Find reference output files (if they are provided).
 147 for i in ret stdout stderr; do
 148   # Search first in build dir, then where the prog is (should be build dir
 149   # too, but we never know) then finally in the SRCDIR.
 150   for f in ./$bprog.$i $prog.$i $SRCDIR/$bprog.$i; do
 151     if [ -f "$f" ]; then
 152       if [ -r "$f" ]; then
 153         # Meta-programming in Sh \o/
 154         eval ref_$i="$f"
 155         eval check_$i='yes'
 156         break
 157       else
 158         echo "$0: warning: "$f" isn't readable" >&2
 159       fi
 160     fi
 161   done
 162 done
 163
 164 # Run the program to check and save outputs.
 165 $prog >$bprog.my_stdout 2>$bprog.my_stderr
 166 my_ret=$?
 167
 168 # Return value of this script.
 169 script_exit=0
 170
 171 # Are we trying to generate reference output files?
 172 if test "x$GEN" = xret; then
 173   echo "$my_ret" >$bprog.my_ret
 174   new_reference_output_file ret "$ref_var"
 175   rm -f $bprog.my_ret
 176 fi
 177
 178 if [ $check_ret = 'no' ]; then # Check that the test returned 0 anyway.
 179   if [ $my_ret -ne 0 ]; then
 180     script_exit=1
 181     echo "$0: bad return value, got $my_ret, expected 0" >&2
 182   fi
 183 else
 184   ref_ret_val=`cat "$ref_ret"`
 185   ref_ret_without_digits=`echo "$ref_ret_val" | sed 's/[0-9]//g'`
 186   if [ x"$ref_ret_without_digits" != x ]; then
 187     script_exit=1
 188     echo "$0: invalid content for $ref_ret, maybe run \`make check GEN=ret'?"
 189   fi
 190   if [ $my_ret -ne "$ref_ret_val" ]; then
 191     script_exit=1
 192     echo "$0: bad return value, got $my_ret, expected $ref_ret_val" >&2
 193   fi
 194 fi
 195
 196 # Check stdout and stderr.
 197 for i in stdout stderr; do
 198   check_var=check_$i
 199   check_var=`eval echo \\\$$check_var`
 200   ref_var=ref_$i
 201   ref_var=`eval echo \\\$$ref_var`
 202
 203   # Display their output (a bit late...).
 204   test "x$VERBOSE" != x && cat $bprog.my_$i
 205
 206   # Are we trying to generate reference output files?
 207   if test "x$GEN" = x$i; then
 208     new_reference_output_file $i "$ref_var"
 209   fi
 210
 211   if [ $check_var = 'yes' ]; then
 212     if cmp -s $bprog.my_$i $ref_var; then
 213       rm -f $bprog.my_$i # Good output, remove temporary file.
 214     else
 215       script_exit=1
 216       echo "$0: wrong output on $i" >&2
 217       diff -u $ref_var $bprog.my_$i
 218     fi
 219   else
 220     rm -f $bprog.my_$i # No reference output => remove temporary file.
 221   fi
 222 done
 223
 224 exit $script_exit