t/test-driver-custom-multitest-recheck.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 # Custom test drivers: try the "recheck" functionality with test protocols
  18 # that allow multiple testcases in a single test script.  This test not
  19 # only checks implementation details in Automake's custom test drivers
  20 # support, but also serves as a "usability test" for our APIs.
  21 # See also related tests 'test-driver-custom-multitest-recheck2.sh'
  22 # and 'parallel-tests-recheck-override.sh'.
  23 # Keep in sync with 'tap-recheck.sh'.
  24
  25 . ./defs || exit 1
  26
  27 cp "$am_testaux_srcdir"/trivial-test-driver . \
  28   || fatal_ "failed to fetch auxiliary script trivial-test-driver"
  29
  30 cat >> configure.ac << 'END'
  31 AC_OUTPUT
  32 END
  33
  34 cat > Makefile.am << 'END'
  35 TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
  36 TESTS = a.test b.test c.test d.test
  37 END
  38
  39 cat > a.test << 'END'
  40 #! /bin/sh
  41 echo PASS: aa
  42 echo PASS: AA
  43 : > a.run
  44 END
  45
  46 cat > b.test << 'END'
  47 #! /bin/sh
  48 echo PASS:
  49 if test -f b.ok; then
  50   echo PASS:
  51 else
  52   echo ERROR:
  53 fi
  54 : > b.run
  55 END
  56
  57 cat > c.test << 'END'
  58 #! /bin/sh
  59 if test -f c.pass; then
  60   echo PASS: c0
  61 else
  62   echo FAIL: c0
  63 fi
  64 if test -f c.xfail; then
  65   echo XFAIL: c1
  66 else
  67   echo XPASS: c1
  68 fi
  69 echo XFAIL: c2
  70 : > c.run
  71 END
  72
  73 cat > d.test << 'END'
  74 #! /bin/sh
  75 echo SKIP: who cares ...
  76 (. ./d.extra) || echo FAIL: d.extra failed
  77 : > d.run
  78 END
  79
  80 chmod a+x *.test
  81
  82 $ACLOCAL
  83 $AUTOCONF
  84 $AUTOMAKE
  85
  86 do_recheck ()
  87 {
  88   case $* in
  89     --fail) on_bad_rc='&&';;
  90     --pass) on_bad_rc='||';;
  91          *) fatal_ "invalid usage of function 'do_recheck'";;
  92   esac
  93   rm -f *.run
  94   eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; exit 1; }; :"
  95   cat stdout; ls -l
  96 }
  97
  98 for vpath in : false; do
  99   if $vpath; then
 100     mkdir build
 101     cd build
 102     srcdir=..
 103   else
 104     srcdir=.
 105   fi
 106
 107   $srcdir/configure
 108
 109   : A "make recheck" in a clean tree should run no tests.
 110   using_gmake || $sleep # Required by BSD make.
 111   do_recheck --pass
 112   cat test-suite.log
 113   test ! -e a.run
 114   test ! -e a.log
 115   test ! -e b.run
 116   test ! -e b.log
 117   test ! -e c.run
 118   test ! -e c.log
 119   test ! -e d.run
 120   test ! -e d.log
 121   count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 122
 123   : Run the tests for the first time.
 124   $MAKE check >stdout && { cat stdout; exit 1; }
 125   cat stdout
 126   ls -l
 127   # All the test scripts should have run.
 128   test -f a.run
 129   test -f b.run
 130   test -f c.run
 131   test -f d.run
 132   count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
 133
 134   : Let us make b.test pass.
 135   using_gmake || $sleep # Required by BSD make.
 136   echo OK > b.ok
 137   do_recheck --fail
 138   # a.test has been successful the first time, so no need to re-run it.
 139   # Similar considerations apply to similar checks, below.
 140   test ! -e a.run
 141   test -f b.run
 142   test -f c.run
 143   test -f d.run
 144   count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
 145
 146   : Let us make the first part of c.test pass.
 147   using_gmake || $sleep # Required by BSD make.
 148   echo OK > c.pass
 149   do_recheck --fail
 150   test ! -e a.run
 151   test ! -e b.run
 152   test -f c.run
 153   test -f d.run
 154   count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
 155
 156   : Let us make also the second part of c.test pass.
 157   using_gmake || $sleep # Required by BSD make.
 158   echo KO > c.xfail
 159   do_recheck --fail
 160   test ! -e a.run
 161   test ! -e b.run
 162   test -f c.run
 163   test -f d.run
 164   count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
 165
 166   : Nothing changed, so only d.test should be run.
 167   for i in 1 2; do
 168     using_gmake || $sleep # Required by BSD make.
 169     do_recheck --fail
 170     test ! -e a.run
 171     test ! -e b.run
 172     test ! -e c.run
 173     test -f d.run
 174     count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
 175   done
 176
 177   : Let us make d.test run more testcases, and experience _more_ failures.
 178   using_gmake || $sleep # Required by BSD make.
 179   unindent > d.extra <<'END'
 180     echo SKIP: s
 181     echo FAIL: f 1
 182     echo PASS: p 1
 183     echo FAIL: f 2
 184     echo XPASS: xp
 185     echo FAIL: f 3
 186     echo FAIL: f 4
 187     echo ERROR: e 1
 188     echo PASS: p 2
 189     echo ERROR: e 2
 190 END
 191   do_recheck --fail
 192   test ! -e a.run
 193   test ! -e b.run
 194   test ! -e c.run
 195   test -f d.run
 196   count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
 197
 198   : Let us finally make d.test pass.
 199   using_gmake || $sleep # Required by BSD make.
 200   echo : > d.extra
 201   do_recheck --pass
 202   test ! -e a.run
 203   test ! -e b.run
 204   test ! -e c.run
 205   test -f d.run
 206   count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
 207
 208   : All tests have been successful or skipped, nothing should be re-run.
 209   using_gmake || $sleep # Required by BSD make.
 210   do_recheck --pass
 211   test ! -e a.run
 212   test ! -e b.run
 213   test ! -e c.run
 214   test ! -e d.run
 215   count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 216
 217   cd $srcdir
 218
 219 done
 220
 221 :