t/tap-recheck.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 # Test the 'recheck' target for TAP test protocol.
  18 # Keep in sync with 'test-driver-custom-multitest-recheck.test'.
  19
  20 . ./defs || exit 1
  21
  22 fetch_tap_driver
  23
  24 cat >> configure.ac <<END
  25 AC_OUTPUT
  26 END
  27
  28 cat > Makefile.am << 'END'
  29 TEST_LOG_DRIVER = $(srcdir)/tap-driver
  30 TESTS = a.test b.test c.test d.test
  31 END
  32
  33 cat > a.test << 'END'
  34 #! /bin/sh
  35 echo 1..2
  36 echo ok 1
  37 echo ok 2
  38 : > a.run
  39 END
  40
  41 cat > b.test << 'END'
  42 #! /bin/sh
  43 : > b.run
  44 echo 1..2
  45 echo ok 1
  46 if test -f b.ok; then
  47   echo ok 2
  48 else
  49   echo 'Bail out!'
  50 fi
  51 END
  52
  53 cat > c.test << 'END'
  54 #! /bin/sh
  55 echo 1..3
  56 if test -f c.pass; then
  57   echo 'ok - c is ok :-)'
  58 else
  59   echo 'not ok - c is ko :-('
  60 fi
  61 if test -f c.xfail; then
  62   echo 'not ok 2 # TODO'
  63 else
  64   echo 'ok 2 # TODO'
  65 fi
  66 echo 'not ok 3 - blah blah # TODO need better diagnostic'
  67 : > c.run
  68 END
  69
  70 cat > d.test << 'END'
  71 #! /bin/sh
  72 test -f ./d.count && . ./d.count
  73 echo 1..${test_count-2}
  74 echo ok 1 '# SKIP: who cares ...'
  75 (. ./d.extra) || echo 'not ok 2 - d.extra failed'
  76 : > d.run
  77 END
  78
  79 chmod a+x *.test
  80
  81 $ACLOCAL
  82 $AUTOCONF
  83 $AUTOMAKE
  84
  85 do_recheck ()
  86 {
  87   case $* in
  88     --fail) on_bad_rc='&&';;
  89     --pass) on_bad_rc='||';;
  90          *) fatal_ "invalid usage of function 'do_recheck'";;
  91   esac
  92   rm -f *.run
  93   eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; exit 1; }; :"
  94   cat stdout; ls -l
  95 }
  96
  97 for vpath in : false; do
  98   if $vpath; then
  99     mkdir build
 100     cd build
 101     srcdir=..
 102   else
 103     srcdir=.
 104   fi
 105
 106   $srcdir/configure
 107
 108   : A "make recheck" in a clean tree should run no tests.
 109   do_recheck --pass
 110   cat test-suite.log
 111   test ! -e a.run
 112   test ! -e a.log
 113   test ! -e b.run
 114   test ! -e b.log
 115   test ! -e c.run
 116   test ! -e c.log
 117   test ! -e d.run
 118   test ! -e d.log
 119   count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 120
 121   : Run the tests for the first time.
 122   $MAKE check >stdout && { cat stdout; exit 1; }
 123   cat stdout
 124   ls -l
 125   # All the test scripts should have run.
 126   test -f a.run
 127   test -f b.run
 128   test -f c.run
 129   test -f d.run
 130   count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
 131
 132   : Let us make b.test pass.
 133   using_gmake || $sleep # Required by BSD make.
 134   echo OK > b.ok
 135   do_recheck --fail
 136   # a.test has been successful the first time, so no need to re-run it.
 137   # Similar considerations apply to similar checks, below.
 138   test ! -e a.run
 139   test -f b.run
 140   test -f c.run
 141   test -f d.run
 142   count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
 143
 144   : Let us make the first part of c.test pass.
 145   using_gmake || $sleep # Required by BSD make.
 146   echo OK > c.pass
 147   do_recheck --fail
 148   test ! -e a.run
 149   test ! -e b.run
 150   test -f c.run
 151   test -f d.run
 152   count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
 153
 154   : Let us make also the second part of c.test pass.
 155   using_gmake || $sleep # Required by BSD make.
 156   echo KO > c.xfail
 157   do_recheck --fail
 158   test ! -e a.run
 159   test ! -e b.run
 160   test -f c.run
 161   test -f d.run
 162   count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
 163
 164   : Nothing changed, so only d.test should be run.
 165   for i in 1 2; do
 166     using_gmake || $sleep # Required by BSD make.
 167     do_recheck --fail
 168     test ! -e a.run
 169     test ! -e b.run
 170     test ! -e c.run
 171     test -f d.run
 172     count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
 173   done
 174
 175   : Let us make d.test run more testcases, and experience _more_ failures.
 176   using_gmake || $sleep # Required by BSD make.
 177   echo 'test_count=9' > d.count
 178   unindent > d.extra <<'END'
 179     echo 'ok # SKIP s'
 180     echo 'not ok - f 1'
 181     echo 'ok - p 1'
 182     echo 'not ok - f 2'
 183     echo 'ok # TODO xp'
 184     echo 'not ok - f 3'
 185     echo 'not ok - f 4'
 186     echo 'ok - p 2'
 187     echo 'ok' # Extra test.
 188     echo 'Bail out!'
 189 END
 190   do_recheck --fail
 191   test ! -e a.run
 192   test ! -e b.run
 193   test ! -e c.run
 194   test -f d.run
 195   count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
 196
 197   : Let us finally make d.test pass.
 198   using_gmake || $sleep # Required by BSD make.
 199   echo 'test_count=1' > d.count
 200   echo : > d.extra
 201   do_recheck --pass
 202   test ! -e a.run
 203   test ! -e b.run
 204   test ! -e c.run
 205   test -f d.run
 206   count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
 207
 208   : All tests have been successful or skipped, nothing should be re-run.
 209   using_gmake || $sleep # Required by BSD make.
 210   do_recheck --pass
 211   test ! -e a.run
 212   test ! -e b.run
 213   test ! -e c.run
 214   test ! -e d.run
 215   count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
 216
 217   cd $srcdir
 218
 219 done
 220
 221 :