t/tap-recheck-logs.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 # TAP support:
  18 # - RECHECK_LOGS
  19
  20 . test-init.sh
  21
  22 cat > Makefile.am << 'END'
  23 TEST_LOG_COMPILER = cat
  24 TESTS = foo.test bar.test baz.test
  25 baz.log: zardoz
  26 END
  27
  28 . tap-setup.sh
  29
  30 : > zardoz
  31
  32 cat > foo.test <<'END'
  33 1..2
  34 ok 1
  35 ok 2
  36 END
  37
  38 cat > bar.test <<'END'
  39 1..1
  40 not ok 1
  41 END
  42
  43 cat > baz.test <<'END'
  44 1..1
  45 Bail out!
  46 END
  47
  48 # Even the tests that are not re-run should contribute to the testsuite
  49 # summary when obtained by "make check RECHECK_LOGS=".
  50 grep_summary ()
  51 {
  52   grep '^# TOTAL: *4$' stdout
  53   grep '^# PASS: *2$' stdout
  54   grep '^# XPASS: *0$' stdout
  55   grep '^# FAIL: *1$' stdout
  56   grep '^# XFAIL: *0$' stdout
  57   grep '^# SKIP: *0$' stdout
  58   grep '^# ERROR: *1$' stdout
  59 }
  60
  61 $MAKE -e check && exit 1
  62 test -f foo.log
  63 test -f bar.log
  64 test -f baz.log
  65
  66 rm -f foo.log bar.log
  67
  68 env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; exit 1; }
  69 cat stdout
  70 test -f foo.log
  71 test -f bar.log
  72 grep '^PASS: foo\.test 1$' stdout
  73 grep '^PASS: foo\.test 2$' stdout
  74 grep '^FAIL: bar\.test 1$' stdout
  75 grep 'baz\.test' stdout && exit 1
  76 grep_summary
  77
  78 $sleep
  79 touch foo.test
  80 # We re-run only a successful test, but the tests that failed in the
  81 # previous run should still be taken into account, and cause an overall
  82 # failure.
  83 env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; exit 1; }
  84 cat stdout
  85 grep '^PASS: foo\.test 1$' stdout
  86 grep '^PASS: foo\.test 2$' stdout
  87 grep 'ba[rz]\.test' stdout && exit 1
  88 is_newest foo.log foo.test
  89 grep_summary
  90
  91 $sleep
  92 touch zardoz
  93 env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; exit 1; }
  94 cat stdout
  95 grep '^ERROR: baz\.test' stdout
  96 $EGREP '(foo|bar)\.test' stdout && exit 1
  97 is_newest baz.log zardoz
  98 grep_summary
  99
 100 # Now, explicitly retry with all test logs already updated, and ensure
 101 # that the summary is still displayed.
 102 env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; exit 1; }
 103 cat stdout
 104 $EGREP '(foo|bar|baz)\.test' stdout && exit 1
 105 grep_summary
 106
 107 # The following should re-run foo.test (and only foo.test), even if its
 108 # log file is up-to-date.
 109 : > older
 110 env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; exit 1; }
 111 cat stdout
 112 grep '^PASS: foo\.test 1$' stdout
 113 grep '^PASS: foo\.test 2$' stdout
 114 grep 'ba[rz]\.test' stdout && exit 1
 115 is_newest foo.log older
 116 grep_summary
 117
 118 :