t/tap-unplanned.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 # TAP support:
  18 #  - unplanned tests are properly reported as errors
  19
  20 . ./defs || exit 1
  21
  22 . tap-setup.sh
  23
  24 cat > all.test <<END
  25 1..1
  26 ok 1
  27 ok 2
  28 END
  29 $MAKE check >stdout && { cat stdout; exit 1; }
  30 cat stdout
  31 count_test_results total=3 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=2
  32 grep '^ERROR: all\.test - too many tests run (expected 1, got 2)$' stdout
  33 grep '^ERROR: all\.test 2 # UNPLANNED$' stdout
  34
  35 cat > all.test <<END
  36 1..2
  37 ok 1
  38 ok 2
  39 ok 3
  40 END
  41 $MAKE check >stdout && { cat stdout; exit 1; }
  42 cat stdout
  43 count_test_results total=4 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=2
  44 grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
  45 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
  46
  47 # Interaction with XFAIL_TESTS.
  48 cat > all.test <<END
  49 1..2
  50 not ok 1
  51 ok 2 # SKIP
  52 ok 3
  53 not ok 4
  54 ok 5 # SKIP
  55 END
  56 XFAIL_TESTS=all.test $MAKE -e check >stdout && { cat stdout; exit 1; }
  57 cat stdout
  58 count_test_results total=6 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=4
  59 grep '^ERROR: all\.test - too many tests run (expected 2, got 5)$' stdout
  60 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
  61 grep '^ERROR: all\.test 4 # UNPLANNED$' stdout
  62 grep '^ERROR: all\.test 5 # UNPLANNED$' stdout
  63
  64 cat > all.test <<END
  65 1..1
  66 ok 1
  67
  68 ok
  69 ok 3
  70 ok foo
  71 ok 5 - bar bar
  72
  73 not ok
  74 not ok 7
  75 not ok foo
  76 not ok 9 - bar bar
  77
  78 ok # TODO
  79 ok 11 # TODO
  80 ok foo # TODO
  81 ok 13 - bar bar # TODO
  82
  83 not ok # TODO
  84 not ok 15 # TODO
  85 not ok foo # TODO
  86 not ok 17 - bar bar # TODO
  87
  88 ok # SKIP
  89 ok 19 # SKIP
  90 ok foo # SKIP
  91 ok 21 - bar bar # SKIP
  92 END
  93
  94 cat > t <<END
  95
  96 2
  97 3
  98 4 foo
  99 5 - bar bar
 100
 101 6
 102 7
 103 8 foo
 104 9 - bar bar
 105
 106 10
 107 11
 108 12 foo
 109 13 - bar bar
 110
 111 14
 112 15
 113 16 foo
 114 17 - bar bar
 115
 116 18
 117 19
 118 20 foo
 119 21 - bar bar
 120
 121 END
 122
 123 $MAKE check >stdout && { cat stdout; exit 1; }
 124 cat stdout
 125 count_test_results total=22 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=21
 126
 127 echo 'PASS: all.test 1' > exp
 128 sed -e '/^$/d' -e 's/.*/ERROR: all.test & # UNPLANNED/' t >> exp
 129 echo 'ERROR: all.test - too many tests run (expected 1, got 21)' >> exp
 130
 131 $FGREP ': all.test' stdout > got
 132
 133 cat exp
 134 cat got
 135 diff exp got
 136
 137 # Note that, if the TAP input has a trailing plan, it is not possible
 138 # to flag unplanned tests as such, since we do not know they're unplanned
 139 # until the plan is reached; still, we should give at least an error
 140 # message about the unmatched number of tests once we've got the plan.
 141
 142 for x in 'ok' 'ok 3' 'not ok' 'not ok # TODO' 'ok # TODO' 'ok # SKIP'; do
 143   unindent > all.test <<END
 144     ok 1
 145     ok 2 # SKIP
 146     $x
 147     1..2
 148 END
 149   $MAKE check >stdout && { cat stdout; exit 1; }
 150   cat stdout
 151   test $($FGREP -c ': all.test' stdout) -eq 4
 152   $EGREP '^PASS: all\.test 1($| )' stdout
 153   $EGREP '^SKIP: all\.test 2($| )' stdout
 154   $EGREP ': all\.test 3($| )' stdout
 155   grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
 156 done
 157
 158 :