t/tap-basic.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2017 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # Basic TAP support:
  18 #  - LOG_COMPILER support;
  19 #  - basic support for TODO and SKIP directives, and "Bail out!" magic;
  20 #  - testsuite progress output on console;
  21 #  - runtime overriding of TESTS and TEST_LOGS;
  22 #  - correct counts of test results (both in summary and in progress
  23 #    output on console).
  24 # Note that some of the features checked here are checked in other
  25 # test cases too, usually in a more thorough and detailed way.
  26
  27 . test-init.sh
  28
  29 fetch_tap_driver
  30
  31 cat >> configure.ac <<END
  32 AC_OUTPUT
  33 END
  34
  35 cat > Makefile.am << 'END'
  36 TEST_LOG_DRIVER = $(srcdir)/tap-driver
  37 ## Defining LOG_COMPILER should work and not intefere with the
  38 ## tap-driver script.
  39 TEST_LOG_COMPILER = cat
  40 TESTS = success.test
  41
  42 ok.test:
  43         echo '1..3' > $@-t
  44         echo 'ok 1' >> $@-t
  45         echo 'not ok 2 # TODO' >>$@-t
  46         echo 'ok 3 # SKIP' >>$@-t
  47         cat $@-t ;: For debugging.
  48         mv -f $@-t $@
  49 END
  50
  51 cat > success.test << 'END'
  52 1..20
  53 ok 1
  54 ok 2 two
  55 ok 3 - three
  56 ok 4 four four
  57 not ok 5
  58 not ok 6 six
  59 not ok 7 - seven
  60 not ok 8 eight eight
  61 ok 9 # TODO
  62 ok 10 ten # TODO
  63 ok 11 - eleven # TODO
  64 ok 12 twelve twelve # TODO
  65 not ok 13 # TODO
  66 not ok 14 fourteen # TODO
  67 not ok 15 - fifteen # TODO
  68 not ok 16 sixteen sixteen # TODO
  69 ok 17 # SKIP
  70 ok 18 eighteen # SKIP
  71 ok 19 - nineteen # SKIP
  72 ok 20 twenty twenty # SKIP
  73 END
  74
  75 $ACLOCAL
  76 $AUTOCONF
  77 $AUTOMAKE
  78
  79 ./configure
  80
  81 # Basilar usage and testsuite progress output.
  82
  83 run_make -O -e FAIL check
  84 count_test_results total=20 pass=4 fail=4 xpass=4 xfail=4 skip=4 error=0
  85
  86 test -f success.log
  87 test -f test-suite.log
  88
  89 cat > exp << 'END'
  90 PASS: success.test 1
  91 PASS: success.test 2 two
  92 PASS: success.test 3 - three
  93 PASS: success.test 4 four four
  94 FAIL: success.test 5
  95 FAIL: success.test 6 six
  96 FAIL: success.test 7 - seven
  97 FAIL: success.test 8 eight eight
  98 XPASS: success.test 9 # TODO
  99 XPASS: success.test 10 ten # TODO
 100 XPASS: success.test 11 - eleven # TODO
 101 XPASS: success.test 12 twelve twelve # TODO
 102 XFAIL: success.test 13 # TODO
 103 XFAIL: success.test 14 fourteen # TODO
 104 XFAIL: success.test 15 - fifteen # TODO
 105 XFAIL: success.test 16 sixteen sixteen # TODO
 106 SKIP: success.test 17 # SKIP
 107 SKIP: success.test 18 eighteen # SKIP
 108 SKIP: success.test 19 - nineteen # SKIP
 109 SKIP: success.test 20 twenty twenty # SKIP
 110 END
 111
 112 $FGREP ': success.test' stdout > got
 113
 114 cat exp
 115 cat got
 116 diff exp got
 117
 118 # Override TESTS from the command line.
 119
 120 rm -f *.log *.test
 121
 122 cat > bail.test <<'END'
 123 1..1
 124 Bail out!
 125 ok 1
 126 END
 127
 128 run_make -O -e FAIL check TESTS=bail.test
 129 count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
 130
 131 test ! -e success.log
 132 test -f bail.log
 133 test -f test-suite.log
 134
 135 grep '^ERROR: bail\.test - Bail out!' stdout
 136 grep '^PASS:' stdout && exit 1
 137 test $($FGREP -c ': bail.test' stdout) -eq 1
 138 $FGREP 'success.test' stdout && exit 1
 139
 140 # Override TEST_LOGS from the command line, making it point to a test
 141 # (ok.test) that has to be generated at make time.
 142
 143 rm -f *.log *.test
 144
 145 run_make -O check TEST_LOGS=ok.log
 146 count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
 147
 148 test -f ok.test
 149 test -f ok.log
 150 test ! -e success.log
 151 test ! -e bail.log
 152 test -f test-suite.log
 153
 154 $EGREP '(bail|success)\.test' stdout && exit 1
 155
 156 cat > exp << 'END'
 157 PASS: ok.test 1
 158 XFAIL: ok.test 2 # TODO
 159 SKIP: ok.test 3 # SKIP
 160 END
 161
 162 $FGREP ': ok.test' stdout > got
 163
 164 cat exp
 165 cat got
 166 diff exp got
 167
 168 :