t/test-metadata-results.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2017 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # Parallel testsuite harness: check APIs for the registering of test
  18 # results in '*.trs' files, as documented in the automake manual.
  19
  20 . test-init.sh
  21
  22 cat >> configure.ac << 'END'
  23 AC_OUTPUT
  24 END
  25
  26 cat > Makefile.am << 'END'
  27 TEST_LOG_DRIVER = ./dummy-driver
  28 TESTS = foo.test bar.test
  29 END
  30
  31 cat > dummy-driver <<'END'
  32 #! /bin/sh
  33 set -e; set -u
  34 while test $# -gt 0; do
  35   case $1 in
  36     --log-file) log_file=$2; shift;;
  37     --trs-file) trs_file=$2; shift;;
  38     --test-name) test_name=$2; shift;;
  39     --expect-failure|--color-tests|--enable-hard-errors) shift;;
  40     --) shift; break;;
  41      *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
  42   esac
  43   shift
  44 done
  45 : > $log_file
  46 cp $1 $trs_file
  47 END
  48 chmod a+x dummy-driver
  49
  50 mk_check ()
  51 {
  52   stat=0
  53   case $1 in
  54     -e) stat=$2; shift 2;;
  55   esac
  56   run_make -O -e $stat -- check
  57   # Our dummy driver make no testsuite progress report.
  58   grep ': .*\.test' stdout && exit 1
  59   # Nor it writes to the log files.
  60   test -s foo.log && exit 1
  61   test -s bar.log && exit 1
  62   : For 'set -e'.
  63 }
  64
  65 # This must be different from the one defined in 'test/defs', as that
  66 # assumes that the driver does proper testsuite progress reporting.
  67 count_test_results ()
  68 {
  69   total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
  70   eval "$@"
  71   st=0
  72   grep "^# TOTAL:  *$total$" stdout || rc=1
  73   grep "^# PASS:  *$pass$"   stdout || rc=1
  74   grep "^# XFAIL:  *$xfail$" stdout || rc=1
  75   grep "^# SKIP:  *$skip$"   stdout || rc=1
  76   grep "^# FAIL:  *$fail$"   stdout || rc=1
  77   grep "^# XPASS:  *$xpass$" stdout || rc=1
  78   grep "^# ERROR:  *$error$" stdout || rc=1
  79   test $st -eq 0 || exit 1
  80 }
  81
  82 $ACLOCAL
  83 $AUTOCONF
  84 $AUTOMAKE
  85
  86 ./configure
  87
  88 # Basic checks.  Also that that ':global-test-result:' fields and
  89 # "old-style" directives with format "RESULT: test-name" are now ignored.
  90
  91 : > foo.test
  92 echo blah blah blah > bar.test
  93 mk_check
  94 count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
  95
  96 cat > foo.test <<END
  97 :test-global-result: PASS
  98 :test-result: FAIL
  99 END
 100 cat > bar.test <<END
 101 :test-result: SKIP
 102 :test-global-result: ERROR
 103 END
 104 mk_check -e FAIL
 105 count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
 106
 107 cat > foo.test <<END
 108 FAIL: foo.test
 109 :test-result: PASS
 110 :test-global-result: XPASS
 111 END
 112 echo ERROR: bar.test > bar.test
 113 mk_check
 114 count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
 115
 116 cat > foo.test <<END
 117 :test-global-result: SKIP
 118 :test-result: FAIL
 119 END
 120 cat > bar.test <<END
 121 :test-global-result: PASS
 122 END
 123 mk_check -e FAIL
 124 count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
 125
 126 cat > foo.test <<END
 127 :test-result: XFAIL
 128 :test-result: PASS
 129 :test-result: SKIP
 130 END
 131 cat > bar.test <<END
 132 :test-result: SKIP
 133 :test-result: PASS
 134 :test-result: SKIP
 135 :test-result: PASS
 136 :test-result: PASS
 137 END
 138 mk_check
 139 count_test_results total=8 pass=4 fail=0 xpass=0 xfail=1 skip=3 error=0
 140
 141 # Check that all results expected to be supported are *really* supported.
 142
 143 cat > foo.test <<END
 144 :test-result: PASS
 145 :test-result: SKIP
 146 :test-result: XFAIL
 147 :test-result: FAIL
 148 :test-result: XPASS
 149 :test-result: ERROR
 150 END
 151 : > bar.test
 152 mk_check -e FAIL
 153 count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
 154
 155 cp foo.test bar.test
 156 mk_check -e FAIL
 157 count_test_results total=12 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=2
 158
 159 # Check that we are liberal w.r.t. whitespace use.
 160
 161 : > foo.test
 162 : > bar.test
 163 for RESULT in PASS FAIL XPASS XFAIL SKIP ERROR; do
 164   sed -e 's/^ *//' -e 's/|//g' >> foo.test <<END
 165     |:test-result:$RESULT|
 166     |:test-result: $tab  $RESULT|
 167     |:test-result:$RESULT  $tab|
 168     |:test-result:$tab$tab  $RESULT$tab  $tab |
 169 END
 170   echo "  $tab $tab$tab   :test-result: $RESULT" >> bar.test
 171 done
 172 cat foo.test # For debugging.
 173 cat bar.test # Likewise.
 174 mk_check -e FAIL
 175 count_test_results total=30 pass=5 fail=5 xpass=5 xfail=5 skip=5 error=5
 176
 177 :