t/test-metadata-global-result.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2018 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # Parallel testsuite harness: check APIs for the registering the
  18 # "global test result" in '*.trs' files, as documented in the automake
  19 # manual.
  20
  21 . test-init.sh
  22
  23 cat >> configure.ac << 'END'
  24 AC_OUTPUT
  25 END
  26
  27 cat > Makefile.am << 'END'
  28 TEST_EXTENSIONS = .test .x
  29 TEST_LOG_DRIVER = ./dummy-driver
  30 X_LOG_DRIVER = ./dummy-driver
  31 TESTS = foo.test zar-doz.test
  32 END
  33
  34 cat > dummy-driver <<'END'
  35 #! /bin/sh
  36 set -e; set -u
  37 while test $# -gt 0; do
  38   case $1 in
  39     --log-file) log_file=$2; shift;;
  40     --trs-file) trs_file=$2; shift;;
  41     --test-name) test_name=$2; shift;;
  42     --expect-failure|--color-tests|--enable-hard-errors) shift;;
  43     --) shift; break;;
  44      *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
  45   esac
  46   shift
  47 done
  48 echo logloglog > $log_file
  49 cp $1 $trs_file
  50 END
  51 chmod a+x dummy-driver
  52
  53 # Do this in a subroutine to avoid quoting problem in the backticked
  54 # command substitution below.
  55 get_escaped_line()
  56 {
  57    sed -e 's,[$^/\\\.],\\&,g' -e 1q "$@"
  58 }
  59
  60 have_result ()
  61 {
  62    cat > exp; echo >> exp; echo logloglog >> exp
  63    eline=$(get_escaped_line exp)
  64    sed -n -e "/^$eline$/,/^logloglog$/p" test-suite.log > got
  65    cat exp; cat got
  66    diff exp got
  67 }
  68
  69 $ACLOCAL
  70 $AUTOCONF
  71 $AUTOMAKE
  72
  73 ./configure
  74
  75 : Basic checks.
  76
  77 echo :global-test-result: PASS > foo.test
  78 echo :global-test-result: ERROR > zar-doz.x
  79
  80 $MAKE check
  81 cat test-suite.log
  82
  83 have_result <<END
  84 PASS: foo
  85 =========
  86 END
  87
  88 have_result <<END
  89 ERROR: zar-doz
  90 ==============
  91 END
  92
  93 : Try usage documented in the manual.
  94
  95 echo :global-test-result: PASS/SKIP > foo.test
  96 echo :global-test-result: ALMOST PASSED > zar-doz.x
  97
  98 $MAKE check
  99 cat test-suite.log
 100
 101 have_result <<END
 102 PASS/SKIP: foo
 103 ==============
 104 END
 105
 106 have_result <<END
 107 ALMOST PASSED: zar-doz
 108 ======================
 109 END
 110
 111 : Fields ':test-result:' does not interfere with the global test result.
 112
 113 cat > foo.test << 'END'
 114 :test-result: FAIL
 115 :global-test-result: PASS
 116 :test-result: ERROR
 117 END
 118
 119 cat > zar-doz.x << 'END'
 120 :global-test-result: FAIL
 121 :test-result: SKIP
 122 :test-result: XFAIL
 123 END
 124
 125 $MAKE check && exit 1
 126 cat test-suite.log
 127
 128 have_result <<END
 129 PASS: foo
 130 =========
 131 END
 132
 133 have_result <<END
 134 FAIL: zar-doz
 135 =============
 136 END
 137
 138 : What happens when ':global-test-result:' is absent.
 139
 140 cat > foo.test << 'END'
 141 :test-result: PASS
 142 :test-result: ERROR
 143 END
 144 : > zar-doz.x
 145
 146 $MAKE check && exit 1
 147 cat test-suite.log
 148
 149 have_result <<END
 150 RUN: foo
 151 ========
 152 END
 153
 154 have_result <<END
 155 RUN: zar-doz
 156 ============
 157 END
 158
 159 # Leading and trailing whitespace gets eaten/normalized.
 160
 161 echo ":global-test-result:SKIP${tab}   ${tab}${tab}" > foo.test
 162 echo ":global-test-result:${tab}   ${tab}XFAIL  ${tab}   " > zar-doz.x
 163
 164 $MAKE check
 165 cat test-suite.log
 166
 167 have_result <<END
 168 SKIP: foo
 169 =========
 170 END
 171
 172 have_result <<END
 173 XFAIL: zar-doz
 174 ==============
 175 END
 176
 177 # Whitespaces before and after ':global-test-result:' are handled OK.
 178
 179 echo "   $tab:global-test-result:PASS" > foo.test
 180 echo "${tab}${tab}:global-test-result:${tab}   ${tab}SKIP" > zar-doz.x
 181
 182 $MAKE check
 183 cat test-suite.log
 184
 185 have_result <<END
 186 PASS: foo
 187 =========
 188 END
 189
 190 have_result <<END
 191 SKIP: zar-doz
 192 =============
 193 END
 194
 195 :