t/ax/tap-summary-aux.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2018 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # Auxiliary script for tests on TAP support: checking testsuite summary.
  18
  19 . test-init.sh
  20
  21 br='============================================================================'
  22
  23 case $use_colors in
  24   yes|no) ;;
  25   *) fatal_ "invalid \$use_colors value '$use_colors'"
  26 esac
  27
  28 fetch_tap_driver
  29
  30 cat > configure.ac <<END
  31 AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
  32 AM_INIT_AUTOMAKE
  33 AC_CONFIG_FILES([Makefile])
  34 AC_OUTPUT
  35 END
  36
  37 cat > Makefile.am << 'END'
  38 TEST_LOG_DRIVER = $(srcdir)/tap-driver
  39 TEST_LOG_COMPILER = cat
  40 TESTS = all.test
  41 END
  42
  43 # The following shell variables are influential for this function:
  44 #  - expect_failure
  45 #  - use_colors
  46 do_check ()
  47 {
  48   case $#,$1 in
  49     1,--pass) expect_failure=no;;
  50     1,--fail) expect_failure=yes;;
  51            *) fatal_ "invalid usage of 'do_check'";;
  52   esac
  53   shift
  54   cat > summary.exp
  55   cat all.test
  56   if test $use_colors = yes; then
  57     # Forced colorization should take place also with non-ANSI terminals;
  58     # hence the "TERM=dumb" definition.
  59     make_args='TERM=dumb AM_COLOR_TESTS=always'
  60   else
  61     make_args=
  62   fi
  63   run_make -O -e IGNORE $make_args check
  64   if test $expect_failure = yes; then
  65     test $am_make_rc -gt 0 || exit 1
  66   else
  67     test $am_make_rc -eq 0 || exit 1
  68   fi
  69   $PERL "$am_testaux_srcdir"/extract-testsuite-summary.pl stdout >summary.got \
  70     || fatal_ "cannot extract testsuite summary"
  71   cat summary.exp
  72   cat summary.got
  73   if test $use_colors = yes; then
  74     # Use cmp, not diff, because the files might contain binary data.
  75     compare=cmp
  76   else
  77     compare=diff
  78   fi
  79   $compare summary.exp summary.got || exit 1
  80 }
  81
  82 if test $use_colors = yes; then
  83   red="$esc[0;31m"
  84   grn="$esc[0;32m"
  85   lgn="$esc[1;32m"
  86   blu="$esc[1;34m"
  87   mgn="$esc[0;35m"
  88   brg="$esc[1m"
  89   std="$esc[m"
  90 else
  91   red= grn= lgn= blu= mgn= brg= std=
  92 fi
  93
  94   success_header="\
  95 ${grn}${br}${std}
  96 ${grn}Testsuite summary for GNU AutoTAP 5.12${std}
  97 ${grn}${br}${std}"
  98
  99   success_footer=${grn}${br}${std}
 100
 101   failure_header="\
 102 ${red}${br}${std}
 103 ${red}Testsuite summary for GNU AutoTAP 5.12${std}
 104 ${red}${br}${std}"
 105
 106   failure_footer="\
 107 ${red}${br}${std}
 108 ${red}See ./test-suite.log${std}
 109 ${red}Please report to bug-automake@gnu.org${std}
 110 ${red}${br}${std}"
 111
 112 $ACLOCAL
 113 $AUTOCONF
 114 $AUTOMAKE --add-missing
 115
 116 ./configure
 117
 118 # 1 pass.
 119 { echo 1..1 && echo ok; } > all.test
 120 do_check --pass <<END
 121 $success_header
 122 ${brg}# TOTAL: 1${std}
 123 ${grn}# PASS:  1${std}
 124 # SKIP:  0
 125 # XFAIL: 0
 126 # FAIL:  0
 127 # XPASS: 0
 128 # ERROR: 0
 129 $success_footer
 130 END
 131
 132 # 1 skip.
 133 { echo 1..1 && echo 'ok # SKIP'; } > all.test
 134 do_check --pass <<END
 135 $success_header
 136 ${brg}# TOTAL: 1${std}
 137 # PASS:  0
 138 ${blu}# SKIP:  1${std}
 139 # XFAIL: 0
 140 # FAIL:  0
 141 # XPASS: 0
 142 # ERROR: 0
 143 $success_footer
 144 END
 145
 146 # 1 xfail.
 147 { echo 1..1 && echo 'not ok # TODO'; } > all.test
 148 do_check --pass <<END
 149 $success_header
 150 ${brg}# TOTAL: 1${std}
 151 # PASS:  0
 152 # SKIP:  0
 153 ${lgn}# XFAIL: 1${std}
 154 # FAIL:  0
 155 # XPASS: 0
 156 # ERROR: 0
 157 $success_footer
 158 END
 159
 160 # 1 fail.
 161 { echo 1..1 && echo not ok; } > all.test
 162 do_check --fail <<END
 163 $failure_header
 164 ${brg}# TOTAL: 1${std}
 165 # PASS:  0
 166 # SKIP:  0
 167 # XFAIL: 0
 168 ${red}# FAIL:  1${std}
 169 # XPASS: 0
 170 # ERROR: 0
 171 $failure_footer
 172 END
 173
 174 # 1 xpass.
 175 { echo 1..1 && echo 'ok # TODO'; } > all.test
 176 do_check --fail <<END
 177 $failure_header
 178 ${brg}# TOTAL: 1${std}
 179 # PASS:  0
 180 # SKIP:  0
 181 # XFAIL: 0
 182 # FAIL:  0
 183 ${red}# XPASS: 1${std}
 184 # ERROR: 0
 185 $failure_footer
 186 END
 187
 188 # 1 hard error.
 189 { echo 1..1 && echo 'Bail out!'; } > all.test
 190 do_check --fail <<END
 191 $failure_header
 192 ${brg}# TOTAL: 1${std}
 193 # PASS:  0
 194 # SKIP:  0
 195 # XFAIL: 0
 196 # FAIL:  0
 197 # XPASS: 0
 198 ${mgn}# ERROR: 1${std}
 199 $failure_footer
 200 END
 201
 202 # 3 non-failing results.
 203 cat > all.test <<END
 204 1..3
 205 ok
 206 not ok # TODO
 207 ok # SKIP
 208 END
 209 do_check --pass <<END
 210 $success_header
 211 ${brg}# TOTAL: 3${std}
 212 ${grn}# PASS:  1${std}
 213 ${blu}# SKIP:  1${std}
 214 ${lgn}# XFAIL: 1${std}
 215 # FAIL:  0
 216 # XPASS: 0
 217 # ERROR: 0
 218 $success_footer
 219 END
 220
 221 # 1 pass, 1 skip, 1 fail.
 222 cat > all.test <<END
 223 1..3
 224 ok
 225 ok # SKIP
 226 not ok
 227 END
 228 do_check --fail <<END
 229 $failure_header
 230 ${brg}# TOTAL: 3${std}
 231 ${grn}# PASS:  1${std}
 232 ${blu}# SKIP:  1${std}
 233 # XFAIL: 0
 234 ${red}# FAIL:  1${std}
 235 # XPASS: 0
 236 # ERROR: 0
 237 $failure_footer
 238 END
 239
 240 # 1 pass, 1 xfail, 1 xpass.
 241 cat > all.test <<END
 242 1..3
 243 ok
 244 ok # TODO
 245 not ok # TODO
 246 END
 247 do_check --fail <<END
 248 $failure_header
 249 ${brg}# TOTAL: 3${std}
 250 ${grn}# PASS:  1${std}
 251 # SKIP:  0
 252 ${lgn}# XFAIL: 1${std}
 253 # FAIL:  0
 254 ${red}# XPASS: 1${std}
 255 # ERROR: 0
 256 $failure_footer
 257 END
 258
 259 # 1 skip, 1 xfail, 1 error.
 260 cat > all.test <<END
 261 1..3
 262 ok # SKIP
 263 not ok # TODO
 264 Bail out!
 265 END
 266 do_check --fail <<END
 267 $failure_header
 268 ${brg}# TOTAL: 3${std}
 269 # PASS:  0
 270 ${blu}# SKIP:  1${std}
 271 ${lgn}# XFAIL: 1${std}
 272 # FAIL:  0
 273 # XPASS: 0
 274 ${mgn}# ERROR: 1${std}
 275 $failure_footer
 276 END
 277
 278 # 1 of each kind
 279 cat > all.test <<END
 280 1..6
 281 ok
 282 not ok
 283 ok # TODO
 284 not ok # TODO
 285 ok # SKIP
 286 Bail out!
 287 END
 288 do_check --fail <<END
 289 $failure_header
 290 ${brg}# TOTAL: 6${std}
 291 ${grn}# PASS:  1${std}
 292 ${blu}# SKIP:  1${std}
 293 ${lgn}# XFAIL: 1${std}
 294 ${red}# FAIL:  1${std}
 295 ${red}# XPASS: 1${std}
 296 ${mgn}# ERROR: 1${std}
 297 $failure_footer
 298 END
 299
 300 # Prepare some common data for later.
 301 for i in 0 1 2 3 4 5 6 7 8 9; do
 302   for j in 0 1 2 3 4 5 6 7 8 9; do
 303     echo "ok"
 304     echo "not ok # TODO"
 305     echo "ok # SKIP"
 306   done
 307 done > tap
 308
 309 # Lots of non-failures (300 per kind).
 310 (cat tap && cat tap && cat tap) > all.test
 311 test $(wc -l <all.test) -eq 900 || exit 99 # Sanity check.
 312 echo 1..900 >> all.test # Test plan.
 313 do_check --pass <<END
 314 $success_header
 315 ${brg}# TOTAL: 900${std}
 316 ${grn}# PASS:  300${std}
 317 ${blu}# SKIP:  300${std}
 318 ${lgn}# XFAIL: 300${std}
 319 # FAIL:  0
 320 # XPASS: 0
 321 # ERROR: 0
 322 $success_footer
 323 END
 324
 325 # 1 failure and lots of non-failures means failure.
 326 (cat tap && echo "not ok" && cat tap) > all.test
 327 test $(wc -l <all.test) -eq 601 || exit 99 # Sanity check.
 328 echo 1..601 >> all.test # Test plan.
 329 do_check --fail <<END
 330 $failure_header
 331 ${brg}# TOTAL: 601${std}
 332 ${grn}# PASS:  200${std}
 333 ${blu}# SKIP:  200${std}
 334 ${lgn}# XFAIL: 200${std}
 335 ${red}# FAIL:  1${std}
 336 # XPASS: 0
 337 # ERROR: 0
 338 $failure_footer
 339 END
 340
 341 # 1 error and lots of non-failures means failure.
 342 (cat tap && sed 30q tap && echo 'Bail out!') > all.test
 343 test $(wc -l <all.test) -eq 331 || exit 99 # Sanity check.
 344 echo 1..331 >> all.test # Test plan.
 345 do_check --fail <<END
 346 $failure_header
 347 ${brg}# TOTAL: 331${std}
 348 ${grn}# PASS:  110${std}
 349 ${blu}# SKIP:  110${std}
 350 ${lgn}# XFAIL: 110${std}
 351 # FAIL:  0
 352 # XPASS: 0
 353 ${mgn}# ERROR: 1${std}
 354 $failure_footer
 355 END
 356
 357 :