tests/tap-summary-aux.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 # Auxiliary script for tests on TAP support: checking testsuite summary.
  18
  19 am_parallel_tests=yes
  20 . ./defs || Exit 1
  21
  22 br='============================================================================'
  23
  24 case $use_colors in
  25   yes|no) ;;
  26   *) fatal_ "invalid \$use_colors value '$use_colors'"
  27 esac
  28
  29 fetch_tap_driver
  30
  31 cat > configure.in <<END
  32 AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
  33 AM_INIT_AUTOMAKE([parallel-tests])
  34 AC_CONFIG_FILES([Makefile])
  35 AC_OUTPUT
  36 END
  37
  38 cat > Makefile.am << 'END'
  39 TEST_LOG_DRIVER = $(srcdir)/tap-driver
  40 TEST_LOG_COMPILER = cat
  41 TESTS = all.test
  42 END
  43
  44 # The following shell variables are influential for this function:
  45 #  - expect_failure
  46 #  - use_colors
  47 do_check ()
  48 {
  49   case $#,$1 in
  50     1,--pass) expect_failure=no;;
  51     1,--fail) expect_failure=yes;;
  52            *) fatal_ "invalid usage of 'do_check'";;
  53   esac
  54   shift
  55   cat > summary.exp
  56   cat all.test
  57   st=0
  58   if test $use_colors = yes; then
  59     make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
  60   else
  61     make_cmd=$MAKE
  62   fi
  63   $make_cmd check > stdout || st=$?
  64   cat stdout
  65   if test $expect_failure = yes; then
  66     test $st -gt 0 || Exit 1
  67   else
  68     test $st -eq 0 || Exit 1
  69   fi
  70   $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
  71     || fatal_ "cannot extract testsuite summary"
  72   cat summary.exp
  73   cat summary.got
  74   if test $use_colors = yes; then
  75     # Use cmp, not diff, because the files might contain binary data.
  76     compare=cmp
  77   else
  78     compare=diff
  79   fi
  80   $compare summary.exp summary.got || Exit 1
  81 }
  82
  83 if test $use_colors = yes; then
  84   red='\e[0;31m'
  85   grn='\e[0;32m'
  86   lgn='\e[1;32m'
  87   blu='\e[1;34m'
  88   mgn='\e[0;35m'
  89   brg='\e[1m'
  90   std='\e[m'
  91   echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
  92 else
  93   red= grn= lgn= blu= mgn= brg= std=
  94 fi
  95
  96   success_header="\
  97 ${grn}${br}${std}
  98 ${grn}Testsuite summary for GNU AutoTAP 5.12${std}
  99 ${grn}${br}${std}"
 100
 101   success_footer=${grn}${br}${std}
 102
 103   failure_header="\
 104 ${red}${br}${std}
 105 ${red}Testsuite summary for GNU AutoTAP 5.12${std}
 106 ${red}${br}${std}"
 107
 108   failure_footer="\
 109 ${red}${br}${std}
 110 ${red}See ./test-suite.log${std}
 111 ${red}Please report to bug-automake@gnu.org${std}
 112 ${red}${br}${std}"
 113
 114 $ACLOCAL
 115 $AUTOCONF
 116 $AUTOMAKE --add-missing
 117
 118 ./configure
 119
 120 # 1 pass.
 121 { echo 1..1 && echo ok; } > all.test
 122 do_check --pass <<END
 123 $success_header
 124 ${brg}# TOTAL: 1${std}
 125 ${grn}# PASS:  1${std}
 126 # SKIP:  0
 127 # XFAIL: 0
 128 # FAIL:  0
 129 # XPASS: 0
 130 # ERROR: 0
 131 $success_footer
 132 END
 133
 134 # 1 skip.
 135 { echo 1..1 && echo 'ok # SKIP'; } > all.test
 136 do_check --pass <<END
 137 $success_header
 138 ${brg}# TOTAL: 1${std}
 139 # PASS:  0
 140 ${blu}# SKIP:  1${std}
 141 # XFAIL: 0
 142 # FAIL:  0
 143 # XPASS: 0
 144 # ERROR: 0
 145 $success_footer
 146 END
 147
 148 # 1 xfail.
 149 { echo 1..1 && echo 'not ok # TODO'; } > all.test
 150 do_check --pass <<END
 151 $success_header
 152 ${brg}# TOTAL: 1${std}
 153 # PASS:  0
 154 # SKIP:  0
 155 ${lgn}# XFAIL: 1${std}
 156 # FAIL:  0
 157 # XPASS: 0
 158 # ERROR: 0
 159 $success_footer
 160 END
 161
 162 # 1 fail.
 163 { echo 1..1 && echo not ok; } > all.test
 164 do_check --fail <<END
 165 $failure_header
 166 ${brg}# TOTAL: 1${std}
 167 # PASS:  0
 168 # SKIP:  0
 169 # XFAIL: 0
 170 ${red}# FAIL:  1${std}
 171 # XPASS: 0
 172 # ERROR: 0
 173 $failure_footer
 174 END
 175
 176 # 1 xpass.
 177 { echo 1..1 && echo 'ok # TODO'; } > all.test
 178 do_check --fail <<END
 179 $failure_header
 180 ${brg}# TOTAL: 1${std}
 181 # PASS:  0
 182 # SKIP:  0
 183 # XFAIL: 0
 184 # FAIL:  0
 185 ${red}# XPASS: 1${std}
 186 # ERROR: 0
 187 $failure_footer
 188 END
 189
 190 # 1 hard error.
 191 { echo 1..1 && echo 'Bail out!'; } > all.test
 192 do_check --fail <<END
 193 $failure_header
 194 ${brg}# TOTAL: 1${std}
 195 # PASS:  0
 196 # SKIP:  0
 197 # XFAIL: 0
 198 # FAIL:  0
 199 # XPASS: 0
 200 ${mgn}# ERROR: 1${std}
 201 $failure_footer
 202 END
 203
 204 # 3 non-failing results.
 205 cat > all.test <<END
 206 1..3
 207 ok
 208 not ok # TODO
 209 ok # SKIP
 210 END
 211 do_check --pass <<END
 212 $success_header
 213 ${brg}# TOTAL: 3${std}
 214 ${grn}# PASS:  1${std}
 215 ${blu}# SKIP:  1${std}
 216 ${lgn}# XFAIL: 1${std}
 217 # FAIL:  0
 218 # XPASS: 0
 219 # ERROR: 0
 220 $success_footer
 221 END
 222
 223 # 1 pass, 1 skip, 1 fail.
 224 cat > all.test <<END
 225 1..3
 226 ok
 227 ok # SKIP
 228 not ok
 229 END
 230 do_check --fail <<END
 231 $failure_header
 232 ${brg}# TOTAL: 3${std}
 233 ${grn}# PASS:  1${std}
 234 ${blu}# SKIP:  1${std}
 235 # XFAIL: 0
 236 ${red}# FAIL:  1${std}
 237 # XPASS: 0
 238 # ERROR: 0
 239 $failure_footer
 240 END
 241
 242 # 1 pass, 1 xfail, 1 xpass.
 243 cat > all.test <<END
 244 1..3
 245 ok
 246 ok # TODO
 247 not ok # TODO
 248 END
 249 do_check --fail <<END
 250 $failure_header
 251 ${brg}# TOTAL: 3${std}
 252 ${grn}# PASS:  1${std}
 253 # SKIP:  0
 254 ${lgn}# XFAIL: 1${std}
 255 # FAIL:  0
 256 ${red}# XPASS: 1${std}
 257 # ERROR: 0
 258 $failure_footer
 259 END
 260
 261 # 1 skip, 1 xfail, 1 error.
 262 cat > all.test <<END
 263 1..3
 264 ok # SKIP
 265 not ok # TODO
 266 Bail out!
 267 END
 268 do_check --fail <<END
 269 $failure_header
 270 ${brg}# TOTAL: 3${std}
 271 # PASS:  0
 272 ${blu}# SKIP:  1${std}
 273 ${lgn}# XFAIL: 1${std}
 274 # FAIL:  0
 275 # XPASS: 0
 276 ${mgn}# ERROR: 1${std}
 277 $failure_footer
 278 END
 279
 280 # 1 of each kind
 281 cat > all.test <<END
 282 1..6
 283 ok
 284 not ok
 285 ok # TODO
 286 not ok # TODO
 287 ok # SKIP
 288 Bail out!
 289 END
 290 do_check --fail <<END
 291 $failure_header
 292 ${brg}# TOTAL: 6${std}
 293 ${grn}# PASS:  1${std}
 294 ${blu}# SKIP:  1${std}
 295 ${lgn}# XFAIL: 1${std}
 296 ${red}# FAIL:  1${std}
 297 ${red}# XPASS: 1${std}
 298 ${mgn}# ERROR: 1${std}
 299 $failure_footer
 300 END
 301
 302 # Prepare some common data for later.
 303 for i in 0 1 2 3 4 5 6 7 8 9; do
 304   for j in 0 1 2 3 4 5 6 7 8 9; do
 305     echo "ok"
 306     echo "not ok # TODO"
 307     echo "ok # SKIP"
 308   done
 309 done > tap
 310
 311 # Lots of non-failures (300 per kind).
 312 (cat tap && cat tap && cat tap) > all.test
 313 test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
 314 echo 1..900 >> all.test # Test plan.
 315 do_check --pass <<END
 316 $success_header
 317 ${brg}# TOTAL: 900${std}
 318 ${grn}# PASS:  300${std}
 319 ${blu}# SKIP:  300${std}
 320 ${lgn}# XFAIL: 300${std}
 321 # FAIL:  0
 322 # XPASS: 0
 323 # ERROR: 0
 324 $success_footer
 325 END
 326
 327 # 1 failure and lots of non-failures means failure.
 328 (cat tap && echo "not ok" && cat tap) > all.test
 329 test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
 330 echo 1..601 >> all.test # Test plan.
 331 do_check --fail <<END
 332 $failure_header
 333 ${brg}# TOTAL: 601${std}
 334 ${grn}# PASS:  200${std}
 335 ${blu}# SKIP:  200${std}
 336 ${lgn}# XFAIL: 200${std}
 337 ${red}# FAIL:  1${std}
 338 # XPASS: 0
 339 # ERROR: 0
 340 $failure_footer
 341 END
 342
 343 # 1 error and lots of non-failures means failure.
 344 (cat tap && sed 30q tap && echo 'Bail out!') > all.test
 345 test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
 346 echo 1..331 >> all.test # Test plan.
 347 do_check --fail <<END
 348 $failure_header
 349 ${brg}# TOTAL: 331${std}
 350 ${grn}# PASS:  110${std}
 351 ${blu}# SKIP:  110${std}
 352 ${lgn}# XFAIL: 110${std}
 353 # FAIL:  0
 354 # XPASS: 0
 355 ${mgn}# ERROR: 1${std}
 356 $failure_footer
 357 END
 358
 359 :