t/tap-out-of-order.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2017 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # TAP support:
  18 #  - out-of-order test results
  19
  20 . test-init.sh
  21
  22 . tap-setup.sh
  23
  24 cat > a.test <<END
  25 1..3
  26 ok
  27 not ok 2 # TODO
  28 ok 71
  29 END
  30
  31 cat > b.test <<END
  32 1..5
  33 ok 1
  34 ok 2
  35 ok 4 foo # SKIP mu
  36 not ok 2 bar # TODO um
  37 ok 5
  38 END
  39
  40 cat > c.test <<END
  41 1..4
  42 ok 1
  43 ok foo
  44 ok 4 - zardoz
  45 not ok 3
  46 END
  47
  48 # This is documented as an "test out of sequence" error in the
  49 # TAP::Parser documentation.  Keep it in sync with the example
  50 # there.
  51 cat > d.test <<'END'
  52 1..5
  53 ok 1
  54 ok 2
  55 ok 15
  56 ok 16
  57 ok 17
  58 END
  59
  60 run_make -O -e FAIL TESTS='a.test b.test c.test d.test' check
  61 count_test_results total=17 pass=8 fail=0 xpass=0 xfail=1 skip=0 error=8
  62
  63 test $($FGREP -c ': a.test' stdout) -eq 3
  64 test $($FGREP -c ': b.test' stdout) -eq 5
  65 test $($FGREP -c ': c.test' stdout) -eq 4
  66 test $($FGREP -c ': d.test' stdout) -eq 5
  67
  68 grep '^ERROR: a\.test 71 # OUT-OF-ORDER (expecting 3)$' stdout
  69 grep '^ERROR: b\.test 4 foo # OUT-OF-ORDER (expecting 3)$' stdout
  70 grep '^ERROR: b\.test 2 bar # OUT-OF-ORDER (expecting 4)$' stdout
  71 grep '^ERROR: c\.test 4 - zardoz # OUT-OF-ORDER (expecting 3)$' stdout
  72 grep '^ERROR: c\.test 3 # OUT-OF-ORDER (expecting 4)$' stdout
  73 grep '^ERROR: d\.test 15 # OUT-OF-ORDER (expecting 3)$' stdout
  74 grep '^ERROR: d\.test 16 # OUT-OF-ORDER (expecting 4)$' stdout
  75 grep '^ERROR: d\.test 17 # OUT-OF-ORDER (expecting 5)$' stdout
  76
  77 :