t/tap-plan-errors.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2017 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # TAP support: the following situations should be flagged as errors:
  18 #  - unmatched test plan (too few tests run)
  19 #  - multiple test plans
  20 #  - missing test plan
  21 #  - misplaced test plan (tests run after a late plan)
  22 # Checks about unplanned tests are performed in 'tap-unplanned.sh'
  23 # and 'tap-skip-whole-badcount.sh'.  More checks about corner-cases
  24 # in TAP plans are performed in 'tap-plan-corner.sh'.
  25
  26 . test-init.sh
  27
  28 . tap-setup.sh
  29
  30 my_check ()
  31 {
  32   cat > all.test
  33   test -n "$err" || fatal_ "\$err not set before calling my_check"
  34   cat all.test # For debugging.
  35   run_make -O -e FAIL check
  36   count_test_results "$@"
  37   grep "^ERROR: all\\.test $err$" stdout
  38   unset err
  39 }
  40
  41 err='- too few tests run (expected 2, got 1)'
  42 my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  43 1..2
  44 ok 1
  45 END
  46
  47 err='- too few tests run (expected 12, got 3)'
  48 my_check total=4 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=1 <<END
  49 ok 1
  50 ok 2
  51 not ok 3 # TODO
  52 1..12
  53 END
  54
  55 err='- too few tests run (expected 1, got 0)'
  56 my_check total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  57 1..1
  58 END
  59
  60 err='2 # AFTER LATE PLAN'
  61 my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  62 ok 1
  63 1..2
  64 ok 2
  65 END
  66
  67 err='5 # AFTER LATE PLAN'
  68 my_check total=5 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  69 ok 1
  70 ok 2
  71 ok 3
  72 ok 4
  73 1..5
  74 ok 5
  75 END
  76
  77 err='- missing test plan'
  78 my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  79 ok 1
  80 END
  81
  82
  83 # The two test plans here are deliberately equal.
  84 err='- multiple test plans'
  85 my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  86 1..2
  87 ok 1
  88 ok 2
  89 1..2
  90 END
  91
  92 # The second plan is diagnosed as extra, and only the first one is
  93 # relevant w.r.t. the number of the expected test.
  94 err='- multiple test plans'
  95 my_check total=4 pass=3 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
  96 1..3
  97 ok 1
  98 ok 2
  99 1..2
 100 ok 3
 101 END
 102
 103 # As above, in a slightly different situation.
 104 err='- multiple test plans'
 105 my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
 106 1..2
 107 ok 1
 108 ok 2
 109 1..4
 110 END
 111
 112 :