t/testsuite-summary-count-many.sh

   1 #! /bin/sh
   2 # Copyright (C) 2011-2024 Free Software Foundation, Inc.
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2, or (at your option)
   7 # any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 # Check test counts in the testsuite summary, with test drivers allowing
  18 # multiple test results per test script, and for a huge number of tests.
  19 # Incidentally, this test also checks that the testsuite summary doesn't
  20 # give any bug-report address if it's not defined.
  21
  22 . test-init.sh
  23
  24 for s in trivial-test-driver extract-testsuite-summary.pl; do
  25   cp "$am_testaux_srcdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
  26 done
  27
  28 br='============================================================================'
  29
  30 header="\
  31 ${br}
  32 Testsuite summary for $me 1.0
  33 ${br}"
  34
  35 footer="\
  36 ${br}
  37 See ./test-suite.log for debugging.
  38 ${br}"
  39
  40 echo AC_OUTPUT >> configure.ac
  41
  42 cat > Makefile.am << 'END'
  43 TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
  44 TESTS = all.test
  45 # Without this, the test driver will be horrendously slow.
  46 END
  47
  48 cat > all.test <<'END'
  49 #!/bin/sh
  50 cat results.txt || { echo ERROR: weird; exit 99; }
  51 END
  52 chmod a+x all.test
  53
  54 $PERL -w -e '
  55   use warnings FATAL => "all";
  56   use strict;
  57
  58   my $base = 1000;
  59   my %count = (
  60     TOTAL => $base * 1000,
  61     PASS  => $base * 700,
  62     SKIP  => $base * 200,
  63     XFAIL => $base * 80,
  64     FAIL  => $base * 10,
  65     XPASS => $base * 7,
  66     ERROR => $base * 3,
  67   );
  68   my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;
  69
  70   open (RES, ">results.txt") or die "opening results.txt: $!\n";
  71   open (CNT, ">count.txt") or die "opening count.txt: $!\n";
  72
  73   printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
  74   for my $res (@results)
  75     {
  76       my $uc_res = uc $res;
  77       print STDERR "Generating list of $res ...\n";
  78       for (1..$count{$res})
  79         {
  80           print RES "$uc_res: $_\n";
  81         }
  82       printf CNT "# %-6s %d\n", $res . ":", $count{$res};
  83     }
  84 '
  85
  86 (echo "$header" && cat count.txt && echo "$footer") > summary.exp
  87
  88 $ACLOCAL
  89 $AUTOMAKE -a
  90 $AUTOCONF
  91
  92 ./configure
  93
  94 ($MAKE check || touch make.fail) | tee stdout
  95 test -f make.fail
  96
  97 $PERL extract-testsuite-summary.pl stdout > summary.got
  98 cat summary.exp
  99 cat summary.got
 100 diff summary.exp summary.got || exit 1
 101
 102 :