testsuite/analyse-results.sh

   1 #!/bin/sh
   2
   3
   4 # case "$1" in
   5 #     totals)
   6 #       totals ;;
   7 #     *)
   8 #       exit ;;
   9 # esac
  10
  11 total_fail=0;
  12 total_pass=0;
  13 total_xfail=0;
  14 total_xpass=0;
  15 total_untested=0;
  16 total_unresolved=0;
  17
  18 echo
  19 echo "[Test Results Summary]"
  20 echo
  21
  22 suitefail=
  23 suitexpass=
  24 for fn in `find testsuite extensions -name testrun.sum | sort`; do
  25     dir=`dirname $fn`
  26     nofail=`grep -c "^FAIL: "   ${fn}`
  27     nopass=`grep -c "^PASS: "   ${fn}`
  28     noxfail=`grep -c "^XFAIL: " ${fn}`
  29     noxpass=`grep -c "^XPASS: " ${fn}`
  30     nounresolved=`grep -c "^UNRESOLVED: " ${fn}`
  31     nountested=`grep -c "^UNTESTED: " ${fn}`
  32     echo -n "Test suite $dir: "
  33     someprint=0
  34     if test $nofail -gt 0; then
  35         suitefail="${suitefail} ${dir}"
  36         echo -n "$nofail real failures"
  37         total_fail=`expr $total_fail + $nofail`
  38         someprint=1
  39     fi
  40     if test $noxpass -gt 0; then
  41         suitexpass="${suitexpass} ${dir}"
  42         if test $someprint -gt 0; then echo -n ", "; fi
  43         echo -n "$noxpass unexpected successes"
  44         total_xpass=`expr $total_xpass + $noxpass`
  45         someprint=1
  46     fi
  47     if test $nopass -gt 0; then
  48         if test $someprint -gt 0; then echo -n ", "; fi
  49         echo -n "$nopass passes"
  50         total_pass=`expr $total_pass + $nopass`
  51         someprint=1
  52     fi
  53     if test $noxfail -gt 0; then
  54         if test $someprint -gt 0; then echo -n ", "; fi
  55         echo -n "${noxfail} expected failure"
  56         total_xfail=`expr $total_xfail + $noxfail`
  57         someprint=1
  58     fi
  59     if test ${nounresolved} -gt 0; then
  60         if test $someprint -gt 0; then echo -n ", "; fi
  61         echo -n "${nounresolved} unresolved"
  62         total_unresolved=`expr ${total_unresolved} + ${nounresolved}`
  63         someprint=1
  64     fi
  65     if test ${nountested} -gt 0; then
  66         if test $someprint -gt 0; then echo -n ", "; fi
  67         echo -n "${nountested} untested"
  68         total_untested=`expr ${total_untested} + ${nountested}`
  69         someprint=1
  70     fi
  71     if test ${nofail} -gt 0 || test ${noxpass} -gt 0; then echo -n " *"; fi
  72     echo
  73 done
  74
  75 echo
  76 echo "Test Result Totals:"
  77 if test ${total_pass} -gt 0; then
  78     echo "      Total passes: $total_pass"
  79 fi
  80 if test ${total_unresolved} -gt 0; then
  81     echo "      Total unresolved: $total_unresolved"
  82 fi
  83 if test ${total_xfail} -gt 0; then
  84     echo "      Total expected failures: ${total_xfail}"
  85 fi
  86 if test ${total_untested} -gt 0; then
  87     echo "      Total untested: ${total_untested}"
  88 fi
  89 if test ${total_fail} -gt 0; then
  90     echo "      * Total real failures: $total_fail"
  91 fi
  92 if test ${total_xpass} -gt 0; then
  93     echo "      --> Total unexpected successes: ${total_xpass}"
  94 fi
  95
  96 echo
  97
  98 # For now, return a failure if any XPASS or FAIL occurred
  99 if test ${total_fail} -gt 0 || test ${total_xpass} -gt 0; then
 100
 101         if test ${total_fail} -gt 0; then
 102                 echo "Unexpected failures follow:"
 103                 for s in ${suitefail}; do
 104                         echo " --=[ ${s} ]=-- "
 105                         grep -w FAIL ${s}/testrun.sum;
 106                 done
 107                 echo
 108         fi
 109
 110         if test ${total_xpass} -gt 0; then
 111                 echo "Unexpected successes follow:"
 112                 for s in ${suitexpass}; do
 113                         echo "--=[ ${s} ]=--- "
 114                         grep -w XPASS ${s}/testrun.sum; done
 115                 echo
 116         fi
 117
 118         exit 1
 119 else
 120         exit 0
 121 fi
 122
 123 #
 124 # Look for regressions
 125 #