testsuite/analyse-results.sh

   1 #!/bin/sh
   2
   3 mode=$1
   4
   5 total_fail=0;
   6 total_pass=0;
   7 total_xfail=0;
   8 total_xpass=0;
   9 total_untested=0;
  10 total_unresolved=0;
  11
  12 echo
  13 echo "[Test Results Summary]"
  14 echo
  15
  16 suitefail=
  17 suitexpass=
  18 for fn in `find testsuite extensions -name testrun.sum | sort`; do
  19     dir=`dirname $fn`
  20     nofail=`grep -c "^FAIL: "   ${fn}`
  21     nopass=`grep -c "^PASS: "   ${fn}`
  22     noxfail=`grep -c "^XFAIL: " ${fn}`
  23     noxpass=`grep -c "^XPASS: " ${fn}`
  24     nounresolved=`grep -c "^UNRESOLVED: " ${fn}`
  25     nountested=`grep -c "^UNTESTED: " ${fn}`
  26     echo -n "Test suite $dir: "
  27     someprint=0
  28     if test $nofail -gt 0; then
  29         suitefail="${suitefail} ${dir}"
  30         echo -n "$nofail real failures"
  31         total_fail=`expr $total_fail + $nofail`
  32         someprint=1
  33     fi
  34     if test $noxpass -gt 0; then
  35         suitexpass="${suitexpass} ${dir}"
  36         if test $someprint -gt 0; then echo -n ", "; fi
  37         echo -n "$noxpass unexpected successes"
  38         total_xpass=`expr $total_xpass + $noxpass`
  39         someprint=1
  40     fi
  41     if test $nopass -gt 0; then
  42         if test $someprint -gt 0; then echo -n ", "; fi
  43         echo -n "$nopass passes"
  44         total_pass=`expr $total_pass + $nopass`
  45         someprint=1
  46     fi
  47     if test $noxfail -gt 0; then
  48         if test $someprint -gt 0; then echo -n ", "; fi
  49         echo -n "${noxfail} expected failure"
  50         total_xfail=`expr $total_xfail + $noxfail`
  51         someprint=1
  52     fi
  53     if test ${nounresolved} -gt 0; then
  54         if test $someprint -gt 0; then echo -n ", "; fi
  55         echo -n "${nounresolved} unresolved"
  56         total_unresolved=`expr ${total_unresolved} + ${nounresolved}`
  57         someprint=1
  58     fi
  59     if test ${nountested} -gt 0; then
  60         if test $someprint -gt 0; then echo -n ", "; fi
  61         echo -n "${nountested} untested"
  62         total_untested=`expr ${total_untested} + ${nountested}`
  63         someprint=1
  64     fi
  65     if test ${nofail} -gt 0 || test ${noxpass} -gt 0; then echo -n " *"; fi
  66     echo
  67 done
  68
  69 echo
  70 echo "Test Result Totals:"
  71 if test ${total_pass} -gt 0; then
  72     echo "      Total passes: $total_pass"
  73 fi
  74 if test ${total_unresolved} -gt 0; then
  75     echo "      Total unresolved: $total_unresolved"
  76 fi
  77 if test ${total_xfail} -gt 0; then
  78     echo "      Total expected failures: ${total_xfail}"
  79 fi
  80 if test ${total_untested} -gt 0; then
  81     echo "      Total untested: ${total_untested}"
  82 fi
  83 if test ${total_fail} -gt 0; then
  84     echo "      * Total real failures: $total_fail"
  85 fi
  86 if test ${total_xpass} -gt 0; then
  87     echo "      --> Total unexpected successes: ${total_xpass}"
  88 fi
  89
  90 echo
  91
  92 # For now, return a failure if any XPASS or FAIL occurred
  93 if test ${total_fail} -gt 0 || test ${total_xpass} -gt 0; then
  94
  95         rc=1
  96         timing=$(dirname $0)/timingissues
  97         for t in ${suitefail}; do
  98                 testruns="${testruns} ${t}/testrun.sum"
  99         done
 100         failures="grep -hw FAIL ${testruns}"
 101
 102         if test ${total_fail} -gt 0; then
 103                 echo "Unexpected failures follow:"
 104                 for s in ${suitefail}; do
 105                         echo " --=[ ${s} ]=-- "
 106                         grep -w FAIL ${s}/testrun.sum
 107                 done
 108                 echo
 109         fi
 110
 111         if test ${total_xpass} -gt 0; then
 112                 echo "Unexpected successes follow:"
 113                 for s in ${suitexpass}; do
 114                         echo " --=[ ${s} ]=-- "
 115                         grep -w XPASS ${s}/testrun.sum
 116                 done
 117                 echo
 118         fi
 119
 120         if test `$failures | grep -cf $timing` -gt 0; then
 121                 if test `$failures | grep -cvf $timing` -eq 0; then
 122                         echo "All failures are time-related. Exiting 0."
 123                         echo "See http://wiki.gnashdev.org/PredictableLoading"
 124                         echo
 125                         rc=0
 126                 else
 127                         echo "Time-related failures follow:"
 128                         $failures | grep -f $timing
 129                         echo
 130                         echo "See http://wiki.gnashdev.org/PredictableLoading"
 131                         echo
 132                         echo "Non time-related failures follow:"
 133                         $failures | grep -vf $timing
 134                         echo
 135                 fi
 136         fi
 137
 138         if [ "$mode" = "verbose" ]; then
 139                 if test ${total_fail} -gt 0; then
 140                         echo "Verbose mode enabled. Displaying following files:"
 141                         echo
 142                         swfdecfail=$($failures | sed -n 's/^.*in \(.*.trace-gnash\).*in \(.*.log\).*$/\1\n\2/p')
 143                         for log in $swfdecfail; do
 144                                 echo " testsuite/swfdec/$log"
 145                                 logfiles="${logfiles} testsuite/swfdec/$log"
 146                         done
 147                         for s in ${suitefail}; do
 148                                 echo " ${s}/testrun.log"
 149                                 logfiles="${logfiles} ${s}/testrun.log"
 150                         done
 151                         echo
 152                         for logfile in ${logfiles}; do
 153                                 echo "= = = = = = = [ ${logfile} file - BEGIN ] = = = = = = ="
 154                                 cat ${logfile}
 155                                 echo "= = = = = = = [ ${logfile} file - END   ] = = = = = = ="
 156                                 echo
 157                         done
 158                 fi
 159         fi
 160
 161         exit $rc
 162 else
 163         exit 0
 164 fi
 165
 166 #
 167 # Look for regressions
 168 #