contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 # exits with 0 if there is nothing of interest
   6 # exits with 1 if there is something interesting
   7 # exits with 2 if an error occurred
   8
   9 # Give two .sum files to compare them
  10
  11 # Written by Mike Stump <mrs@cygnus.com>
  12
  13 tool=gxx
  14
  15 tmp1=/tmp/$tool-testing.$$a
  16 tmp2=/tmp/$tool-testing.$$b
  17 now_s=/tmp/$tool-testing.$$d
  18 before_s=/tmp/$tool-testing.$$e
  19
  20 if [ "$2" = "" ]; then
  21         echo "Usage: $0 previous current" >&2
  22         exit 2
  23 fi
  24
  25 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
  26 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp2
  27
  28 before=$tmp1
  29 now=$tmp2
  30
  31 exit_status=0
  32 trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
  33
  34 if sort -k 2 </dev/null >/dev/null 2>&1; then
  35   skip1='-k 2'
  36 else
  37   skip1='+1'
  38 fi
  39
  40 sort -t ':' $skip1 "$now" > "$now_s"
  41 sort -t ':' $skip1 "$before" > "$before_s"
  42
  43 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
  44 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
  45
  46 grep -s . $tmp2 >/dev/null
  47 if [ $? = 0 ]; then
  48         echo "Tests that now fail, but worked before:"
  49         echo
  50         cat $tmp2
  51         echo
  52         exit_status=1
  53 fi
  54
  55 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  56 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
  57
  58 grep -s . $tmp2 >/dev/null
  59 if [ $? = 0 ]; then
  60         echo "Tests that now work, but didn't before:"
  61         echo
  62         cat $tmp2
  63         echo
  64 fi
  65
  66 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  67 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
  68
  69 grep -s . $tmp2 >/dev/null
  70 if [ $? = 0 ]; then
  71         echo "New tests that FAIL:"
  72         echo
  73         cat $tmp2
  74         echo
  75         exit_status=1
  76 fi
  77
  78 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  79 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
  80
  81 grep -s . $tmp2 >/dev/null
  82 if [ $? = 0 ]; then
  83         echo "New tests that PASS:"
  84         echo
  85         cat $tmp2
  86         echo
  87 fi
  88
  89 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
  90 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
  91
  92 grep -s . $tmp2 >/dev/null
  93 if [ $? = 0 ]; then
  94         echo "Old tests that passed, that have disappeared: (Eeek!)"
  95         echo
  96         cat $tmp2
  97         echo
  98 fi
  99
 100 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 101 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 102
 103 grep -s . $tmp2 >/dev/null
 104 if [ $? = 0 ]; then
 105         echo "Old tests that failed, that have disappeared: (Eeek!)"
 106         echo
 107         cat $tmp2
 108         echo
 109 fi
 110
 111 exit $exit_status