contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 usage()
   6 {
   7         if [ -n "$1" ] ; then
   8                 echo "$0: Error: $1" >&2
   9                 echo >&2
  10         fi
  11         cat >&2 <<EOUSAGE
  12 Usage: $0 [-strict] PREVIOUS CURRENT
  13
  14 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
  15
  16         If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
  17
  18         Unless -strict is given, these discrepancies are not counted as errors:
  19                 missing/extra .sum files when comparing directories
  20                 tests that failed in PREVIOUS but pass in CURRENT
  21                 tests that were not in PREVIOUS but appear in CURRENT
  22                 tests in PREVIOUS that are missing in CURRENT
  23
  24         Exit with the following values:
  25                 0 if there is nothing of interest
  26                 1 if there are errors when comparing single test case files
  27                 N for the number of errors found when comparing directories
  28 EOUSAGE
  29         exit 2
  30 }
  31
  32 # Written by Mike Stump <mrs@cygnus.com>
  33 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
  34
  35 tool=gxx
  36
  37 tmp1=/tmp/$tool-testing.$$a
  38 tmp2=/tmp/$tool-testing.$$b
  39 now_s=/tmp/$tool-testing.$$d
  40 before_s=/tmp/$tool-testing.$$e
  41 lst1=/tmp/$tool-lst1.$$
  42 lst2=/tmp/$tool-lst2.$$
  43 lst3=/tmp/$tool-lst3.$$
  44 lst4=/tmp/$tool-lst4.$$
  45 lst5=/tmp/$tool-lst5.$$
  46 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
  47
  48 [ "$1" = "-strict" ] && strict=$1 && shift
  49 [ "$1" = "-?" ] && usage
  50 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
  51
  52 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
  53 exit_status=0
  54
  55 if [ -d "$1" -a -d "$2" ] ; then
  56         find "$1" -name '*.sum' >$lst1
  57         find "$2" -name '*.sum' >$lst2
  58         echo "# Comparing directories"
  59         echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
  60         echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
  61         echo
  62         # remove leading directory components to compare
  63         sed -e "s|^$1/||" $lst1 | sort >$lst3
  64         sed -e "s|^$2/||" $lst2 | sort >$lst4
  65         comm -23 $lst3 $lst4 >$lst5
  66         if [ -s $lst5 ] ; then
  67                 echo "# Extra sum files in Dir1=$1"
  68                 sed -e "s|^|< $1/|" $lst5
  69                 echo
  70                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  71         fi
  72         comm -13 $lst3 $lst4 >$lst5
  73         if [ -s $lst5 ] ; then
  74                 echo "# Extra sum files in Dir2=$2"
  75                 sed -e "s|^|> $2/|" $lst5
  76                 echo
  77                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  78         fi
  79         comm -12 $lst3 $lst4 | sort -u >$lst5
  80         if [ ! -s $lst5 ] ; then
  81                 echo "# No common sum files"
  82                 exit_status=`expr $exit_status + 1`
  83                 exit $exit_status
  84         fi
  85         cmnsums=`cat $lst5 | wc -l`
  86         echo "# Comparing $cmnsums common sum files"
  87         ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
  88         ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
  89         echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
  90         ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
  91         ret=$?
  92         if [ $ret -ne 0 ]; then
  93                 exit_status=`expr $exit_status + 1`
  94                 echo "## Differences found: $fname"
  95         fi
  96         if [ $exit_status -ne 0 ]; then
  97                 echo "# $exit_status differences in $cmnsums common sum files found"
  98         else
  99                 echo "# No differences found in $cmnsums common sum files"
 100         fi
 101         exit $exit_status
 102 elif [ -d "$1" -o -d "$2" ] ; then
 103         usage "Must specify either two directories or two files"
 104 fi
 105
 106 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
 107 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp2
 108
 109 before=$tmp1
 110 now=$tmp2
 111
 112
 113 if sort -k 2 </dev/null >/dev/null 2>&1; then
 114   skip1='-k 2'
 115 else
 116   skip1='+1'
 117 fi
 118
 119 sort -t ':' $skip1 "$now" > "$now_s"
 120 sort -t ':' $skip1 "$before" > "$before_s"
 121
 122 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 123 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 124
 125 grep -s . $tmp2 >/dev/null
 126 if [ $? = 0 ]; then
 127         echo "Tests that now fail, but worked before:"
 128         echo
 129         cat $tmp2
 130         echo
 131         exit_status=1
 132 fi
 133
 134 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 135 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 136
 137 grep -s . $tmp2 >/dev/null
 138 if [ $? = 0 ]; then
 139         echo "Tests that now work, but didn't before:"
 140         echo
 141         cat $tmp2
 142         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 143         echo
 144 fi
 145
 146 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 147 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 148
 149 grep -s . $tmp2 >/dev/null
 150 if [ $? = 0 ]; then
 151         echo "New tests that FAIL:"
 152         echo
 153         cat $tmp2
 154         echo
 155         exit_status=1
 156 fi
 157
 158 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 159 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 160
 161 grep -s . $tmp2 >/dev/null
 162 if [ $? = 0 ]; then
 163         echo "New tests that PASS:"
 164         echo
 165         cat $tmp2
 166         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 167         echo
 168 fi
 169
 170 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 171 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 172
 173 grep -s . $tmp2 >/dev/null
 174 if [ $? = 0 ]; then
 175         echo "Old tests that passed, that have disappeared: (Eeek!)"
 176         echo
 177         cat $tmp2
 178         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 179         echo
 180 fi
 181
 182 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 183 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 184
 185 grep -s . $tmp2 >/dev/null
 186 if [ $? = 0 ]; then
 187         echo "Old tests that failed, that have disappeared: (Eeek!)"
 188         echo
 189         cat $tmp2
 190         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 191         echo
 192 fi
 193
 194 exit $exit_status