contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 # Written by Mike Stump <mrs@cygnus.com>
   6 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
   7
   8 usage()
   9 {
  10         if [ -n "$1" ] ; then
  11                 echo "$0: Error: $1" >&2
  12                 echo >&2
  13         fi
  14         cat >&2 <<EOUSAGE
  15 Usage: $0 [-strict] PREVIOUS CURRENT
  16
  17 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
  18
  19         If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
  20
  21         Unless -strict is given, these discrepancies are not counted as errors:
  22                 missing/extra .sum files when comparing directories
  23                 tests that failed in PREVIOUS but pass in CURRENT
  24                 tests that were not in PREVIOUS but appear in CURRENT
  25                 tests in PREVIOUS that are missing in CURRENT
  26
  27         Exit with the following values:
  28                 0 if there is nothing of interest
  29                 1 if there are errors when comparing single test case files
  30                 N for the number of errors found when comparing directories
  31 EOUSAGE
  32         exit 2
  33 }
  34
  35 export LC_ALL=C
  36
  37 tool=gxx
  38
  39 TMPDIR=${TMPDIR:-/tmp}
  40 tmp1=$TMPDIR/$tool-testing.$$a
  41 tmp2=$TMPDIR/$tool-testing.$$b
  42 now_s=$TMPDIR/$tool-testing.$$d
  43 before_s=$TMPDIR/$tool-testing.$$e
  44 lst1=$TMPDIR/$tool-lst1.$$
  45 lst2=$TMPDIR/$tool-lst2.$$
  46 lst3=$TMPDIR/$tool-lst3.$$
  47 lst4=$TMPDIR/$tool-lst4.$$
  48 lst5=$TMPDIR/$tool-lst5.$$
  49 sum1=$TMPDIR/$tool-sum1.$$
  50 sum2=$TMPDIR/$tool-sum2.$$
  51 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
  52
  53 [ "$1" = "-strict" ] && strict=$1 && shift
  54 [ "$1" = "-?" ] && usage
  55 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
  56
  57 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
  58 exit_status=0
  59
  60 if [ -d "$1" -a -d "$2" ] ; then
  61         find "$1/" -name '*.sum' >$lst1
  62         find "$2/" -name '*.sum' >$lst2
  63         echo "# Comparing directories"
  64         echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
  65         echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
  66         echo
  67         # remove leading directory components to compare
  68         sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
  69         sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
  70         comm -23 $lst3 $lst4 >$lst5
  71         if [ -s $lst5 ] ; then
  72                 echo "# Extra sum files in Dir1=$1"
  73                 sed -e "s|^|< $1/|" $lst5
  74                 echo
  75                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  76         fi
  77         comm -13 $lst3 $lst4 >$lst5
  78         if [ -s $lst5 ] ; then
  79                 echo "# Extra sum files in Dir2=$2"
  80                 sed -e "s|^|> $2/|" $lst5
  81                 echo
  82                 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
  83         fi
  84         comm -12 $lst3 $lst4 | sort -u >$lst5
  85         if [ ! -s $lst5 ] ; then
  86                 echo "# No common sum files"
  87                 exit_status=`expr $exit_status + 1`
  88                 exit $exit_status
  89         fi
  90         cmnsums=`cat $lst5 | wc -l`
  91         echo "# Comparing $cmnsums common sum files"
  92         ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
  93         ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
  94         echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
  95         ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
  96         ret=$?
  97         if [ $ret -ne 0 ]; then
  98                 exit_status=`expr $exit_status + 1`
  99                 echo "## Differences found: $fname"
 100         fi
 101         if [ $exit_status -ne 0 ]; then
 102                 echo "# $exit_status differences in $cmnsums common sum files found"
 103         else
 104                 echo "# No differences found in $cmnsums common sum files"
 105         fi
 106         exit $exit_status
 107 elif [ -d "$1" -o -d "$2" ] ; then
 108         usage "Must specify either two directories or two files"
 109 fi
 110
 111 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
 112 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
 113
 114 before=$tmp1
 115 now=$tmp2
 116
 117
 118 if sort -k 2 </dev/null >/dev/null 2>&1; then
 119   skip1='-k 2'
 120 else
 121   skip1='+1'
 122 fi
 123
 124 sort -t ':' $skip1 "$now" > "$now_s"
 125 sort -t ':' $skip1 "$before" > "$before_s"
 126
 127 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 128 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 129
 130 grep -s . $tmp2 >/dev/null
 131 if [ $? = 0 ]; then
 132         echo "Tests that now fail, but worked before:"
 133         echo
 134         cat $tmp2
 135         echo
 136         exit_status=1
 137 fi
 138
 139 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 140 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
 141
 142 grep -s . $tmp2 >/dev/null
 143 if [ $? = 0 ]; then
 144         echo "Tests that now work, but didn't before:"
 145         echo
 146         cat $tmp2
 147         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 148         echo
 149 fi
 150
 151 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 152 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 153
 154 grep -s . $tmp2 >/dev/null
 155 if [ $? = 0 ]; then
 156         echo "New tests that FAIL:"
 157         echo
 158         cat $tmp2
 159         echo
 160         exit_status=1
 161 fi
 162
 163 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
 164 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
 165
 166 grep -s . $tmp2 >/dev/null
 167 if [ $? = 0 ]; then
 168         echo "New tests that PASS:"
 169         echo
 170         cat $tmp2
 171         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 172         echo
 173 fi
 174
 175 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 176 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 177
 178 grep -s . $tmp2 >/dev/null
 179 if [ $? = 0 ]; then
 180         echo "Old tests that passed, that have disappeared: (Eeek!)"
 181         echo
 182         cat $tmp2
 183         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 184         echo
 185 fi
 186
 187 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
 188 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
 189
 190 grep -s . $tmp2 >/dev/null
 191 if [ $? = 0 ]; then
 192         echo "Old tests that failed, that have disappeared: (Eeek!)"
 193         echo
 194         cat $tmp2
 195         [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 196         echo
 197 fi
 198
 199 exit $exit_status