/cp
[official-gcc.git] / contrib / compare_tests
blob51a3719ff6a832778f6dafd5944dd172242cbc64
1 #!/bin/sh
2 # This script automatically test the given tool with the tool's test cases,
3 # reporting anything of interest.
5 # Written by Mike Stump <mrs@cygnus.com>
6 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
8 usage()
10 if [ -n "$1" ] ; then
11 echo "$0: Error: $1" >&2
12 echo >&2
14 cat >&2 <<EOUSAGE
15 Usage: $0 [-strict] PREVIOUS CURRENT
17 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
19 If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
21 Unless -strict is given, these discrepancies are not counted as errors:
22 missing/extra .sum files when comparing directories
23 tests that failed in PREVIOUS but pass in CURRENT
24 tests that were not in PREVIOUS but appear in CURRENT
25 tests in PREVIOUS that are missing in CURRENT
27 Exit with the following values:
28 0 if there is nothing of interest
29 1 if there are errors when comparing single test case files
30 N for the number of errors found when comparing directories
31 EOUSAGE
32 exit 2
35 export LC_ALL=C
37 tool=gxx
39 TMPDIR=${TMPDIR:-/tmp}
40 tmp1=$TMPDIR/$tool-testing.$$a
41 tmp2=$TMPDIR/$tool-testing.$$b
42 now_s=$TMPDIR/$tool-testing.$$d
43 before_s=$TMPDIR/$tool-testing.$$e
44 lst1=$TMPDIR/$tool-lst1.$$
45 lst2=$TMPDIR/$tool-lst2.$$
46 lst3=$TMPDIR/$tool-lst3.$$
47 lst4=$TMPDIR/$tool-lst4.$$
48 lst5=$TMPDIR/$tool-lst5.$$
49 sum1=$TMPDIR/$tool-sum1.$$
50 sum2=$TMPDIR/$tool-sum2.$$
51 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
53 [ "$1" = "-strict" ] && strict=$1 && shift
54 [ "$1" = "-?" ] && usage
55 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
57 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
58 exit_status=0
60 if [ -d "$1" -a -d "$2" ] ; then
61 find "$1/" -name '*.sum' >$lst1
62 find "$2/" -name '*.sum' >$lst2
63 echo "# Comparing directories"
64 echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
65 echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
66 echo
67 # remove leading directory components to compare
68 sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
69 sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
70 comm -23 $lst3 $lst4 >$lst5
71 if [ -s $lst5 ] ; then
72 echo "# Extra sum files in Dir1=$1"
73 sed -e "s|^|< $1/|" $lst5
74 echo
75 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
77 comm -13 $lst3 $lst4 >$lst5
78 if [ -s $lst5 ] ; then
79 echo "# Extra sum files in Dir2=$2"
80 sed -e "s|^|> $2/|" $lst5
81 echo
82 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
84 comm -12 $lst3 $lst4 | sort -u >$lst5
85 if [ ! -s $lst5 ] ; then
86 echo "# No common sum files"
87 exit_status=`expr $exit_status + 1`
88 exit $exit_status
90 cmnsums=`cat $lst5 | wc -l`
91 echo "# Comparing $cmnsums common sum files"
92 ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
93 ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
94 echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
95 ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
96 ret=$?
97 if [ $ret -ne 0 ]; then
98 exit_status=`expr $exit_status + 1`
99 echo "## Differences found: $fname"
101 if [ $exit_status -ne 0 ]; then
102 echo "# $exit_status differences in $cmnsums common sum files found"
103 else
104 echo "# No differences found in $cmnsums common sum files"
106 exit $exit_status
107 elif [ -d "$1" -o -d "$2" ] ; then
108 usage "Must specify either two directories or two files"
111 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
112 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
114 before=$tmp1
115 now=$tmp2
118 if sort -k 2 </dev/null >/dev/null 2>&1; then
119 skip1='-k 2'
120 else
121 skip1='+1'
124 sort -t ':' $skip1 "$now" > "$now_s"
125 sort -t ':' $skip1 "$before" > "$before_s"
127 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
128 grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
130 grep -s . $tmp2 >/dev/null
131 if [ $? = 0 ]; then
132 echo "Tests that now fail, but worked before:"
133 echo
134 cat $tmp2
135 echo
136 exit_status=1
139 grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
140 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
142 grep -s . $tmp2 >/dev/null
143 if [ $? = 0 ]; then
144 echo "Tests that now work, but didn't before:"
145 echo
146 cat $tmp2
147 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
148 echo
151 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
152 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
154 grep -s . $tmp2 >/dev/null
155 if [ $? = 0 ]; then
156 echo "New tests that FAIL:"
157 echo
158 cat $tmp2
159 echo
160 exit_status=1
163 grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
164 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
166 grep -s . $tmp2 >/dev/null
167 if [ $? = 0 ]; then
168 echo "New tests that PASS:"
169 echo
170 cat $tmp2
171 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
172 echo
175 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
176 grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
178 grep -s . $tmp2 >/dev/null
179 if [ $? = 0 ]; then
180 echo "Old tests that passed, that have disappeared: (Eeek!)"
181 echo
182 cat $tmp2
183 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
184 echo
187 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
188 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
190 grep -s . $tmp2 >/dev/null
191 if [ $? = 0 ]; then
192 echo "Old tests that failed, that have disappeared: (Eeek!)"
193 echo
194 cat $tmp2
195 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
196 echo
199 exit $exit_status