2016-10-07 Steven G. Kargl <kargl@gcc.gnu.org>
[official-gcc.git] / contrib / compare_tests
blobd16e7e9737e85e50d3b97b8cb12e28e24e489dc0
1 #!/bin/sh
2 # This script automatically test the given tool with the tool's test cases,
3 # reporting anything of interest.
5 # Written by Mike Stump <mrs@cygnus.com>
6 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
8 usage()
10 if [ -n "$1" ] ; then
11 echo "$0: Error: $1" >&2
12 echo >&2
14 cat >&2 <<EOUSAGE
15 Usage: $0 [-strict] PREVIOUS CURRENT
17 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
19 If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
21 Unless -strict is given, these discrepancies are not counted as errors:
22 missing/extra .sum files when comparing directories
23 tests that failed in PREVIOUS but pass in CURRENT
24 tests that were not in PREVIOUS but appear in CURRENT
25 tests in PREVIOUS that are missing in CURRENT
27 Exit with the following values:
28 0 if there is nothing of interest
29 1 if there are errors when comparing single test case files
30 N for the number of errors found when comparing directories
31 EOUSAGE
32 exit 2
35 export LC_ALL=C
37 tool=gxx
39 tmp1=/tmp/$tool-testing.$$a
40 tmp2=/tmp/$tool-testing.$$b
41 now_s=/tmp/$tool-testing.$$d
42 before_s=/tmp/$tool-testing.$$e
43 lst1=/tmp/$tool-lst1.$$
44 lst2=/tmp/$tool-lst2.$$
45 lst3=/tmp/$tool-lst3.$$
46 lst4=/tmp/$tool-lst4.$$
47 lst5=/tmp/$tool-lst5.$$
48 sum1=/tmp/$tool-sum1.$$
49 sum2=/tmp/$tool-sum2.$$
50 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
52 [ "$1" = "-strict" ] && strict=$1 && shift
53 [ "$1" = "-?" ] && usage
54 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
56 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
57 exit_status=0
59 if [ -d "$1" -a -d "$2" ] ; then
60 find "$1/" -name '*.sum' >$lst1
61 find "$2/" -name '*.sum' >$lst2
62 echo "# Comparing directories"
63 echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
64 echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
65 echo
66 # remove leading directory components to compare
67 sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
68 sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
69 comm -23 $lst3 $lst4 >$lst5
70 if [ -s $lst5 ] ; then
71 echo "# Extra sum files in Dir1=$1"
72 sed -e "s|^|< $1/|" $lst5
73 echo
74 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
76 comm -13 $lst3 $lst4 >$lst5
77 if [ -s $lst5 ] ; then
78 echo "# Extra sum files in Dir2=$2"
79 sed -e "s|^|> $2/|" $lst5
80 echo
81 [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
83 comm -12 $lst3 $lst4 | sort -u >$lst5
84 if [ ! -s $lst5 ] ; then
85 echo "# No common sum files"
86 exit_status=`expr $exit_status + 1`
87 exit $exit_status
89 cmnsums=`cat $lst5 | wc -l`
90 echo "# Comparing $cmnsums common sum files"
91 ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
92 ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
93 echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
94 ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
95 ret=$?
96 if [ $ret -ne 0 ]; then
97 exit_status=`expr $exit_status + 1`
98 echo "## Differences found: $fname"
100 if [ $exit_status -ne 0 ]; then
101 echo "# $exit_status differences in $cmnsums common sum files found"
102 else
103 echo "# No differences found in $cmnsums common sum files"
105 exit $exit_status
106 elif [ -d "$1" -o -d "$2" ] ; then
107 usage "Must specify either two directories or two files"
110 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
111 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
113 before=$tmp1
114 now=$tmp2
117 if sort -k 2 </dev/null >/dev/null 2>&1; then
118 skip1='-k 2'
119 else
120 skip1='+1'
123 sort -t ':' $skip1 "$now" > "$now_s"
124 sort -t ':' $skip1 "$before" > "$before_s"
126 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
127 grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
129 grep -s . $tmp2 >/dev/null
130 if [ $? = 0 ]; then
131 echo "Tests that now fail, but worked before:"
132 echo
133 cat $tmp2
134 echo
135 exit_status=1
138 grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
139 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2
141 grep -s . $tmp2 >/dev/null
142 if [ $? = 0 ]; then
143 echo "Tests that now work, but didn't before:"
144 echo
145 cat $tmp2
146 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
147 echo
150 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
151 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
153 grep -s . $tmp2 >/dev/null
154 if [ $? = 0 ]; then
155 echo "New tests that FAIL:"
156 echo
157 cat $tmp2
158 echo
159 exit_status=1
162 grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
163 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2
165 grep -s . $tmp2 >/dev/null
166 if [ $? = 0 ]; then
167 echo "New tests that PASS:"
168 echo
169 cat $tmp2
170 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
171 echo
174 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
175 grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
177 grep -s . $tmp2 >/dev/null
178 if [ $? = 0 ]; then
179 echo "Old tests that passed, that have disappeared: (Eeek!)"
180 echo
181 cat $tmp2
182 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
183 echo
186 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1
187 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2
189 grep -s . $tmp2 >/dev/null
190 if [ $? = 0 ]; then
191 echo "Old tests that failed, that have disappeared: (Eeek!)"
192 echo
193 cat $tmp2
194 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
195 echo
198 exit $exit_status