2018-05-29 Richard Biener <rguenther@suse.de>
[official-gcc.git] / contrib / patch_tester.sh
blob56bb1ebf9b760faad90af71cbbad8616702955d6
1 #!/bin/sh
3 # Tests a set of patches from a directory.
4 # Copyright (C) 2007, 2008, 2011 Free Software Foundation, Inc.
5 # Contributed by Sebastian Pop <sebastian.pop@amd.com>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 cat <<EOF
23 WARNING: This script should only be fed with patches from known
24 authorized and trusted sources. Don't even think about
25 hooking it up to a raw feed from the gcc-patches list or
26 you'll regret it.
28 EOF
30 args=$@
32 svnpath=svn://gcc.gnu.org/svn/gcc
33 dashj=
34 default_standby=1
35 standby=$default_standby
36 default_watermark=0.60
37 watermark=$default_watermark
38 savecompilers=false
39 nopristinecache=false
40 nogpg=false
41 stop=false
43 usage() {
44 cat <<EOF
45 patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
46 [-svnpath URL] [-stop] [-nopristinecache]
47 <source_dir> [patches_dir [state_dir [build_dir]]]
49 J is the flag passed to make. Default is empty string.
51 STANDBY is the number of minutes between checks for new patches in
52 PATCHES_DIR. Default is ${default_standby} minutes.
54 WATERMARK is the 5 minute average system charge under which a new
55 compile can start. Default is ${default_watermark}.
57 SAVECOMPILERS copies the compilers in the same directory as the
58 test results for the non patched version. Default is not copy.
60 NOPRISTINECACHE prevents use of cached test results from any earlier
61 test runs on the pristine version of the branch and revision under
62 test (the default behaviour). This should be used when testing the
63 same revision and patch with multiple sets of configure options, as
64 these may affect the set of baseline failures.
66 NOGPG can be used to avoid checking the GPG signature of patches.
68 URL is the location of the GCC SVN repository. The default is
69 ${svnpath}.
71 STOP exits when PATCHES_DIR is empty.
73 SOURCE_DIR is the directory containing GCC's toplevel configure.
75 PATCHES_DIR is the directory containing the patches to be tested.
76 Default is SOURCE_DIR/patches.
78 STATE_DIR is where the tester maintains its internal state.
79 Default is SOURCE_DIR/state.
81 BUILD_DIR is the build tree, a temporary directory that this
82 script will delete and recreate. Default is SOURCE_DIR/obj.
84 EOF
85 exit 1
88 makedir () {
89 DIRNAME=$1
90 mkdir -p $DIRNAME
91 if [ $? -ne 0 ]; then
92 echo "ERROR: could not make directory $DIRNAME"
93 exit 1
97 while [ $# -ne 0 ]; do
98 case $1 in
99 -j*)
100 dashj=$1; shift
102 -standby)
103 [[ $# > 2 ]] || usage
104 standby=$2; shift; shift
106 -watermark)
107 [[ $# > 2 ]] || usage
108 watermark=$2; shift; shift
110 -savecompilers)
111 savecompilers=true; shift
113 -nopristinecache)
114 nopristinecache=true; shift
116 -nogpg)
117 nogpg=true; shift
119 -stop)
120 stop=true; shift
122 -svnpath)
123 svnpath=$2; shift; shift
125 -*)
126 echo "Invalid option: $1"
127 usage
130 break
132 esac
133 done
135 test $# -eq 0 && usage
137 SOURCE=$1
138 PATCHES=
139 STATE=
140 BUILD=
142 if [[ $# < 2 ]]; then
143 PATCHES=$SOURCE/patches
144 else
145 PATCHES=$2
147 if [[ $# < 3 ]]; then
148 STATE=$SOURCE/state
149 else
150 STATE=$3
152 if [[ $# < 4 ]]; then
153 BUILD=$SOURCE/obj
154 else
155 BUILD=$4
158 [ -d $PATCHES ] || makedir $PATCHES
159 [ -d $STATE ] || makedir $STATE
160 [ -d $STATE/patched ] || makedir $STATE/patched
161 [ -d $SOURCE ] || makedir $SOURCE
162 [ -f $SOURCE/config.guess ] || {
163 cd $SOURCE
164 svn -q co $svnpath/trunk .
165 if [ $? -ne 0 ]; then
166 echo "ERROR: initial svn checkout failed"
167 exit 1
171 # This can contain required local settings:
172 # default_config configure options, always passed
173 # default_make make bootstrap options, always passed
174 # default_check make check options, always passed
175 [ -f $STATE/defaults ] && . $STATE/defaults
177 VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
179 exec >> $STATE/tester.log 2>&1 || exit 1
180 set -x
182 TESTING=$STATE/testing
183 REPORT=$TESTING/report
184 PRISTINE=$TESTING/pristine
185 PATCHED=$TESTING/patched
186 PATCH=
187 TARGET=`$SOURCE/config.guess || exit 1`
188 TESTLOGS="gcc/testsuite/gcc/gcc.sum
189 gcc/testsuite/gfortran/gfortran.sum
190 gcc/testsuite/g++/g++.sum
191 gcc/testsuite/objc/objc.sum
192 $TARGET/libstdc++-v3/testsuite/libstdc++.sum
193 $TARGET/libffi/testsuite/libffi.sum
194 $TARGET/libgomp/testsuite/libgomp.sum
195 $TARGET/libmudflap/testsuite/libmudflap.sum"
196 COMPILERS="gcc/cc1
197 gcc/cc1obj
198 gcc/cc1plus
199 gcc/f951
200 gcc/jc1
201 gcc/gnat1
202 gcc/tree1"
204 now () {
205 echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
208 report () {
209 echo "$@" >> $REPORT
212 freport () {
213 if [ -s $1 ]; then
214 report "(cat $1"
215 cat $1 >> $REPORT
216 report "tac)"
220 cleanup () {
221 cd $SOURCE
222 svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
225 selfexec () {
226 exec ${CONFIG_SHELL-/bin/sh} $0 $args
229 update () {
230 svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
231 if [ x$svn_branch = x ]; then
232 svn_branch=trunk
235 svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
236 if [ x$svn_revision = x ]; then
237 svn_revision=HEAD
240 cleanup
241 cd $SOURCE
242 case $svn_branch in
243 trunk)
244 if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
245 report "failed to update svn sources with"
246 report "svn switch -r $svn_revision $svnpath/trunk"
247 freport $TESTING/svn
248 return 1
252 ${svnpath}*)
253 if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
254 report "failed to update svn sources with"
255 report "svn switch -r $svn_revision $svn_branch"
256 freport $TESTING/svn
257 return 1
262 if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
263 report "failed to update svn sources with"
264 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
265 freport $TESTING/svn
266 return 1
269 esac
270 contrib/gcc_update --touch
272 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
273 if [[ $VERSION < $current_version ]]; then
274 if [ -f $SOURCE/contrib/patch_tester.sh ]; then
275 selfexec
279 return 0
282 apply_patch () {
283 if [ $nogpg = false ]; then
284 if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
285 report "your patch failed to verify:"
286 freport $TESTING/gpgverify
287 return 1
291 cd $SOURCE
292 if ! patch -p0 < $PATCH &> $TESTING/patching ; then
293 report "your patch failed to apply:"
294 report "(check that the patch was created at the top level)"
295 freport $TESTING/patching
296 return 1
299 # Just assume indexes for now -- not really great, but svn always
300 # makes them.
301 grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
302 # If the patch resulted in an empty file, delete it.
303 # This is how svn reports deletions.
304 if [ ! -s $file ]; then
305 rm -f $file
306 report "Deleting empty file $file"
308 done
311 save_compilers () {
312 for COMPILER in $COMPILERS ; do
313 if [ -f $BUILD/$COMPILER ]; then
314 cp $BUILD/$COMPILER $PRISTINE
316 done
319 bootntest () {
320 rm -rf $BUILD
321 mkdir $BUILD
322 cd $BUILD
324 CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
325 CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
326 if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
327 report "configure with `basename $1` version failed with:"
328 freport $1/configure
329 return 1
332 MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
333 MAKE_ARGS="$default_make $MAKE_ARGS"
334 if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
335 report "bootstrap with `basename $1` version failed with last lines:"
336 tail -30 $1/bootstrap > $1/last_bootstrap
337 freport $1/last_bootstrap
338 report "grep --context=20 Error bootstrap:"
339 grep --context=20 Error $1/bootstrap > $1/bootstrap_error
340 freport $1/bootstrap_error
341 return 1
344 CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
345 CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
346 eval make $dashj $CHECK_OPTIONS -k check &> $1/check
348 SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
349 if [ x$SUITESRUN = x ]; then
350 report "check with `basename $1` version failed, no testsuites were run"
351 return 1
354 for LOG in $TESTLOGS ; do
355 if [ -f $BUILD/$LOG ]; then
356 mv $BUILD/$LOG $1
357 mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
359 done
361 return 0
364 bootntest_patched () {
365 cleanup
366 mkdir -p $PATCHED
367 apply_patch && bootntest $PATCHED
368 return $?
371 # Build the pristine tree with exactly the same options as the patch under test.
372 bootntest_pristine () {
373 cleanup
374 current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
375 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
376 PRISTINE=$STATE/$current_branch/$current_version
378 if [ $nopristinecache = true ]; then
379 rm -rf $PRISTINE
381 if [ -d $PRISTINE ]; then
382 ln -s $PRISTINE $TESTING/pristine
383 return 0
384 else
385 mkdir -p $PRISTINE
386 ln -s $PRISTINE $TESTING/pristine
387 bootntest $PRISTINE
388 RETVAL=$?
389 if [ $RETVAL = 0 -a $savecompilers = true ]; then
390 save_compilers
392 return $RETVAL
396 regtest () {
397 touch $1/report
398 touch $1/passes
399 touch $1/failed
400 touch $1/regress
402 for LOG in $TESTLOGS ; do
403 NLOG=`basename $LOG`
404 if [ -f $1/$NLOG ]; then
405 awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
407 done | sort | uniq > $1/failed
409 comm -12 $1/failed $1/passes >> $1/regress
410 NUMREGRESS=`wc -l < $1/regress | tr -d ' '`
412 if [ $NUMREGRESS -eq 0 ] ; then
413 for LOG in $TESTLOGS ; do
414 NLOG=`basename $LOG`
415 if [ -f $1/$NLOG ] ; then
416 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
418 done | sort | uniq | comm -23 - $1/failed > $1/passes
419 echo "there are no regressions with your patch." >> $1/report
420 else
421 echo "with your patch there are $NUMREGRESS regressions." >> $1/report
422 echo "list of regressions with your patch:" >> $1/report
423 cat $1/regress >> $1/report
427 contrib_compare_tests () {
428 report "comparing logs with contrib/compare_tests:"
429 for LOG in $TESTLOGS ; do
430 NLOG=`basename $LOG`
431 if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
432 $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
433 freport $TESTING/compare_$NLOG
435 done
438 compare_passes () {
439 regtest $PRISTINE
440 cp $PRISTINE/passes $PATCHED
441 regtest $PATCHED
442 freport $PATCHED/report
443 report "FAILs with patched version:"
444 freport $PATCHED/failed
445 report "FAILs with pristine version:"
446 freport $PRISTINE/failed
448 # contrib_compare_tests
451 write_report () {
452 backup_patched=$STATE/patched/`now`
453 report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."
455 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
456 if [ x$EMAIL != x ]; then
457 mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
460 mv $TESTING $backup_patched
463 announce () {
464 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
465 if [ x$EMAIL != x ]; then
467 START_REPORT=$TESTING/start_report
468 echo "Hi, " >> $START_REPORT
469 echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
470 echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
471 echo "Bye, your automatic tester." >> $START_REPORT
472 mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
476 # After selfexec, $TESTING is already set up.
477 if [ -d $TESTING ]; then
478 # The only file in $TESTING is the patch.
479 PATCH=`ls -rt -1 $TESTING | head -1`
480 PATCH=$TESTING/$PATCH
481 if [ -f $PATCH ]; then
482 bootntest_patched && bootntest_pristine && compare_passes
483 write_report
487 firstpatch=true
488 while true; do
489 PATCH=`ls -rt -1 $PATCHES | head -1`
490 if [ x$PATCH = x ]; then
491 if [ $stop = true ]; then
492 if [ $firstpatch = true ]; then
493 echo "No patches ready to test, quitting."
494 exit 1
495 else
496 echo "No more patches to test."
497 exit 0
500 sleep ${standby}m
501 else
502 firstpatch=false
503 sysload=`uptime | cut -d, -f 5`
504 if [[ $sysload > $watermark ]]; then
505 # Wait a bit when system load is too high.
506 sleep ${standby}m
507 else
508 mkdir -p $TESTING
509 mv $PATCHES/$PATCH $TESTING/
510 PATCH=$TESTING/$PATCH
512 announce
513 update && bootntest_patched && bootntest_pristine && compare_passes
514 write_report
517 done