contrib/patch_tester.sh

   1 #!/bin/sh
   2
   3 # Tests a set of patches from a directory.
   4 # Copyright (C) 2007, 2008  Free Software Foundation, Inc.
   5 # Contributed by Sebastian Pop <sebastian.pop@amd.com>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 cat <<EOF
  22
  23 WARNING: This script should only be fed with patches from known
  24          authorized and trusted sources.  Don't even think about
  25          hooking it up to a raw feed from the gcc-patches list or
  26          you'll regret it.
  27
  28 EOF
  29
  30 args=$@
  31
  32 svnpath=svn://gcc.gnu.org/svn/gcc
  33 dashj=
  34 default_standby=1
  35 standby=$default_standby
  36 default_watermark=0.60
  37 watermark=$default_watermark
  38 savecompilers=false
  39 nogpg=false
  40 stop=false
  41
  42 usage() {
  43     cat <<EOF
  44 patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
  45                 [-svnpath URL] [-stop]
  46                 <source_dir> [patches_dir [state_dir [build_dir]]]
  47
  48     J is the flag passed to make.  Default is empty string.
  49
  50     STANDBY is the number of minutes between checks for new patches in
  51     PATCHES_DIR.  Default is ${default_standby} minutes.
  52
  53     WATERMARK is the 5 minute average system charge under which a new
  54     compile can start.  Default is ${default_watermark}.
  55
  56     SAVECOMPILERS copies the compilers in the same directory as the
  57     test results for the non patched version.  Default is not copy.
  58
  59     NOGPG can be used to avoid checking the GPG signature of patches.
  60
  61     URL is the location of the GCC SVN repository.  The default is
  62     ${svnpath}.
  63
  64     STOP exits when PATCHES_DIR is empty.
  65
  66     SOURCE_DIR is the directory containing GCC's toplevel configure.
  67
  68     PATCHES_DIR is the directory containing the patches to be tested.
  69     Default is SOURCE_DIR/patches.
  70
  71     STATE_DIR is where the tester maintains its internal state.
  72     Default is SOURCE_DIR/state.
  73
  74     BUILD_DIR is the build tree, a temporary directory that this
  75     script will delete and recreate.  Default is SOURCE_DIR/obj.
  76
  77 EOF
  78     exit 1
  79 }
  80
  81 makedir () {
  82     DIRNAME=$1
  83     mkdir -p $DIRNAME
  84     if [ $? -ne 0 ]; then
  85         echo "ERROR: could not make directory $DIRNAME"
  86         exit 1
  87     fi
  88 }
  89
  90 while [ $# -ne 0 ]; do
  91     case $1 in
  92         -j*)
  93             dashj=$1; shift
  94             ;;
  95         -standby)
  96             [[ $# > 2 ]] || usage
  97             standby=$2; shift; shift
  98             ;;
  99         -watermark)
 100             [[ $# > 2 ]] || usage
 101             watermark=$2; shift; shift
 102             ;;
 103         -savecompilers)
 104             savecompilers=true; shift
 105             ;;
 106         -nogpg)
 107             nogpg=true; shift
 108             ;;
 109         -stop)
 110             stop=true; shift
 111             ;;
 112         -svnpath)
 113             svnpath=$2; shift; shift
 114             ;;
 115         -*)
 116             echo "Invalid option: $1"
 117             usage
 118             ;;
 119         *)
 120             break
 121             ;;
 122     esac
 123 done
 124
 125 test $# -eq 0 && usage
 126
 127 SOURCE=$1
 128 PATCHES=
 129 STATE=
 130 BUILD=
 131
 132 if [[ $# < 2 ]]; then
 133     PATCHES=$SOURCE/patches
 134 else
 135     PATCHES=$2
 136 fi
 137 if [[ $# < 3 ]]; then
 138     STATE=$SOURCE/state
 139 else
 140     STATE=$3
 141 fi
 142 if [[ $# < 4 ]]; then
 143     BUILD=$SOURCE/obj
 144 else
 145     BUILD=$4
 146 fi
 147
 148 [ -d $PATCHES ] || makedir $PATCHES
 149 [ -d $STATE ] || makedir $STATE
 150 [ -d $STATE/patched ] || makedir $STATE/patched
 151 [ -d $SOURCE ] || makedir $SOURCE
 152 [ -f $SOURCE/config.guess ] || {
 153     cd $SOURCE
 154     svn -q co $svnpath/trunk .
 155     if [ $? -ne 0 ]; then
 156         echo "ERROR: initial svn checkout failed"
 157         exit 1
 158     fi
 159 }
 160
 161 # This can contain required local settings:
 162 #  default_config  configure options, always passed
 163 #  default_make    make bootstrap options, always passed
 164 #  default_check   make check options, always passed
 165 [ -f $STATE/defaults ] && . $STATE/defaults
 166
 167 VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 168
 169 exec >> $STATE/tester.log 2>&1 || exit 1
 170 set -x
 171
 172 TESTING=$STATE/testing
 173 REPORT=$TESTING/report
 174 PRISTINE=$TESTING/pristine
 175 PATCHED=$TESTING/patched
 176 PATCH=
 177 TARGET=`$SOURCE/config.guess || exit 1`
 178 TESTLOGS="gcc/testsuite/gcc/gcc.sum
 179 gcc/testsuite/gfortran/gfortran.sum
 180 gcc/testsuite/g++/g++.sum
 181 gcc/testsuite/objc/objc.sum
 182 $TARGET/libstdc++-v3/testsuite/libstdc++.sum
 183 $TARGET/libffi/testsuite/libffi.sum
 184 $TARGET/libjava/testsuite/libjava.sum
 185 $TARGET/libgomp/testsuite/libgomp.sum
 186 $TARGET/libmudflap/testsuite/libmudflap.sum"
 187 COMPILERS="gcc/cc1
 188 gcc/cc1obj
 189 gcc/cc1plus
 190 gcc/f951
 191 gcc/jc1
 192 gcc/gnat1
 193 gcc/tree1"
 194
 195 now () {
 196     echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
 197 }
 198
 199 report () {
 200     echo "$@" >> $REPORT
 201 }
 202
 203 freport () {
 204     if [ -s $1 ]; then
 205         report "(cat $1"
 206         cat $1 >> $REPORT
 207         report "tac)"
 208     fi
 209 }
 210
 211 cleanup () {
 212     cd $SOURCE
 213     svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
 214 }
 215
 216 selfexec () {
 217     exec ${CONFIG_SHELL-/bin/sh} $0 $args
 218 }
 219
 220 update () {
 221     svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
 222     if [ x$svn_branch = x ]; then
 223         svn_branch=trunk
 224     fi
 225
 226     svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
 227     if [ x$svn_revision = x ]; then
 228         svn_revision=HEAD
 229     fi
 230
 231     cleanup
 232     cd $SOURCE
 233     case $svn_branch in
 234         trunk)
 235             if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
 236                 report "failed to update svn sources with"
 237                 report "svn switch -r $svn_revision $svnpath/trunk"
 238                 freport $TESTING/svn
 239                 return 1
 240             fi
 241             ;;
 242
 243         ${svnpath}*)
 244             if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
 245                 report "failed to update svn sources with"
 246                 report "svn switch -r $svn_revision $svn_branch"
 247                 freport $TESTING/svn
 248                 return 1
 249             fi
 250             ;;
 251
 252         *)
 253             if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
 254                 report "failed to update svn sources with"
 255                 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
 256                 freport $TESTING/svn
 257                 return 1
 258             fi
 259             ;;
 260     esac
 261     contrib/gcc_update --touch
 262
 263     current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 264     if [[ $VERSION < $current_version ]]; then
 265         if [ -f $SOURCE/contrib/patch_tester.sh ]; then
 266             selfexec
 267         fi
 268     fi
 269
 270     return 0
 271 }
 272
 273 apply_patch () {
 274     if [ $nogpg = false ]; then
 275         if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
 276             report "your patch failed to verify:"
 277             freport $TESTING/gpgverify
 278             return 1
 279         fi
 280     fi
 281
 282     cd $SOURCE
 283     if ! patch -p0 < $PATCH &> $TESTING/patching ; then
 284         report "your patch failed to apply:"
 285         report "(check that the patch was created at the top level)"
 286         freport $TESTING/patching
 287         return 1
 288     fi
 289
 290     # Just assume indexes for now -- not really great, but svn always
 291     # makes them.
 292     grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
 293         # If the patch resulted in an empty file, delete it.
 294         # This is how svn reports deletions.
 295         if [ ! -s $file ]; then
 296             rm -f $file
 297             report "Deleting empty file $file"
 298         fi
 299     done
 300 }
 301
 302 save_compilers () {
 303     for COMPILER in $COMPILERS ; do
 304         if [ -f $BUILD/$COMPILER ]; then
 305             cp $BUILD/$COMPILER $PRISTINE
 306         fi
 307     done
 308 }
 309
 310 bootntest () {
 311     rm -rf $BUILD
 312     mkdir $BUILD
 313     cd $BUILD
 314
 315     CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
 316     CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
 317     if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
 318         report "configure with `basename $1` version failed with:"
 319         freport $1/configure
 320         return 1
 321     fi
 322
 323     MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
 324     MAKE_ARGS="$default_make $MAKE_ARGS"
 325     if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
 326         report "bootstrap with `basename $1` version failed with last lines:"
 327         tail -30 $1/bootstrap > $1/last_bootstrap
 328         freport $1/last_bootstrap
 329         report "grep --context=20 Error bootstrap:"
 330         grep --context=20 Error $1/bootstrap > $1/bootstrap_error
 331         freport $1/bootstrap_error
 332         return 1
 333     fi
 334
 335     CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
 336     CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
 337     eval make $dashj $CHECK_OPTIONS -k check &> $1/check
 338
 339     SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
 340     if [ x$SUITESRUN = x ]; then
 341         report "check with `basename $1` version failed, no testsuites were run"
 342         return 1
 343     fi
 344
 345     for LOG in $TESTLOGS ; do
 346         if [ -f $BUILD/$LOG ]; then
 347             mv $BUILD/$LOG $1
 348             mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
 349         fi
 350     done
 351
 352     return 0
 353 }
 354
 355 bootntest_patched () {
 356     cleanup
 357     mkdir -p $PATCHED
 358     apply_patch && bootntest $PATCHED
 359     return $?
 360 }
 361
 362 # Build the pristine tree with exactly the same options as the patch under test.
 363 bootntest_pristine () {
 364     cleanup
 365     current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
 366     current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 367     PRISTINE=$STATE/$current_branch/$current_version
 368
 369     if [ -d $PRISTINE ]; then
 370         ln -s $PRISTINE $TESTING/pristine
 371         return 0
 372     else
 373         mkdir -p $PRISTINE
 374         ln -s $PRISTINE $TESTING/pristine
 375         bootntest $PRISTINE
 376         RETVAL=$?
 377         if [ $RETVAL = 0 -a $savecompilers = true ]; then
 378             save_compilers
 379         fi
 380         return $RETVAL
 381     fi
 382 }
 383
 384 regtest () {
 385     touch $1/report
 386     touch $1/passes
 387     touch $1/failed
 388     touch $1/regress
 389
 390     for LOG in $TESTLOGS ; do
 391         NLOG=`basename $LOG`
 392         if [ -f $1/$NLOG ]; then
 393             awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
 394         fi
 395     done | sort | uniq > $1/failed
 396
 397     comm -12 $1/failed $1/passes >> $1/regress
 398     NUMREGRESS=`wc -l < $1/regress | tr -d ' '`
 399
 400     if [ $NUMREGRESS -eq 0 ] ; then
 401         for LOG in $TESTLOGS ; do
 402             NLOG=`basename $LOG`
 403             if [ -f $1/$NLOG ] ; then
 404                 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
 405             fi
 406         done | sort | uniq | comm -23 - $1/failed > $1/passes
 407         echo "there are no regressions with your patch." >> $1/report
 408     else
 409         echo "with your patch there are $NUMREGRESS regressions." >> $1/report
 410         echo "list of regressions with your patch:" >> $1/report
 411         cat $1/regress >> $1/report
 412     fi
 413 }
 414
 415 contrib_compare_tests () {
 416     report "comparing logs with contrib/compare_tests:"
 417     for LOG in $TESTLOGS ; do
 418         NLOG=`basename $LOG`
 419         if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
 420             $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
 421             freport $TESTING/compare_$NLOG
 422         fi
 423     done
 424 }
 425
 426 compare_passes () {
 427     regtest $PRISTINE
 428     cp $PRISTINE/passes $PATCHED
 429     regtest $PATCHED
 430     freport $PATCHED/report
 431     report "FAILs with patched version:"
 432     freport $PATCHED/failed
 433     report "FAILs with pristine version:"
 434     freport $PRISTINE/failed
 435
 436     # contrib_compare_tests
 437 }
 438
 439 write_report () {
 440     backup_patched=$STATE/patched/`now`
 441     report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."
 442
 443     EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
 444     if [ x$EMAIL != x ]; then
 445         mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
 446     fi
 447
 448     mv $TESTING $backup_patched
 449 }
 450
 451 announce () {
 452     EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
 453     if [ x$EMAIL != x ]; then
 454
 455         START_REPORT=$TESTING/start_report
 456         echo "Hi, " >> $START_REPORT
 457         echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
 458         echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
 459         echo "Bye, your automatic tester." >> $START_REPORT
 460         mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
 461     fi
 462 }
 463
 464 # After selfexec, $TESTING is already set up.
 465 if [ -d $TESTING ]; then
 466     # The only file in $TESTING is the patch.
 467     PATCH=`ls -rt -1 $TESTING | head -1`
 468     PATCH=$TESTING/$PATCH
 469     if [ -f $PATCH ]; then
 470         bootntest_patched && bootntest_pristine && compare_passes
 471         write_report
 472     fi
 473 fi
 474
 475 firstpatch=true
 476 while true; do
 477     PATCH=`ls -rt -1 $PATCHES | head -1`
 478     if [ x$PATCH = x ]; then
 479         if [ $stop = true ]; then
 480             if [ $firstpatch = true ]; then
 481                 echo "No patches ready to test, quitting."
 482                 exit 1
 483             else
 484                 echo "No more patches to test."
 485                 exit 0
 486             fi
 487         fi
 488         sleep ${standby}m
 489     else
 490         firstpatch=false
 491         sysload=`uptime | cut -d, -f 5`
 492         if [[ $sysload > $watermark ]]; then
 493             # Wait a bit when system load is too high.
 494             sleep ${standby}m
 495         else
 496             mkdir -p $TESTING
 497             mv $PATCHES/$PATCH $TESTING/
 498             PATCH=$TESTING/$PATCH
 499
 500             announce
 501             update && bootntest_patched && bootntest_pristine && compare_passes
 502             write_report
 503         fi
 504     fi
 505 done