contrib/patch_tester.sh

   1 #!/bin/sh
   2
   3 # Tests a set of patches from a directory.
   4 # Copyright (C) 2007, 2008, 2011  Free Software Foundation, Inc.
   5 # Contributed by Sebastian Pop <sebastian.pop@amd.com>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 cat <<EOF
  22
  23 WARNING: This script should only be fed with patches from known
  24          authorized and trusted sources.  Don't even think about
  25          hooking it up to a raw feed from the gcc-patches list or
  26          you'll regret it.
  27
  28 EOF
  29
  30 args=$@
  31
  32 svnpath=svn://gcc.gnu.org/svn/gcc
  33 dashj=
  34 default_standby=1
  35 standby=$default_standby
  36 default_watermark=0.60
  37 watermark=$default_watermark
  38 savecompilers=false
  39 nopristinecache=false
  40 nogpg=false
  41 stop=false
  42
  43 usage() {
  44     cat <<EOF
  45 patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
  46                 [-svnpath URL] [-stop] [-nopristinecache]
  47                 <source_dir> [patches_dir [state_dir [build_dir]]]
  48
  49     J is the flag passed to make.  Default is empty string.
  50
  51     STANDBY is the number of minutes between checks for new patches in
  52     PATCHES_DIR.  Default is ${default_standby} minutes.
  53
  54     WATERMARK is the 5 minute average system charge under which a new
  55     compile can start.  Default is ${default_watermark}.
  56
  57     SAVECOMPILERS copies the compilers in the same directory as the
  58     test results for the non patched version.  Default is not copy.
  59
  60     NOPRISTINECACHE prevents use of cached test results from any earlier
  61     test runs on the pristine version of the branch and revision under
  62     test (the default behaviour).  This should be used when testing the
  63     same revision and patch with multiple sets of configure options, as
  64     these may affect the set of baseline failures.
  65
  66     NOGPG can be used to avoid checking the GPG signature of patches.
  67
  68     URL is the location of the GCC SVN repository.  The default is
  69     ${svnpath}.
  70
  71     STOP exits when PATCHES_DIR is empty.
  72
  73     SOURCE_DIR is the directory containing GCC's toplevel configure.
  74
  75     PATCHES_DIR is the directory containing the patches to be tested.
  76     Default is SOURCE_DIR/patches.
  77
  78     STATE_DIR is where the tester maintains its internal state.
  79     Default is SOURCE_DIR/state.
  80
  81     BUILD_DIR is the build tree, a temporary directory that this
  82     script will delete and recreate.  Default is SOURCE_DIR/obj.
  83
  84 EOF
  85     exit 1
  86 }
  87
  88 makedir () {
  89     DIRNAME=$1
  90     mkdir -p $DIRNAME
  91     if [ $? -ne 0 ]; then
  92         echo "ERROR: could not make directory $DIRNAME"
  93         exit 1
  94     fi
  95 }
  96
  97 while [ $# -ne 0 ]; do
  98     case $1 in
  99         -j*)
 100             dashj=$1; shift
 101             ;;
 102         -standby)
 103             [[ $# > 2 ]] || usage
 104             standby=$2; shift; shift
 105             ;;
 106         -watermark)
 107             [[ $# > 2 ]] || usage
 108             watermark=$2; shift; shift
 109             ;;
 110         -savecompilers)
 111             savecompilers=true; shift
 112             ;;
 113         -nopristinecache)
 114             nopristinecache=true; shift
 115             ;;
 116         -nogpg)
 117             nogpg=true; shift
 118             ;;
 119         -stop)
 120             stop=true; shift
 121             ;;
 122         -svnpath)
 123             svnpath=$2; shift; shift
 124             ;;
 125         -*)
 126             echo "Invalid option: $1"
 127             usage
 128             ;;
 129         *)
 130             break
 131             ;;
 132     esac
 133 done
 134
 135 test $# -eq 0 && usage
 136
 137 SOURCE=$1
 138 PATCHES=
 139 STATE=
 140 BUILD=
 141
 142 if [[ $# < 2 ]]; then
 143     PATCHES=$SOURCE/patches
 144 else
 145     PATCHES=$2
 146 fi
 147 if [[ $# < 3 ]]; then
 148     STATE=$SOURCE/state
 149 else
 150     STATE=$3
 151 fi
 152 if [[ $# < 4 ]]; then
 153     BUILD=$SOURCE/obj
 154 else
 155     BUILD=$4
 156 fi
 157
 158 [ -d $PATCHES ] || makedir $PATCHES
 159 [ -d $STATE ] || makedir $STATE
 160 [ -d $STATE/patched ] || makedir $STATE/patched
 161 [ -d $SOURCE ] || makedir $SOURCE
 162 [ -f $SOURCE/config.guess ] || {
 163     cd $SOURCE
 164     svn -q co $svnpath/trunk .
 165     if [ $? -ne 0 ]; then
 166         echo "ERROR: initial svn checkout failed"
 167         exit 1
 168     fi
 169 }
 170
 171 # This can contain required local settings:
 172 #  default_config  configure options, always passed
 173 #  default_make    make bootstrap options, always passed
 174 #  default_check   make check options, always passed
 175 [ -f $STATE/defaults ] && . $STATE/defaults
 176
 177 VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 178
 179 exec >> $STATE/tester.log 2>&1 || exit 1
 180 set -x
 181
 182 TESTING=$STATE/testing
 183 REPORT=$TESTING/report
 184 PRISTINE=$TESTING/pristine
 185 PATCHED=$TESTING/patched
 186 PATCH=
 187 TARGET=`$SOURCE/config.guess || exit 1`
 188 TESTLOGS="gcc/testsuite/gcc/gcc.sum
 189 gcc/testsuite/gfortran/gfortran.sum
 190 gcc/testsuite/g++/g++.sum
 191 gcc/testsuite/objc/objc.sum
 192 $TARGET/libstdc++-v3/testsuite/libstdc++.sum
 193 $TARGET/libffi/testsuite/libffi.sum
 194 $TARGET/libgomp/testsuite/libgomp.sum
 195 $TARGET/libmudflap/testsuite/libmudflap.sum"
 196 COMPILERS="gcc/cc1
 197 gcc/cc1obj
 198 gcc/cc1plus
 199 gcc/f951
 200 gcc/jc1
 201 gcc/gnat1
 202 gcc/tree1"
 203
 204 now () {
 205     echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
 206 }
 207
 208 report () {
 209     echo "$@" >> $REPORT
 210 }
 211
 212 freport () {
 213     if [ -s $1 ]; then
 214         report "(cat $1"
 215         cat $1 >> $REPORT
 216         report "tac)"
 217     fi
 218 }
 219
 220 cleanup () {
 221     cd $SOURCE
 222     svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
 223 }
 224
 225 selfexec () {
 226     exec ${CONFIG_SHELL-/bin/sh} $0 $args
 227 }
 228
 229 update () {
 230     svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
 231     if [ x$svn_branch = x ]; then
 232         svn_branch=trunk
 233     fi
 234
 235     svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
 236     if [ x$svn_revision = x ]; then
 237         svn_revision=HEAD
 238     fi
 239
 240     cleanup
 241     cd $SOURCE
 242     case $svn_branch in
 243         trunk)
 244             if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
 245                 report "failed to update svn sources with"
 246                 report "svn switch -r $svn_revision $svnpath/trunk"
 247                 freport $TESTING/svn
 248                 return 1
 249             fi
 250             ;;
 251
 252         ${svnpath}*)
 253             if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
 254                 report "failed to update svn sources with"
 255                 report "svn switch -r $svn_revision $svn_branch"
 256                 freport $TESTING/svn
 257                 return 1
 258             fi
 259             ;;
 260
 261         *)
 262             if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
 263                 report "failed to update svn sources with"
 264                 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
 265                 freport $TESTING/svn
 266                 return 1
 267             fi
 268             ;;
 269     esac
 270     contrib/gcc_update --touch
 271
 272     current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 273     if [[ $VERSION < $current_version ]]; then
 274         if [ -f $SOURCE/contrib/patch_tester.sh ]; then
 275             selfexec
 276         fi
 277     fi
 278
 279     return 0
 280 }
 281
 282 apply_patch () {
 283     if [ $nogpg = false ]; then
 284         if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
 285             report "your patch failed to verify:"
 286             freport $TESTING/gpgverify
 287             return 1
 288         fi
 289     fi
 290
 291     cd $SOURCE
 292     if ! patch -p0 < $PATCH &> $TESTING/patching ; then
 293         report "your patch failed to apply:"
 294         report "(check that the patch was created at the top level)"
 295         freport $TESTING/patching
 296         return 1
 297     fi
 298
 299     # Just assume indexes for now -- not really great, but svn always
 300     # makes them.
 301     grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
 302         # If the patch resulted in an empty file, delete it.
 303         # This is how svn reports deletions.
 304         if [ ! -s $file ]; then
 305             rm -f $file
 306             report "Deleting empty file $file"
 307         fi
 308     done
 309 }
 310
 311 save_compilers () {
 312     for COMPILER in $COMPILERS ; do
 313         if [ -f $BUILD/$COMPILER ]; then
 314             cp $BUILD/$COMPILER $PRISTINE
 315         fi
 316     done
 317 }
 318
 319 bootntest () {
 320     rm -rf $BUILD
 321     mkdir $BUILD
 322     cd $BUILD
 323
 324     CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
 325     CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
 326     if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
 327         report "configure with `basename $1` version failed with:"
 328         freport $1/configure
 329         return 1
 330     fi
 331
 332     MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
 333     MAKE_ARGS="$default_make $MAKE_ARGS"
 334     if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
 335         report "bootstrap with `basename $1` version failed with last lines:"
 336         tail -30 $1/bootstrap > $1/last_bootstrap
 337         freport $1/last_bootstrap
 338         report "grep --context=20 Error bootstrap:"
 339         grep --context=20 Error $1/bootstrap > $1/bootstrap_error
 340         freport $1/bootstrap_error
 341         return 1
 342     fi
 343
 344     CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
 345     CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
 346     eval make $dashj $CHECK_OPTIONS -k check &> $1/check
 347
 348     SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
 349     if [ x$SUITESRUN = x ]; then
 350         report "check with `basename $1` version failed, no testsuites were run"
 351         return 1
 352     fi
 353
 354     for LOG in $TESTLOGS ; do
 355         if [ -f $BUILD/$LOG ]; then
 356             mv $BUILD/$LOG $1
 357             mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
 358         fi
 359     done
 360
 361     return 0
 362 }
 363
 364 bootntest_patched () {
 365     cleanup
 366     mkdir -p $PATCHED
 367     apply_patch && bootntest $PATCHED
 368     return $?
 369 }
 370
 371 # Build the pristine tree with exactly the same options as the patch under test.
 372 bootntest_pristine () {
 373     cleanup
 374     current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
 375     current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
 376     PRISTINE=$STATE/$current_branch/$current_version
 377
 378     if [ $nopristinecache = true ]; then
 379       rm -rf $PRISTINE
 380     fi
 381     if [ -d $PRISTINE ]; then
 382         ln -s $PRISTINE $TESTING/pristine
 383         return 0
 384     else
 385         mkdir -p $PRISTINE
 386         ln -s $PRISTINE $TESTING/pristine
 387         bootntest $PRISTINE
 388         RETVAL=$?
 389         if [ $RETVAL = 0 -a $savecompilers = true ]; then
 390             save_compilers
 391         fi
 392         return $RETVAL
 393     fi
 394 }
 395
 396 regtest () {
 397     touch $1/report
 398     touch $1/passes
 399     touch $1/failed
 400     touch $1/regress
 401
 402     for LOG in $TESTLOGS ; do
 403         NLOG=`basename $LOG`
 404         if [ -f $1/$NLOG ]; then
 405             awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
 406         fi
 407     done | sort | uniq > $1/failed
 408
 409     comm -12 $1/failed $1/passes >> $1/regress
 410     NUMREGRESS=`wc -l < $1/regress | tr -d ' '`
 411
 412     if [ $NUMREGRESS -eq 0 ] ; then
 413         for LOG in $TESTLOGS ; do
 414             NLOG=`basename $LOG`
 415             if [ -f $1/$NLOG ] ; then
 416                 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
 417             fi
 418         done | sort | uniq | comm -23 - $1/failed > $1/passes
 419         echo "there are no regressions with your patch." >> $1/report
 420     else
 421         echo "with your patch there are $NUMREGRESS regressions." >> $1/report
 422         echo "list of regressions with your patch:" >> $1/report
 423         cat $1/regress >> $1/report
 424     fi
 425 }
 426
 427 contrib_compare_tests () {
 428     report "comparing logs with contrib/compare_tests:"
 429     for LOG in $TESTLOGS ; do
 430         NLOG=`basename $LOG`
 431         if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
 432             $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
 433             freport $TESTING/compare_$NLOG
 434         fi
 435     done
 436 }
 437
 438 compare_passes () {
 439     regtest $PRISTINE
 440     cp $PRISTINE/passes $PATCHED
 441     regtest $PATCHED
 442     freport $PATCHED/report
 443     report "FAILs with patched version:"
 444     freport $PATCHED/failed
 445     report "FAILs with pristine version:"
 446     freport $PRISTINE/failed
 447
 448     # contrib_compare_tests
 449 }
 450
 451 write_report () {
 452     backup_patched=$STATE/patched/`now`
 453     report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."
 454
 455     EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
 456     if [ x$EMAIL != x ]; then
 457         mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
 458     fi
 459
 460     mv $TESTING $backup_patched
 461 }
 462
 463 announce () {
 464     EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
 465     if [ x$EMAIL != x ]; then
 466
 467         START_REPORT=$TESTING/start_report
 468         echo "Hi, " >> $START_REPORT
 469         echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
 470         echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
 471         echo "Bye, your automatic tester." >> $START_REPORT
 472         mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
 473     fi
 474 }
 475
 476 # After selfexec, $TESTING is already set up.
 477 if [ -d $TESTING ]; then
 478     # The only file in $TESTING is the patch.
 479     PATCH=`ls -rt -1 $TESTING | head -1`
 480     PATCH=$TESTING/$PATCH
 481     if [ -f $PATCH ]; then
 482         bootntest_patched && bootntest_pristine && compare_passes
 483         write_report
 484     fi
 485 fi
 486
 487 firstpatch=true
 488 while true; do
 489     PATCH=`ls -rt -1 $PATCHES | head -1`
 490     if [ x$PATCH = x ]; then
 491         if [ $stop = true ]; then
 492             if [ $firstpatch = true ]; then
 493                 echo "No patches ready to test, quitting."
 494                 exit 1
 495             else
 496                 echo "No more patches to test."
 497                 exit 0
 498             fi
 499         fi
 500         sleep ${standby}m
 501     else
 502         firstpatch=false
 503         sysload=`uptime | cut -d, -f 5`
 504         if [[ $sysload > $watermark ]]; then
 505             # Wait a bit when system load is too high.
 506             sleep ${standby}m
 507         else
 508             mkdir -p $TESTING
 509             mv $PATCHES/$PATCH $TESTING/
 510             PATCH=$TESTING/$PATCH
 511
 512             announce
 513             update && bootntest_patched && bootntest_pristine && compare_passes
 514             write_report
 515         fi
 516     fi
 517 done