Misc/build.sh

   1 #!/bin/sh
   2
   3 ## Script to build and test the latest python from svn.  It basically
   4 ## does this:
   5 ##   svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
   6 ##
   7 ## Logs are kept and rsync'ed to the host.  If there are test failure(s),
   8 ## information about the failure(s) is mailed.
   9 ##
  10 ## This script is run on the PSF's machine as user neal via crontab.
  11 ##
  12 ## Yes, this script would probably be easier in python, but then
  13 ## there's a bootstrap problem.  What if Python doesn't build?
  14 ##
  15 ## This script should be fairly clean Bourne shell, ie not too many
  16 ## bash-isms.  We should try to keep it portable to other Unixes.
  17 ## Even though it will probably only run on Linux.  I'm sure there are
  18 ## several GNU-isms currently (date +%s and readlink).
  19 ##
  20 ## Perhaps this script should be broken up into 2 (or more) components.
  21 ## Building doc is orthogonal to the rest of the python build/test.
  22 ##
  23
  24 ## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
  25
  26 ## FIXME: we should run valgrind
  27 ## FIXME: we should run code coverage
  28
  29 ## Utilities invoked in this script include:
  30 ##    basename, date, dirname, expr, grep, readlink, uname
  31 ##    cksum, make, mutt, rsync, svn
  32
  33 ## remember where did we started from
  34 DIR=`dirname $0`
  35 if [ "$DIR" = "" ]; then
  36    DIR="."
  37 fi
  38
  39 ## make directory absolute
  40 DIR=`readlink -f $DIR`
  41 FULLPATHNAME="$DIR/`basename $0`"
  42 ## we want Misc/..
  43 DIR=`dirname $DIR`
  44
  45 ## Configurable options
  46
  47 FAILURE_SUBJECT="Python Regression Test Failures"
  48 #FAILURE_MAILTO="python-checkins@python.org"
  49 #FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
  50 FAILURE_MAILTO="nnorwitz@gmail.com"
  51
  52 REMOTE_SYSTEM="neal@dinsdale.python.org"
  53 REMOTE_DIR="/data/ftp.python.org/pub/docs.python.org/dev/"
  54 RESULT_FILE="$DIR/build/index.html"
  55 INSTALL_DIR="/tmp/python-test/local"
  56 RSYNC_OPTS="-aC -e ssh"
  57
  58 REFLOG="build/reflog.txt.out"
  59
  60 ## utility functions
  61 current_time() {
  62     date +%s
  63 }
  64
  65 update_status() {
  66     now=`current_time`
  67     time=`expr $now - $3`
  68     echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE
  69 }
  70
  71 mail_on_failure() {
  72     if [ "$NUM_FAILURES" != "0" ]; then
  73         mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $FAILURE_MAILTO < $2
  74     fi
  75 }
  76
  77 ## setup
  78 cd $DIR
  79 mkdir -p build
  80 rm -f $RESULT_FILE build/*.out
  81 rm -rf $INSTALL_DIR
  82
  83 ## create results file
  84 TITLE="Automated Python Build Results"
  85 echo "<html>" >> $RESULT_FILE
  86 echo "  <head>" >> $RESULT_FILE
  87 echo "    <title>$TITLE</title>" >> $RESULT_FILE
  88 echo "    <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE
  89 echo "  </head>" >> $RESULT_FILE
  90 echo "<body>" >> $RESULT_FILE
  91 echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE
  92 echo "<table>" >> $RESULT_FILE
  93 echo "  <tr>" >> $RESULT_FILE
  94 echo "    <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE
  95 echo "  </tr><tr>" >> $RESULT_FILE
  96 echo "    <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE
  97 echo "  </tr><tr>" >> $RESULT_FILE
  98 echo "    <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE
  99 echo "  </tr>" >> $RESULT_FILE
 100 echo "</table>" >> $RESULT_FILE
 101 echo "<ul>" >> $RESULT_FILE
 102
 103 ## update, build, and test
 104 ORIG_CHECKSUM=`cksum $FULLPATHNAME`
 105 F=svn-update.out
 106 start=`current_time`
 107 svn update >& build/$F
 108 err=$?
 109 update_status "Updating" "$F" $start
 110 if [ $err = 0 ]; then
 111     ## FIXME: we should check if this file has changed.
 112     ##  If it has changed, we should re-run the script to pick up changes.
 113     if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then
 114         exec $FULLPATHNAME $@
 115     fi
 116
 117     F=svn-stat.out
 118     start=`current_time`
 119     svn stat >& build/$F
 120     ## ignore some of the diffs
 121     NUM_DIFFS=`egrep -vc '^.      (@test|db_home|Lib/test/(regrtest\.py|db_home))$' build/$F`
 122     update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start
 123
 124     F=configure.out
 125     start=`current_time`
 126     ./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F
 127     err=$?
 128     update_status "Configuring" "$F" $start
 129     if [ $err = 0 ]; then
 130         F=make.out
 131         start=`current_time`
 132         make >& build/$F
 133         err=$?
 134         warnings=`grep warning build/$F | egrep -vc "te?mpnam(_r|)' is dangerous,"`
 135         update_status "Building ($warnings warnings)" "$F" $start
 136         if [ $err = 0 ]; then
 137             ## make install
 138             F=make-install.out
 139             start=`current_time`
 140             make install >& build/$F
 141             update_status "Installing" "$F" $start
 142
 143             ## make and run basic tests
 144             F=make-test.out
 145             start=`current_time`
 146             make test >& build/$F
 147             NUM_FAILURES=`grep -ic fail build/$F`
 148             update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start
 149             ## FIXME: should mail since -uall below should find same problems
 150             mail_on_failure "basics" build/$F
 151
 152             ## run the tests looking for leaks
 153             F=make-test-refleak.out
 154             start=`current_time`
 155             ./python ./Lib/test/regrtest.py -R 4:3:$REFLOG -u network >& build/$F
 156             NUM_FAILURES=`grep -ic leak $REFLOG`
 157             update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start
 158             mail_on_failure "refleak" $REFLOG
 159
 160             ## now try to run all the tests
 161             F=make-testall.out
 162             start=`current_time`
 163             ## skip curses when running from cron since there's no terminal
 164             ## skip sound since it's not setup on the PSF box (/dev/dsp)
 165             ./python -E -tt ./Lib/test/regrtest.py -uall -x test_curses test_linuxaudiodev test_ossaudiodev >& build/$F
 166             NUM_FAILURES=`grep -ic fail build/$F`
 167             update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start
 168             mail_on_failure "all" build/$F
 169         fi
 170     fi
 171 fi
 172
 173
 174 ## make doc
 175 cd Doc
 176 F="make-doc.out"
 177 start=`current_time`
 178 make >& ../build/$F
 179 err=$?
 180 update_status "Making doc" "$F" $start
 181 if [ $err != 0 ]; then
 182     NUM_FAILURES=1
 183     mail_on_failure "doc" ../build/$F
 184 fi
 185
 186 echo "</ul>" >> $RESULT_FILE
 187 echo "</body>" >> $RESULT_FILE
 188 echo "</html>" >> $RESULT_FILE
 189
 190 ## copy results
 191 rsync $RSYNC_OPTS html/ $REMOTE_SYSTEM:$REMOTE_DIR
 192 cd ../build
 193 rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/
 194