sandbox/infrastructure/docutils-update.local

   1 #! /bin/bash
   2 # $Id$
   3 #
   4 # This script updates the Docutils web site.
   5 # ATTENTION
   6 # Any .html document with a corresponding .txt file is regenerated
   7 # if the .txt has changed, but no new .html files will be generated.
   8 #
   9 # Options:
  10 #   -f    Do not give feedback.
  11 #   -t    Run the script in trace mode ("set -o xtrace").
  12 #   -u    Regenerate .html unconditionally.
  13 #   -v    Run verbosely.
  14 #
  15 # Prerequisites:
  16 #
  17
  18 # exit on error
  19 set -e
  20
  21 # make all newly created files group writeable
  22 umask 002
  23
  24 # URL for SVN project checkout:
  25 svnurl=http://svn.berlios.de/svnroot/repos/docutils/trunk
  26
  27 basedir=`pwd`/update-dir
  28 if [ ! -e $basedir ] ; then
  29     test -d $basedir || mkdir $basedir
  30 fi
  31 project=docutils
  32 # $auxdir is non-public.
  33 auxdir=$basedir/aux
  34 test -d $auxdir || mkdir $auxdir
  35 # $htdocsdest is the destination for htdocs and will be moved to
  36 # another server later; so we keep it non-public (under $auxdir).
  37 htdocsdest=$auxdir/htdocs
  38 test -d $htdocsdest || mkdir $htdocsdest
  39 # Where to create the snapshots (non-public).
  40 snapshotdir=$auxdir/snapshots
  41 test -d $snapshotdir || mkdir $snapshotdir
  42 # # Where to publish the snapshots (public).
  43 # snapshotdest=/home/groups/ftp/pub/docutils
  44 bindir=$auxdir/bin
  45 test -d $bindir || mkdir $bindir
  46
  47 htdocs_patchfile=$auxdir/htdocs.patch
  48 htdocs_tarball=$auxdir/htdocs.tar
  49 htdocs_new_tarball=$auxdir/htdocs.new.tar
  50 # htdocs directory on SF.net
  51 remoteproject=/home/groups/d/do/docutils
  52 remotehtdocs=$remoteproject/htdocs
  53 # local checkout
  54 pylib=$auxdir/lib/python
  55 lib=$pylib/$project
  56 # Lock directory.
  57 lockdir=$auxdir/lock
  58 # SSH stuff.
  59 sshdir=$auxdir/.ssh
  60 sshhost=docutilsupdate,docutils@web.sourceforge.net
  61 sshcommand="ssh -i $lockdir/id_dsa
  62             -o UserKnownHostsFile=$sshdir/known_hosts $sshhost"
  63 # Project base URL (for sitemap) without trailing slash.
  64 baseurl="http://docutils.sourceforge.net"
  65
  66 export PYTHONPATH=$pylib:$lib:$lib/extras
  67 export PATH=$lib/tools:$PATH
  68
  69 trace=0
  70 unconditional=0
  71 verbose=0
  72 feedback=1
  73
  74 while getopts ftuv opt
  75 do
  76     case $opt in
  77         f)  feedback=;;
  78         t)  trace=1;;
  79         u)  unconditional=1;;
  80         v)  verbose=1;;
  81         \?) exit 2;;
  82     esac
  83 done
  84 shift `expr $OPTIND - 1`
  85
  86 test $feedback && echo 'Starting docutils-update run...' || true
  87
  88 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
  89     set -o xtrace
  90 fi
  91
  92 # Acquire lock.
  93 if ! mkdir $lockdir; then
  94     echo
  95     echo Could not create lock directory at
  96     echo $lockdir
  97     echo
  98     echo Please ensure no other user is running this script
  99     echo and delete the directory.
 100     exit 1
 101 fi
 102 # Always clean up on exit.
 103 trap "rm -rf $lockdir; trap - 0; exit 1" 0 1 2 3 15
 104 # Make sure the lock directory is deletable (i.e. rwx) by other group
 105 # members (in case this script crashes after copying files into the
 106 # directory) and un-readable by world (because we'll be storing the
 107 # key in it).
 108 chmod 0770 $lockdir
 109
 110
 111 # update library area
 112 if [ -e $lib ] ; then
 113     cd $lib
 114     svn up --quiet
 115 else
 116     test -d $pylib || mkdir -p $pylib
 117     cd $pylib
 118     svn checkout $svnurl/docutils
 119 fi
 120
 121 # -------------------- Snapshots: --------------------
 122
 123 # gather the materials
 124 cd $snapshotdir
 125 for DIR in docutils sandbox web ; do
 126     test -d $DIR || svn checkout $svnurl/$DIR
 127 done
 128 svn -q revert $project/$project/__init__.py
 129 haschanges="`svn up docutils sandbox web | grep -v '^At revision '; true`"
 130
 131 # update __version_details__ string
 132 version_details="snapshot `date -u +%Y-%m-%d`, r`svn info docutils | grep ^Revision: | sed 's/^Revision: //'`"
 133 (echo ",s/^__version_details__ = .*\$/__version_details__ = '$version_details'/";
 134     echo wq) | ed $project/$project/__init__.py 2> /dev/null
 135
 136 # Ensure proper directory permissions are set so that the files can be
 137 # modified by several users.  Changing permissions of files is
 138 # probably not necessary because files can be deleted and re-created.
 139 # Do not change permissions of aux directory to keep it non-public
 140 # (but change permissions for all subdirectories).
 141 #find $basedir -type f -print0 | xargs -0 chmod ug+rw 2> /dev/null || true
 142 find $basedir -name aux -o -type d -print0 | xargs -0 chmod ug+rwxs 2> /dev/null || true
 143
 144 # create the snapshots
 145 exclude='--exclude=.svn'
 146 tar -cz $exclude -f $project-snapshot.tgz $project
 147 tar -cz $exclude -f $project-sandbox-snapshot.tgz sandbox
 148 tar -cz $exclude -f $project-web-snapshot.tgz web
 149 ( cd sandbox/gschwant ;
 150   tar -cz $exclude -f ../../docfactory-snapshot.tgz docfactory )
 151
 152 # plant the snapshots
 153 mv -f *snapshot.tgz $htdocsdest
 154
 155 # revert and touch (to avoid updating the web site only because of the
 156 # changed timestamp)
 157 svn -q revert $project/$project/__init__.py
 158 touch $project/$project/__init__.py --date \
 159     "`svn info $project/$project/__init__.py | \
 160     grep 'Last Changed Date:' | sed 's/[^:]*: //'`"
 161
 162 # -------------------- htdocs: --------------------
 163
 164 cd $snapshotdir
 165
 166 function copy_to_htdocsdest() {
 167     find "$@" -type d -name .svn -prune -o \( -type f -o -type l \) -print0 | \
 168         xargs -0 cp --no-dereference --update --parents \
 169             --target-directory=$htdocsdest
 170 }
 171
 172 # update htdocs
 173 copy_to_htdocsdest sandbox
 174 (cd $project; copy_to_htdocsdest *)
 175 (cd web; copy_to_htdocsdest * .[^.]*)
 176
 177 # update HTML docs
 178 cd $htdocsdest/tools
 179
 180 if [ $trace -eq 0 ] ; then
 181     set +o xtrace
 182 fi
 183
 184 for makefile in `find .. -name Makefile.docutils-update` ; do
 185     dir=`dirname $makefile`
 186     ( cd $dir ; make -f Makefile.docutils-update -s )
 187 done
 188
 189 for htmlfile in `find .. -name '*.html'` ; do
 190     dir=`dirname $htmlfile`
 191     base=`basename $htmlfile .html`
 192     txtfile=$dir/$base.txt
 193     if [ "$base" == "standalone_rst_html4strict" ] ; then
 194         # breaks web update
 195         echo "skipped: $dir $base"
 196     elif [ -e $txtfile ] ; then
 197         if [ $unconditional -eq 1 -o $txtfile -nt $htmlfile ] ; then
 198             if [ "${base:0:4}" == "pep-" ] ; then
 199                 test $feedback && echo "$txtfile (PEP)" || true
 200                 python $lib/tools/rstpep2html.py --config=$dir/docutils.conf $txtfile $htmlfile
 201                 haschanges=1
 202             else
 203                 test $feedback && echo $txtfile || true
 204                 python $lib/tools/rst2html.py --halt=5 --config=$dir/docutils.conf $txtfile $htmlfile
 205                 haschanges=1
 206             fi
 207         fi
 208     fi
 209 done
 210
 211 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 212     set -o xtrace
 213 fi
 214
 215 # -------------------- XML sitemap for search engines: --------------------
 216
 217 cd $htdocsdest
 218
 219 # Update the sitemap only if something has changed because it takes
 220 # very much CPU time.
 221 if test -n "$haschanges"; then
 222     (
 223         echo '<?xml version="1.0" encoding="UTF-8"?>'
 224         echo '<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">'
 225         if [ $trace -eq 0 ] ; then
 226             set +o xtrace
 227         fi
 228         find . -name '.[^.]*' -prune -o -type d -printf '%p/\n' \
 229                 -o \( -type f -o -type l \) -print | \
 230             while read i; do
 231                 # i is the file name.
 232                 if test "$i" == ./; then
 233                     # Homepage.
 234                     i=index.html
 235                     url="$baseurl/"
 236                 elif test "$i" == ./sitemap -o "${i: -1}" == / -a -f "${i}index.html"; then
 237                     # This is a directory and it has an index.html, so we
 238                     # don't need to include it.
 239                     continue
 240                 else
 241                     url="$baseurl${i:1}"
 242                     url="${url// /%20}"
 243                 fi
 244                 lastmod="`date --iso-8601=seconds -u -r "$i"`"
 245                 # Google wants a colon in front of the last two digits.
 246                 lastmod="${lastmod::22}:00"
 247                 if test "${i: -5}" == .html; then
 248                     # HTML files (including the home page) have highest priority.
 249                     priority=1.0
 250                 elif test "${i: -4}" == .txt; then
 251                     # Text files have medium priority.
 252                     priority=0.5
 253                 else
 254                     # Everything else (source files etc.) has low priority.
 255                     priority=0.2
 256                 fi
 257                 echo "<url><loc>$url</loc><lastmod>$lastmod</lastmod><priority>$priority</priority></url>"
 258             done
 259         if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 260             set -o xtrace
 261         fi
 262         echo '</urlset>'
 263     ) > sitemap
 264     gzip -f sitemap
 265 fi
 266
 267 # -------------------- Push changes to remote server. --------------------
 268
 269 # sourceforge no longer allows shell access, use rsync via ssh
 270 # specify your user in your .ssh/config
 271
 272 cd $htdocsdest
 273 # do not use -a to avoid "failed to set permissions"
 274 rsync -e ssh -r ./ web.sourceforge.net:$remotehtdocs
 275
 276 trap - 0 1 2 3 15
 277 rm -rf $lockdir
 278 test $feedback && echo '...docutils-update done.' || true
 279
 280 # Local Variables:
 281 # indent-tabs-mode: nil
 282 # End: