sandbox/infrastructure/docutils-update.local

   1 #! /bin/bash
   2 # $Id$
   3 #
   4 # This script updates the Docutils web site.
   5 #
   6 # The web-root contains
   7 #
   8 # * files and directories from ``trunk/web``:
   9 #
  10 # * files and directories from ``trunk/docutils``:
  11 #   All files for easy referencing in mails.
  12 #
  13 # * and ``trunk/sandbox``.
  14 #
  15 # TODO
  16 #
  17 # * sourceforge does not offer cron, but shell or web instead:
  18 #   maybe use cgi-bin script to rebuild the website now and then,
  19 #   e.g. if accessed and more than one day old and svn is newer.
  20 # * use htmlfiles.lst to know which html-files have to be generated.
  21 #   not exclusively but for directory docs at least.
  22 # * maybe skip sandbox .txt files, as they might be input
  23 #   for tests for example.
  24 #
  25 # ATTENTION
  26 #
  27 # Any .html document with a corresponding .txt file is regenerated
  28 # if the .txt has changed, but no new .html files will be generated.
  29 #
  30 # ATTENTION
  31 #
  32 # Directories might contain Makefile.docutils-update files with
  33 # special instructions. only used in docs/user to call rst2s5.
  34 # Maybe add special treatment and remove this general solution.
  35 #
  36 # Options:
  37 #   -f    Do not give feedback.
  38 #   -t    Run the script in trace mode ("set -o xtrace").
  39 #   -u    Regenerate .html unconditionally.
  40 #   -v    Run verbosely.
  41 #
  42 # Prerequisites:
  43 #
  44
  45 # exit on error
  46 set -e
  47
  48 # make all newly created files group writeable
  49 umask 002
  50
  51 # URL for SVN project checkout:
  52 svnurl=https://docutils.svn.sourceforge.net/svnroot/docutils/trunk
  53
  54 basedir=`pwd`/update-dir
  55 if [ ! -e $basedir ] ; then
  56     test -d $basedir || mkdir $basedir
  57 fi
  58 project=docutils
  59 # $auxdir is non-public.
  60 auxdir=$basedir/aux
  61 test -d $auxdir || mkdir $auxdir
  62 # $htdocsdest is the destination for htdocs and will be moved to
  63 # another server later; so we keep it non-public (under $auxdir).
  64 htdocsdest=$auxdir/htdocs
  65 test -d $htdocsdest || mkdir $htdocsdest
  66 # Where to create the snapshots (non-public).
  67 snapshotdir=$auxdir/snapshots
  68 test -d $snapshotdir || mkdir $snapshotdir
  69 # # Where to publish the snapshots (public).
  70 # snapshotdest=/home/groups/ftp/pub/docutils
  71 bindir=$auxdir/bin
  72 test -d $bindir || mkdir $bindir
  73
  74 # htdocs directory on SF.net
  75 remoteproject=/home/groups/d/do/docutils
  76 remotehtdocs=$remoteproject/htdocs
  77 # local checkout
  78 pylib=$auxdir/lib/python
  79 lib=$pylib/$project
  80 # Lock directory.
  81 lockdir=$auxdir/lock
  82 # SSH stuff.
  83 sshdir=$auxdir/.ssh
  84 sshhost=docutilsupdate,docutils@web.sourceforge.net
  85 sshcommand="ssh -i $lockdir/id_dsa
  86             -o UserKnownHostsFile=$sshdir/known_hosts $sshhost"
  87 # Project base URL (for sitemap) without trailing slash.
  88 baseurl="http://docutils.sourceforge.net"
  89
  90 export PYTHONPATH=$pylib:$lib:$lib/extras
  91 export PATH=$lib/tools:$PATH
  92
  93 trace=0
  94 unconditional=0
  95 verbose=0
  96 feedback=1
  97
  98 while getopts ftuv opt
  99 do
 100     case $opt in
 101         f)  feedback=;;
 102         t)  trace=1;;
 103         u)  unconditional=1;;
 104         v)  verbose=1;;
 105         \?) exit 2;;
 106     esac
 107 done
 108 shift `expr $OPTIND - 1`
 109
 110 test $feedback && echo 'Starting docutils-update run...' || true
 111
 112 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 113     set -o xtrace
 114 fi
 115
 116 # Acquire lock.
 117 if ! mkdir $lockdir; then
 118     echo
 119     echo Could not create lock directory at
 120     echo $lockdir
 121     echo
 122     echo Please ensure no other user is running this script
 123     echo and delete the directory.
 124     exit 1
 125 fi
 126 # Always clean up on exit.
 127 trap "rm -rf $lockdir; trap - 0; exit 1" 0 1 2 3 15
 128 # Make sure the lock directory is deletable (i.e. rwx) by other group
 129 # members (in case this script crashes after copying files into the
 130 # directory) and un-readable by world (because we'll be storing the
 131 # key in it).
 132 chmod 0770 $lockdir
 133
 134
 135 # update library area
 136 if [ -e $lib ] ; then
 137     cd $lib
 138     svn up --quiet
 139 else
 140     test -d $pylib || mkdir -p $pylib
 141     cd $pylib
 142     svn checkout $svnurl/docutils
 143 fi
 144
 145 # -------------------- Snapshots: --------------------
 146
 147 # gather the materials
 148 cd $snapshotdir
 149 for DIR in docutils sandbox web ; do
 150     test -d $DIR || svn checkout $svnurl/$DIR
 151 done
 152 svn -q revert $project/$project/__init__.py
 153 haschanges="`svn up docutils sandbox web | grep -v '^At revision '; true`"
 154
 155 # update __version_details__ string
 156 version_details="snapshot `date -u +%Y-%m-%d`, r`svn info docutils | grep ^Revision: | sed 's/^Revision: //'`"
 157 (echo ",s/^__version_details__ = .*\$/__version_details__ = '$version_details'/";
 158     echo wq) | ed $project/$project/__init__.py 2> /dev/null
 159
 160 # Ensure proper directory permissions are set so that the files can be
 161 # modified by several users.  Changing permissions of files is
 162 # probably not necessary because files can be deleted and re-created.
 163 # Do not change permissions of aux directory to keep it non-public
 164 # (but change permissions for all subdirectories).
 165 #find $basedir -type f -print0 | xargs -0 chmod ug+rw 2> /dev/null || true
 166 find $basedir -name aux -o -type d -print0 | xargs -0 chmod ug+rwxs 2> /dev/null || true
 167
 168 # revert and touch (to avoid updating the web site only because of the
 169 # changed timestamp)
 170 svn -q revert $project/$project/__init__.py
 171 touch $project/$project/__init__.py --date \
 172     "`svn info $project/$project/__init__.py | \
 173     grep 'Last Changed Date:' | sed 's/[^:]*: //'`"
 174
 175 # -------------------- htdocs: --------------------
 176
 177 cd $snapshotdir
 178
 179 function copy_to_htdocsdest() {
 180     find "$@" -type d -name .svn -prune -o \( -type f -o -type l \) -print0 | \
 181         xargs -0 cp --no-dereference --update --parents \
 182             --target-directory=$htdocsdest
 183 }
 184
 185 # update htdocs
 186 copy_to_htdocsdest sandbox
 187 (cd $project; copy_to_htdocsdest *)
 188 (cd web; copy_to_htdocsdest * .[^.]*)
 189
 190 # update HTML docs
 191 cd $htdocsdest/tools
 192
 193 if [ $trace -eq 0 ] ; then
 194     set +o xtrace
 195 fi
 196
 197 for makefile in `find .. -name Makefile.docutils-update` ; do
 198     dir=`dirname $makefile`
 199     ( cd $dir ; make -f Makefile.docutils-update -s )
 200 done
 201
 202 for htmlfile in `find .. -name '*.html'` ; do
 203     dir=`dirname $htmlfile`
 204     base=`basename $htmlfile .html`
 205     txtfile=$dir/$base.txt
 206     if [ "$base" == "standalone_rst_html4strict" ] ; then
 207         # breaks web update
 208         echo "skipped: $dir $base"
 209     elif [ -e $txtfile ] ; then
 210         if [ $unconditional -eq 1 -o $txtfile -nt $htmlfile ] ; then
 211             if [ "${base:0:4}" == "pep-" ] ; then
 212                 test $feedback && echo "$txtfile (PEP)" || true
 213                 python $lib/tools/rstpep2html.py --config=$dir/docutils.conf $txtfile $htmlfile
 214                 haschanges=1
 215             else
 216                 test $feedback && echo $txtfile || true
 217                 python $lib/tools/rst2html.py --config=$dir/docutils.conf $txtfile $htmlfile
 218                 haschanges=1
 219             fi
 220         fi
 221     fi
 222 done
 223
 224 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 225     set -o xtrace
 226 fi
 227
 228 # -------------------- XML sitemap for search engines: --------------------
 229
 230 cd $htdocsdest
 231
 232 # Update the sitemap only if something has changed because it takes
 233 # very much CPU time.
 234 if test -n "$haschanges"; then
 235     (
 236         echo '<?xml version="1.0" encoding="UTF-8"?>'
 237         echo '<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">'
 238         if [ $trace -eq 0 ] ; then
 239             set +o xtrace
 240         fi
 241         find . -name '.[^.]*' -prune -o -type d -printf '%p/\n' \
 242                 -o \( -type f -o -type l \) -print | \
 243             while read i; do
 244                 # i is the file name.
 245                 if test "$i" == ./; then
 246                     # Homepage.
 247                     i=index.html
 248                     url="$baseurl/"
 249                 elif test "$i" == ./sitemap -o "${i: -1}" == / -a -f "${i}index.html"; then
 250                     # This is a directory and it has an index.html, so we
 251                     # don't need to include it.
 252                     continue
 253                 else
 254                     url="$baseurl${i:1}"
 255                     url="${url// /%20}"
 256                 fi
 257                 lastmod="`date --iso-8601=seconds -u -r "$i"`"
 258                 # Google wants a colon in front of the last two digits.
 259                 lastmod="${lastmod::22}:00"
 260                 if test "${i: -5}" == .html; then
 261                     # HTML files (including the home page) have highest priority.
 262                     priority=1.0
 263                 elif test "${i: -4}" == .txt; then
 264                     # Text files have medium priority.
 265                     priority=0.5
 266                 else
 267                     # Everything else (source files etc.) has low priority.
 268                     priority=0.2
 269                 fi
 270                 echo "<url><loc>$url</loc><lastmod>$lastmod</lastmod><priority>$priority</priority></url>"
 271             done
 272         if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 273             set -o xtrace
 274         fi
 275         echo '</urlset>'
 276     ) > sitemap
 277     gzip -f sitemap
 278 fi
 279
 280 # -------------------- Push changes to remote server. --------------------
 281
 282 # sourceforge no longer allows shell access, use rsync via ssh
 283 # specify your user in your .ssh/config
 284
 285 cd $htdocsdest
 286 # do not use -a to avoid "failed to set permissions"
 287 # -t preserve modification times. But a new svn checkout has new modtime.
 288 rsync -e ssh -r -t ./ web.sourceforge.net:$remotehtdocs
 289
 290 trap - 0 1 2 3 15
 291 rm -rf $lockdir
 292 test $feedback && echo '...docutils-update done.' || true
 293
 294 # Local Variables:
 295 # indent-tabs-mode: nil
 296 # End: