sandbox/infrastructure/docutils-update.local

   1 #! /bin/bash
   2 # $Id$
   3 #
   4 # This script updates the Docutils web site.
   5 #
   6 # The web-root contains
   7 #
   8 # * files and directories from ``trunk/web``:
   9 #
  10 # * files and directories from ``trunk/docutils``:
  11 #   All files for easy referencing in mails.
  12 #
  13 # * and ``trunk/sandbox``.
  14 #
  15 # TODO
  16 #
  17 # * sourceforge does not offer cron, but shell or web instead:
  18 #   maybe use cgi-bin script to rebuild the website now and then,
  19 #   e.g. if accessed and more than one day old and svn is newer.
  20 # * use htmlfiles.lst to know which html-files have to be generated.
  21 #   not exclusively but for directory docs at least.
  22 # * maybe skip sandbox .txt files, as they might be input
  23 #   for tests for example.
  24 # * this file might become easier if cleanest cron task handling is
  25 #   cut out, as it is not run by cron.
  26 #
  27 # ATTENTION
  28 #
  29 # Any .html document with a corresponding .txt file is regenerated
  30 # if the .txt has changed, but no new .html files will be generated.
  31 #
  32 # * Funny thing: sf hides README.txt files.
  33 #
  34 # ATTENTION
  35 #
  36 # Directories might contain Makefile.docutils-update files with
  37 # special instructions. only used in docs/user to call rst2s5.
  38 # Maybe add special treatment and remove this general solution.
  39 #
  40 # Options:
  41 #   -f    Do not give feedback.
  42 #   -t    Run the script in trace mode ("set -o xtrace").
  43 #   -u    Regenerate .html unconditionally.
  44 #   -v    Run verbosely.
  45 #
  46 # Prerequisites:
  47 #
  48
  49 # exit on error
  50 set -e
  51
  52 # make all newly created files group writeable
  53 umask 002
  54
  55 # URL for SVN project checkout:
  56 svnurl=https://docutils.svn.sourceforge.net/svnroot/docutils/trunk
  57
  58 basedir=`pwd`/update-dir
  59 if [ ! -e $basedir ] ; then
  60     test -d $basedir || mkdir $basedir
  61 fi
  62 project=docutils
  63 # $auxdir is non-public.
  64 auxdir=$basedir/aux
  65 test -d $auxdir || mkdir $auxdir
  66 # $htdocsdest is the destination for htdocs and will be moved to
  67 # another server later; so we keep it non-public (under $auxdir).
  68 htdocsdest=$auxdir/htdocs
  69 test -d $htdocsdest || mkdir $htdocsdest
  70 # Where to create the snapshots (non-public).
  71 snapshotdir=$auxdir/snapshots
  72 test -d $snapshotdir || mkdir $snapshotdir
  73
  74 # htdocs directory on SF.net
  75 remoteproject=/home/project-web/docutils
  76 remotehtdocs=$remoteproject/htdocs
  77
  78 # local checkout
  79 pylib=$auxdir/lib/python
  80 lib=$pylib/$project
  81 # Lock directory.
  82 lockdir=$auxdir/lock
  83
  84 # Project base URL (for sitemap) without trailing slash.
  85 baseurl="http://docutils.sourceforge.net"
  86
  87 export PYTHONPATH=$pylib:$lib:$lib/extras
  88 export PATH=$lib/tools:$PATH
  89
  90 trace=0
  91 unconditional=0
  92 verbose=0
  93 feedback=1
  94
  95 while getopts ftuv opt
  96 do
  97     case $opt in
  98         f)  feedback=;;
  99         t)  trace=1;;
 100         u)  unconditional=1;;
 101         v)  verbose=1;;
 102         \?) exit 2;;
 103     esac
 104 done
 105 shift `expr $OPTIND - 1`
 106
 107 test $feedback && echo 'Starting docutils-update run...' || true
 108
 109 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 110     set -o xtrace
 111 fi
 112
 113 # Acquire lock.
 114 if ! mkdir $lockdir; then
 115     echo
 116     echo Could not create lock directory at
 117     echo $lockdir
 118     echo
 119     echo Please ensure no other user is running this script
 120     echo and delete the directory.
 121     exit 1
 122 fi
 123 # Always clean up on exit.
 124 trap "rm -rf $lockdir; trap - 0; exit 1" 0 1 2 3 15
 125 # Make sure the lock directory is deletable (i.e. rwx) by other group
 126 # members (in case this script crashes after copying files into the
 127 # directory)
 128 chmod 0770 $lockdir
 129
 130 # update library area
 131 if [ -e $lib ] ; then
 132     cd $lib
 133     svn up --quiet
 134 else
 135     test -d $pylib || mkdir -p $pylib
 136     cd $pylib
 137     svn checkout $svnurl/docutils
 138 fi
 139
 140 # -------------------- Snapshots: --------------------
 141
 142 # gather the materials
 143 cd $snapshotdir
 144 for DIR in docutils sandbox web ; do
 145     test -d $DIR || svn checkout $svnurl/$DIR
 146 done
 147 haschanges="`svn up docutils sandbox web | grep -v '^At revision '; true`"
 148
 149 # update __version_details__ string
 150 version_details="snapshot `date -u +%Y-%m-%d`, r`svn info docutils | grep ^Revision: | sed 's/^Revision: //'`"
 151 (echo ",s/^__version_details__ = .*\$/__version_details__ = '$version_details'/";
 152     echo wq) | ed $project/$project/__init__.py 2> /dev/null
 153
 154 # Ensure proper directory permissions are set so that the files can be
 155 # modified by several users.  Changing permissions of files is
 156 # probably not necessary because files can be deleted and re-created.
 157 # Do not change permissions of aux directory to keep it non-public
 158 # (but change permissions for all subdirectories).
 159 #find $basedir -type f -print0 | xargs -0 chmod ug+rw 2> /dev/null || true
 160 find $basedir -name aux -o -type d -print0 | xargs -0 chmod ug+rwxs 2> /dev/null || true
 161
 162 # revert and touch (to avoid updating the web site only because of the
 163 # changed timestamp)
 164 svn -q revert $project/$project/__init__.py
 165 touch $project/$project/__init__.py --date \
 166     "`svn info $project/$project/__init__.py | \
 167     grep 'Last Changed Date:' | sed 's/[^:]*: //'`"
 168
 169 # -------------------- htdocs: --------------------
 170
 171 cd $snapshotdir
 172
 173 function copy_to_htdocsdest() {
 174     find "$@" -type d -name .svn -prune -o \( -type f -o -type l \) -print0 | \
 175         xargs -0 cp --no-dereference --update --parents \
 176             --target-directory=$htdocsdest
 177 }
 178
 179 # update htdocs
 180 copy_to_htdocsdest sandbox
 181 (cd $project; copy_to_htdocsdest *)
 182 (cd web; copy_to_htdocsdest * .[^.]*)
 183
 184 # update HTML docs
 185 cd $htdocsdest/tools
 186
 187 if [ $trace -eq 0 ] ; then
 188     set +o xtrace
 189 fi
 190
 191 for makefile in `find .. -name Makefile.docutils-update` ; do
 192     dir=`dirname $makefile`
 193     ( cd $dir ; make -f Makefile.docutils-update -s )
 194 done
 195
 196 for htmlfile in `find .. -name '*.html'` ; do
 197     dir=`dirname $htmlfile`
 198     base=`basename $htmlfile .html`
 199     txtfile=$dir/$base.txt
 200     if [ "$base" == "standalone_rst_html4strict" ] ; then
 201         # breaks web update
 202         echo "skipped: $dir $base"
 203     elif [ -e $txtfile ] ; then
 204         if [ $unconditional -eq 1 -o $txtfile -nt $htmlfile ] ; then
 205             if [ "${base:0:4}" == "pep-" ] ; then
 206                 test $feedback && echo "$txtfile (PEP)" || true
 207                 python $lib/tools/rstpep2html.py --config=$dir/docutils.conf $txtfile $htmlfile
 208                 haschanges=1
 209             else
 210                 test $feedback && echo $txtfile || true
 211                 python $lib/tools/rst2html.py --config=$dir/docutils.conf $txtfile $htmlfile
 212                 haschanges=1
 213             fi
 214         fi
 215     fi
 216 done
 217
 218 if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 219     set -o xtrace
 220 fi
 221
 222 # -------------------- XML sitemap for search engines: --------------------
 223
 224 cd $htdocsdest
 225
 226 # Update the sitemap only if something has changed because it takes
 227 # very much CPU time.
 228 if test -n "$haschanges"; then
 229     (
 230         echo '<?xml version="1.0" encoding="UTF-8"?>'
 231         echo '<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">'
 232         if [ $trace -eq 0 ] ; then
 233             set +o xtrace
 234         fi
 235         find . -name '.[^.]*' -prune -o -type d -printf '%p/\n' \
 236                 -o \( -type f -o -type l \) -print | \
 237             while read i; do
 238                 # i is the file name.
 239                 if test "$i" == ./; then
 240                     # Homepage.
 241                     i=index.html
 242                     url="$baseurl/"
 243                 elif test "$i" == ./sitemap -o "${i: -1}" == / -a -f "${i}index.html"; then
 244                     # This is a directory and it has an index.html, so we
 245                     # don't need to include it.
 246                     continue
 247                 else
 248                     url="$baseurl${i:1}"
 249                     url="${url// /%20}"
 250                 fi
 251                 lastmod="`date --iso-8601=seconds -u -r "$i"`"
 252                 # Google wants a colon in front of the last two digits.
 253                 lastmod="${lastmod::22}:00"
 254                 if test "${i: -5}" == .html; then
 255                     # HTML files (including the home page) have highest priority.
 256                     priority=1.0
 257                 elif test "${i: -4}" == .txt; then
 258                     # Text files have medium priority.
 259                     priority=0.5
 260                 else
 261                     # Everything else (source files etc.) has low priority.
 262                     priority=0.2
 263                 fi
 264                 echo "<url><loc>$url</loc><lastmod>$lastmod</lastmod><priority>$priority</priority></url>"
 265             done
 266         if [ $trace -eq 1 -o $verbose -eq 1 ] ; then
 267             set -o xtrace
 268         fi
 269         echo '</urlset>'
 270     ) > sitemap
 271     gzip -f sitemap
 272 fi
 273
 274 # -------------------- Push changes to remote server. --------------------
 275
 276 # sourceforge no longer allows shell access, use rsync via ssh
 277 # specify your user in your .ssh/config
 278
 279 cd $htdocsdest
 280 # do not use -a to avoid "failed to set permissions"
 281 # -t preserve modification times. But a new svn checkout has new modtime.
 282 rsync -e ssh -r -t ./ web.sourceforge.net:$remotehtdocs
 283
 284 trap - 0 1 2 3 15
 285 rm -rf $lockdir
 286 test $feedback && echo '...docutils-update done.' || true
 287
 288 # Local Variables:
 289 # indent-tabs-mode: nil
 290 # End: