jobd/update.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4
   5 set -e
   6
   7 if [ $# -ne 1 ]; then
   8         echo "Usage: update.sh projname" >&2
   9         exit 1
  10 fi
  11
  12 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
  13 datefmt='+%a, %d %b %Y %T %z'
  14
  15 # darcs fast-export | git fast-import with error handling
  16 git_darcs_fetch() (
  17         set_utf8_locale
  18         _err1=
  19         _err2=
  20         exec 3>&1
  21         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  22         $(
  23                 exec 4>&3 3>&1 1>&4 4>&-
  24                 {
  25                         _e1=0
  26                         "$cfg_basedir"/bin/darcs-fast-export \
  27                                 --export-marks="$(pwd)/dfe-marks" \
  28                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  29                         echo $_e1 >&3
  30                 } | \
  31                 {
  32                         _e2=0
  33                         git fast-import \
  34                                 --export-marks="$(pwd)/gfi-marks" \
  35                                 --export-pack-edges="$(pwd)/gfi-packs" \
  36                                 --import-marks="$(pwd)/gfi-marks" \
  37                                 --force 3>&- || _e2=$?
  38                         echo $_e2 >&3
  39                 }
  40         )
  41         EOT
  42         exec 3>&-
  43         [ "$_err1" = 0 -a "$_err2" = 0 ]
  44         return $?
  45 )
  46
  47 # bzr fast-export | git fast-import with error handling
  48 git_bzr_fetch() (
  49         set_utf8_locale
  50         BZR_LOG=/dev/null
  51         export BZR_LOG
  52         _err1=
  53         _err2=
  54         exec 3>&1
  55         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  56         $(
  57                 exec 4>&3 3>&1 1>&4 4>&-
  58                 {
  59                         _e1=0
  60                         bzr fast-export --plain \
  61                                 --export-marks="$(pwd)/dfe-marks" \
  62                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  63                         echo $_e1 >&3
  64                 } | \
  65                 {
  66                         _e2=0
  67                         git fast-import \
  68                                 --export-marks="$(pwd)/gfi-marks" \
  69                                 --export-pack-edges="$(pwd)/gfi-packs" \
  70                                 --import-marks="$(pwd)/gfi-marks" \
  71                                 --force 3>&- || _e2=$?
  72                         echo $_e2 >&3
  73                 }
  74         )
  75         EOT
  76         exec 3>&-
  77         [ "$_err1" = 0 -a "$_err2" = 0 ]
  78         return $?
  79 )
  80
  81 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; }
  82
  83 umask 002
  84 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  85
  86 proj="${1%.git}"
  87 cd "$cfg_reporoot/$proj.git"
  88
  89 trap 'if [ $? != 0 ]; then echo "update failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
  90 trap 'exit 130' INT
  91 trap 'exit 143' TERM
  92
  93 if check_interval lastrefresh $cfg_min_mirror_interval; then
  94         progress "= [$proj] update skip (last at $(config_get lastrefresh))"
  95         exit 0
  96 fi
  97 progress "+ [$proj] update (`date`)"
  98
  99 # Any pre-existing FETCH_HEAD from a previous fetch failed or not is garbage
 100 rm -f FETCH_HEAD
 101
 102 # A previous failed update attempt can leave a huge tmp_pack_XXXXXX file behind.
 103 # Since no pushes are allowed to mirrors, we know that any such files that exist
 104 # at this point in time are garbage and can be safely deleted, we do not even
 105 # need to check how old they are.  A tmp_idx_XXXXXX file is also created during
 106 # the later stages of the fetch process, so we kill any of those as well.
 107 find objects/pack -maxdepth 1 -type f -name "tmp_pack_?*" -print0 | xargs -0 rm -f
 108 find objects/pack -maxdepth 1 -type f -name "tmp_idx_?*" -print0 | xargs -0 rm -f
 109
 110 bang_setup
 111 bang_once=1
 112 bang_action="update"
 113
 114 bang echo "Project: $proj"
 115 bang echo ""
 116 mail="$(config_get owner || :)"
 117 url="$(config_get baseurl || :)"
 118 case "$url" in *"       "*|*" "*|"")
 119         bang_eval 'echo "Bad mirror URL (\"$url\")"; ! :'
 120         exit 1
 121 esac
 122 bang echo "Mirroring from URL \"$url\""
 123 bang echo ""
 124 statusok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 125 mailaddrs=''
 126 [ "$statusok" = "false" -o -z "$mail" ] || mailaddrs="$mail"
 127 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
 128 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 129
 130 bang_eval "git for-each-ref --format '%(refname)        %(objectname)' >.refs-temp"
 131 bang_eval "LC_ALL=C sort -k1b,1 <.refs-temp >.refs-before"
 132 ! [ -e .delaygc ] || > .allowgc || :
 133
 134 case "$url" in
 135         svn://* | svn+http://* | svn+https://* | svn+file://*)
 136                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; }
 137                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 138                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 139                 GIT_ASKPASS_PASSWORD=anonsvn
 140                 export GIT_ASKPASS_PASSWORD
 141                 # Update the git svn url to match baseurl but be cognizant of any
 142                 # needed prefix changes.  See the comments in taskd/clone.sh about
 143                 # why we need to put up with a prefix in the first place.
 144                 svnurl="${url#svn+}"
 145                 svnurl="${svnurl%/}"
 146                 svnurlold="$(config_get svnurl || :)"
 147                 if [ "$svnurl" != "$svnurlold" ]; then
 148                         # We better already have an svn-remote.svn.fetch setting
 149                         bang test -n "$(git config --get-all svn-remote.svn.fetch || :)"
 150                         # the only way to truly know what the proper prefix is
 151                         # is to attempt a fresh git-svn init -s on the new url
 152                         rm -rf svn-new-url || :
 153                         # We require svn info to succeed on the URL otherwise it's
 154                         # simply not a valid URL and without using -s on the init it
 155                         # will not otherwise be tested until the fetch
 156                         bang eval 'svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null'
 157                         bang mkdir svn-new-url
 158                         GIT_DIR=svn-new-url bang git init --bare --quiet
 159                         # We initially use -s for the init which will possibly shorten
 160                         # the URL.  However, the shortening can fail if a password is
 161                         # not required for the longer version but is for the shorter,
 162                         # so try again without -s if the -s version fails.
 163                         cmdstr='git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null > /dev/null 2>&1 || '
 164                         cmdstr="$cmdstr"'git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null > /dev/null 2>&1'
 165                         GIT_DIR=svn-new-url bang eval "$cmdstr"
 166                         gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url || :)"
 167                         gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch || :)"
 168                         gitsvnprefixnew="${gitsvnfetch%%:*}"
 169                         gitsvnsuffixnew="${gitsvnprefixnew##*/}"
 170                         gitsvnprefixnew="${gitsvnprefixnew%$gitsvnsuffixnew}"
 171                         rm -rf svn-new-url || :
 172                         # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
 173                         GIT_DIR=.
 174                         if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
 175                                 # The url has been changed.
 176                                 # We must update the url and replace the prefix on all config items
 177                                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1 || :)"
 178                                 gitsvnprefixold="${gitsvnfetch%%:*}"
 179                                 gitsvnsuffixold="${gitsvnprefixold##*/}"
 180                                 gitsvnprefixold="${gitsvnprefixold%$gitsvnsuffixold}"
 181                                 git config --remove-section 'svn-remote.svnnew' 2>/dev/null || :
 182                                 git config 'svn-remote.svnnew.url' "$gitsvnurl"
 183                                 { git config --get-regexp '^svn-remote\.svn\.' || :; } | \
 184                                 { while read sname sval; do
 185                                         case "$sname" in
 186                                         svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags)
 187                                                 sname="${sname#svn-remote.svn.}"
 188                                                 sval="${sval#$gitsvnprefixold}"
 189                                                 bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
 190                                         esac
 191                                 done; }
 192                                 bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl"
 193                                 bang git config --remove-section svn-remote.svn
 194                                 bang git config --rename-section svn-remote.svnnew svn-remote.svn
 195                         fi
 196                         bang config_set svnurl "$svnurl"
 197                 fi
 198                 # remove any stale *.lock files greater than 1 hour old in case
 199                 # git-svn was killed on the last update because it took too long
 200                 find svn -type f -name '*.lock' -mmin +60 -print0 2>/dev/null | xargs -0 rm -f
 201                 GIT_DIR=. bang git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 202                 # git svn does not preserve group permissions in the svn subdirectory
 203                 chmod -R ug+rw,o+r svn
 204                 # git svn also leaves behind ref turds that end with @nnn
 205                 # We get rid of them now
 206                 git for-each-ref --format='%(objectname) %(refname)' | \
 207                 { while read sha1 ref; do
 208                         case "$ref" in
 209                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 210                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 211                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 212                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 213                                 git update-ref -d "$ref"
 214                         esac
 215                 done; }
 216                 unset GIT_ASKPASS_PASSWORD
 217                 ;;
 218         darcs://*)
 219                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; }
 220                 httpurl="http://${url#darcs://}"
 221                 # remove any stale lock files greater than 1 hour old in case
 222                 # darcs_fast_export was killed on the last update because it took too long
 223                 find *.darcs -maxdepth 2 -type f -name 'lock' -mmin +60 -print0 2>/dev/null | xargs -0 rm -f
 224                 bang git_darcs_fetch "$httpurl"
 225                 ;;
 226         bzr://*)
 227                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; }
 228                 bzrurl="${url#bzr://}"
 229                 bang git_bzr_fetch "$bzrurl"
 230                 ;;
 231         hg+http://* | hg+https://*)
 232                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
 233                 # We just remove hg+ here, so hg+http://... becomes http://...
 234                 hgurl="${url#hg+}"
 235                 # Fetch any new updates
 236                 bang hg -R "$(pwd)/repo.hg" pull
 237                 # Do the fast-export | fast-import
 238                 bang git_hg_fetch
 239                 ;;
 240         *)
 241                 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
 242                 pruneopt=--prune
 243                 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 244                 GIT_SSL_NO_VERIFY=1 bang git remote update $pruneopt
 245                 ;;
 246 esac
 247
 248 # The objects subdirectories permissions must be updated now.
 249 # In the case of a dumb http clone, the permissions will not be correct
 250 # (missing group write) despite the core.sharedrepository=1 setting!
 251 # The objects themselves seem to have the correct permissions.
 252 # This problem appears to have been fixed in the most recent git versions.
 253 perms=g+w
 254 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 255 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 256
 257 bang git update-server-info
 258
 259 # We maintain the last refresh date in two places deliberately
 260 # so that it's available as part of the config data and also
 261 # as a standalone file timestamp that can be accessed without git.
 262 bang config_set lastrefresh "$(date "$datefmt")"
 263 { >.last_refresh; } 2>/dev/null || :
 264
 265 # Look at which refs changed and trigger ref-change for these
 266 bang_eval "git for-each-ref --format '%(refname)        %(objectname)' >.refs-temp"
 267 bang_eval "LC_ALL=C sort -k1b,1 <.refs-temp >.refs-after"
 268 sockpath="$cfg_chroot/etc/taskd.socket"
 269 if ! cmp -s .refs-before .refs-after; then
 270         bang config_set lastreceive "$(date '+%a, %d %b %Y %T %z')"
 271         if [ -S "$sockpath" ]; then
 272                 LC_ALL=C join -j 1 .refs-before .refs-after |
 273                         while read ref old new; do
 274                                 [ "$old" != "$new" ] || continue
 275                                 echo "ref-change %$proj% $proj $old $new $ref" | nc_openbsd -w 1 -U "$sockpath"
 276                         done
 277                 LC_ALL=C join -j 1 -v 1 .refs-before .refs-after |
 278                         while read ref old; do
 279                                 echo "ref-change %$proj% $proj $old 0000000000000000000000000000000000000000 $ref" | \
 280                                 nc_openbsd -w 1 -U "$sockpath"
 281                         done
 282                 LC_ALL=C join -j 1 -v 2 .refs-before .refs-after |
 283                         while read ref new; do
 284                                 echo "ref-change %$proj% $proj 0000000000000000000000000000000000000000 $new $ref" | \
 285                                 nc_openbsd -w 1 -U "$sockpath"
 286                         done
 287         fi
 288         bang config_set lastchange "$(date '+%a, %d %b %Y %T %z')"
 289         bang_eval "git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 290                 --count=1 refs/heads > info/lastactivity"
 291         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 292         rm -f .delaygc .allowgc
 293 fi
 294
 295 rm -f .refs-before .refs-after .refs-temp FETCH_HEAD
 296
 297 if is_banged; then
 298         [ -z "$mailaddrs" ] ||
 299         {
 300                 echo "$proj update succeeded - failure recovery"
 301                 echo "this status message may be disabled on the project admin page"
 302         } | mail -s "[$cfg_name] $proj update succeeded" "$mailaddrs" || :
 303         bang_reset
 304 fi
 305
 306 progress "- [$proj] update (`date`)"