jobd/update.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4
   5 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
   6 datefmt='+%a, %d %b %Y %T %z'
   7
   8 # darcs fast-export | git fast-import with error handling
   9 git_darcs_fetch() (
  10         LC_ALL="$use_locale"
  11         export LC_ALL
  12         _err1=
  13         _err2=
  14         exec 3>&1
  15         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  16         $(
  17                 exec 4>&3 3>&1 1>&4 4>&-
  18                 {
  19                         _e1=0
  20                         "$cfg_basedir"/bin/darcs-fast-export \
  21                                 --export-marks="$(pwd)/dfe-marks" \
  22                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  23                         echo $_e1 >&3
  24                 } | \
  25                 {
  26                         _e2=0
  27                         git fast-import \
  28                                 --export-marks="$(pwd)/gfi-marks" \
  29                                 --export-pack-edges="$(pwd)/gfi-packs" \
  30                                 --import-marks="$(pwd)/gfi-marks" \
  31                                 --force 3>&- || _e2=$?
  32                         echo $_e2 >&3
  33                 }
  34         )
  35         EOT
  36         exec 3>&-
  37         [ "$_err1" = 0 -a "$_err2" = 0 ]
  38         return $?
  39 )
  40
  41 # bzr fast-export | git fast-import with error handling
  42 git_bzr_fetch() (
  43         LC_ALL="$use_locale"
  44         export LC_ALL
  45         _err1=
  46         _err2=
  47         exec 3>&1
  48         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  49         $(
  50                 exec 4>&3 3>&1 1>&4 4>&-
  51                 {
  52                         _e1=0
  53                         bzr fast-export \
  54                                 --export-marks="$(pwd)/dfe-marks" \
  55                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  56                         echo $_e1 >&3
  57                 } | \
  58                 {
  59                         _e2=0
  60                         git fast-import \
  61                                 --export-marks="$(pwd)/gfi-marks" \
  62                                 --export-pack-edges="$(pwd)/gfi-packs" \
  63                                 --import-marks="$(pwd)/gfi-marks" \
  64                                 --force 3>&- || _e2=$?
  65                         echo $_e2 >&3
  66                 }
  67         )
  68         EOT
  69         exec 3>&-
  70         [ "$_err1" = 0 -a "$_err2" = 0 ]
  71         return $?
  72 )
  73
  74 set -e
  75 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; }
  76 trap 'if [ $? != 0 ]; then echo "update failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
  77
  78 umask 002
  79 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  80
  81 proj="$1"
  82 cd "$cfg_reporoot/$proj.git"
  83
  84 if check_interval lastrefresh $cfg_min_mirror_interval; then
  85         progress "= [$proj] update skip (last at $(config_get lastrefresh))"
  86         exit 0
  87 fi
  88 progress "+ [$proj] update (`date`)"
  89
  90 bang_setup
  91 bang_once=1
  92 bang_action="update"
  93
  94 mail="$(config_get owner || :)"
  95 url="$(config_get baseurl || :)"
  96 case "$url" in *"       "*|*" "*|"")
  97         bang_eval 'echo "Bad mirror URL (\"$url\")"; ! :'
  98         exit 1
  99 esac
 100 bang echo "Mirroring from URL \"$url\""
 101 bang echo ""
 102 statusok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 103 mailaddrs=''
 104 [ "$statusok" = "false" -o -z "$mail" ] || mailaddrs="$mail"
 105 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
 106 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 107
 108 bang_eval "git for-each-ref --format '%(refname)        %(objectname)' >.refs-temp"
 109 bang_eval "LC_ALL=C sort -k1b,1 <.refs-temp >.refs-before"
 110
 111 case "$url" in
 112         svn://* | svn+http://* | svn+https://*)
 113                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; }
 114                 # Update the git svn url to match baseurl but be cognizant of any
 115                 # needed prefix changes.  See the comments in taskd/clone.sh about
 116                 # why we need to put up with a prefix in the first place.
 117                 svnurl="${url#svn+}"
 118                 svnurl="${svnurl%/}"
 119                 svnurlold="$(config_get svnurl || :)"
 120                 if [ "$svnurl" != "$svnurlold" ]; then
 121                         # We better already have an svn-remote.svn.fetch setting
 122                         bang test -n "$(git config --get-all svn-remote.svn.fetch || :)"
 123                         # the only way to truly know what the proper prefix is
 124                         # is to attempt a fresh git-svn init -s on the new url
 125                         rm -rf svn-new-url || :
 126                         # We require svn info to succeed on the URL otherwise it's
 127                         # simply not a valid URL and without using -s on the init it
 128                         # will not otherwise be tested until the fetch
 129                         bang eval 'svn --non-interactive info "$svnurl" > /dev/null'
 130                         bang mkdir svn-new-url
 131                         GIT_DIR=svn-new-url bang git init --bare --quiet
 132                         # We initially use -s for the init which will possibly shorten
 133                         # the URL.  However, the shortening can fail if a password is
 134                         # not required for the longer version but is for the shorter,
 135                         # so try again without -s if the -s version fails.
 136                         cmdstr='git svn init --prefix "" -s "$svnurl" < /dev/null > /dev/null 2>&1 || '
 137                         cmdstr="$cmdstr"'git svn init --prefix "" "$svnurl" < /dev/null > /dev/null 2>&1'
 138                         GIT_DIR=svn-new-url bang eval "$cmdstr"
 139                         gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url || :)"
 140                         gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch || :)"
 141                         gitsvnprefixnew="${gitsvnfetch%%:*}"
 142                         gitsvnsuffixnew="${gitsvnprefixnew##*/}"
 143                         gitsvnprefixnew="${gitsvnprefixnew%$gitsvnsuffixnew}"
 144                         rm -rf svn-new-url || :
 145                         # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
 146                         GIT_DIR=.
 147                         if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
 148                                 # The url has been changed.
 149                                 # We must update the url and replace the prefix on all config items
 150                                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1 || :)"
 151                                 gitsvnprefixold="${gitsvnfetch%%:*}"
 152                                 gitsvnsuffixold="${gitsvnprefixold##*/}"
 153                                 gitsvnprefixold="${gitsvnprefixold%$gitsvnsuffixold}"
 154                                 git config --remove-section 'svn-remote.svnnew' 2>/dev/null || :
 155                                 git config 'svn-remote.svnnew.url' "$gitsvnurl"
 156                                 { git config --get-regexp '^svn-remote\.svn\.' || :; } | \
 157                                 { while read sname sval; do
 158                                         case "$sname" in
 159                                         svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags)
 160                                                 sname="${sname#svn-remote.svn.}"
 161                                                 sval="${sval#$gitsvnprefixold}"
 162                                                 bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
 163                                         esac
 164                                 done; }
 165                                 bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl"
 166                                 bang git config --remove-section svn-remote.svn
 167                                 bang git config --rename-section svn-remote.svnnew svn-remote.svn
 168                         fi
 169                         bang config_set svnurl "$svnurl"
 170                 fi
 171                 # remove any stale *.lock files greater than 1 hour old in case
 172                 # git-svn was killed on the last update because it took too long
 173                 find svn -type f -name '*.lock' -mmin +60 -print0 2>/dev/null | xargs -0 rm -f
 174                 GIT_DIR=. bang git svn fetch --quiet < /dev/null
 175                 # git svn does not preserve group permissions in the svn subdirectory
 176                 chmod -R ug+rw,o+r svn
 177                 # git svn also leaves behind ref turds that end with @nnn
 178                 # We get rid of them now
 179                 git show-ref | \
 180                 { while read sha1 ref; do
 181                         case "$ref" in
 182                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 183                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 184                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 185                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 186                                 git update-ref -d "$ref"
 187                         esac
 188                 done; }
 189                 ;;
 190         darcs://*)
 191                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; }
 192                 httpurl="http://${url#darcs://}"
 193                 # remove any stale lock files greater than 1 hour old in case
 194                 # darcs_fast_export was killed on the last update because it took too long
 195                 find *.darcs -maxdepth 2 -type f -name 'lock' -mmin +60 -print0 2>/dev/null | xargs -0 rm -f
 196                 bang git_darcs_fetch "$httpurl"
 197                 ;;
 198         bzr://*)
 199                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; }
 200                 bzrurl="${url#bzr://}"
 201                 bang git_bzr_fetch "$bzrurl"
 202                 ;;
 203         hg+http://* | hg+https://*)
 204                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
 205                 # We just remove hg+ here, so hg+http://... becomes http://...
 206                 hgurl="${url#hg+}"
 207                 # Fetch any new updates
 208                 bang hg -R "$(pwd)/repo.hg" pull
 209                 # Do the fast-export | fast-import
 210                 bang git_hg_fetch
 211                 ;;
 212         *)
 213                 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
 214                 GIT_SSL_NO_VERIFY=1 bang git remote update
 215                 GIT_SSL_NO_VERIFY=1 bang git remote prune origin
 216                 ;;
 217 esac
 218
 219 # The objects subdirectories permissions must be updated now.
 220 # In the case of a dumb http clone, the permissions will not be correct
 221 # (missing group write) despite the core.sharedrepository=1 setting!
 222 # The objects themselves seem to have the correct permissions.
 223 # This problem appears to have been fixed in the most recent git versions.
 224 perms=g+w
 225 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 226 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 227
 228 bang git update-server-info
 229 bang config_set lastrefresh "$(date "$datefmt")"
 230
 231 # Look at which refs changed and trigger ref-change for these
 232 bang_eval "git for-each-ref --format '%(refname)        %(objectname)' >.refs-temp"
 233 bang_eval "LC_ALL=C sort -k1b,1 <.refs-temp >.refs-after"
 234 sockpath="$cfg_chroot/etc/taskd.socket"
 235 if ! cmp -s .refs-before .refs-after; then
 236         bang config_set lastreceive "$(date '+%a, %d %b %Y %T %z')"
 237         if [ -S "$sockpath" ]; then
 238                 join -j 1 .refs-before .refs-after |
 239                         while read ref old new; do
 240                                 [ "$old" != "$new" ] || continue
 241                                 echo "ref-change %$proj% $proj $old $new $ref" | nc_openbsd -w 1 -U "$sockpath"
 242                         done
 243                 join -j 1 -v 1 .refs-before .refs-after |
 244                         while read ref old; do
 245                                 echo "ref-change %$proj% $proj $old 0000000000000000000000000000000000000000 $ref" | \
 246                                 nc_openbsd -w 1 -U "$sockpath"
 247                         done
 248                 join -j 1 -v 2 .refs-before .refs-after |
 249                         while read ref new; do
 250                                 echo "ref-change %$proj% $proj 0000000000000000000000000000000000000000 $new $ref" | \
 251                                 nc_openbsd -w 1 -U "$sockpath"
 252                         done
 253         fi
 254         bang config_set lastchange "$(date '+%a, %d %b %Y %T %z')"
 255         bang_eval "git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 256                 --count=1 refs/heads > info/lastactivity"
 257 fi
 258
 259 rm -f .refs-before .refs-after .refs-temp
 260
 261 if [ -e .banged ]; then
 262         [ -z "$mailaddrs" ] ||
 263         {
 264                 echo "$proj update succeeded - failure recovery"
 265                 echo "this status message may be disabled on the project admin page"
 266         } | mail -s "[$cfg_name] $proj update succeeded" "$mailaddrs" || :
 267         rm -f .banged .bangagain
 268 fi
 269
 270 progress "- [$proj] update (`date`)"