taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() {
   9         _err1=
  10         _err2=
  11         exec 3>&1
  12         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  13         $(
  14                 exec 4>&3 3>&1 1>&4 4>&-
  15                 {
  16                         _e1=0
  17                         "$cfg_basedir"/bin/darcs-fast-export \
  18                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  19                         echo $_e1 >&3
  20                 } | \
  21                 {
  22                         _e2=0
  23                         git fast-import \
  24                                 --export-marks="$(pwd)/gfi-marks" \
  25                                 --export-pack-edges="$(pwd)/gfi-packs" \
  26                                 --force 3>&- || _e2=$?
  27                         echo $_e2 >&3
  28                 }
  29         )
  30         EOT
  31         exec 3>&-
  32         [ "$_err1" = 0 -a "$_err2" = 0 ]
  33         return $?
  34 }
  35
  36 # bzr fast-export | git fast-import with error handling
  37 git_bzr_fetch() {
  38         _err1=
  39         _err2=
  40         exec 3>&1
  41         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  42         $(
  43                 exec 4>&3 3>&1 1>&4 4>&-
  44                 {
  45                         _e1=0
  46                         bzr fast-export \
  47                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  48                         echo $_e1 >&3
  49                 } | \
  50                 {
  51                         _e2=0
  52                         git fast-import \
  53                                 --export-marks="$(pwd)/gfi-marks" \
  54                                 --export-pack-edges="$(pwd)/gfi-packs" \
  55                                 --force 3>&- || _e2=$?
  56                         echo $_e2 >&3
  57                 }
  58         )
  59         EOT
  60         exec 3>&-
  61         [ "$_err1" = 0 -a "$_err2" = 0 ]
  62         return $?
  63 }
  64
  65 set -e
  66
  67 umask 002
  68 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  69
  70 projdir="$1"
  71 proj="${projdir%.git}"
  72
  73 cd "$cfg_reporoot/$projdir"
  74 trap "echo '@OVER@'; touch .clone_failed" EXIT
  75 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
  76 url="$(config_get baseurl)"
  77
  78 if [ "$cfg_project_owners" = "source" ]; then
  79         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
  80 fi
  81
  82 mailaddrs="$(config_get owner || :)"
  83 [ -z "$cfg_admin" ] || \
  84 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
  85
  86 # Initial mirror
  87 echo "Initiating mirroring..."
  88 case "$url" in
  89         svn://* | svn+http://* | svn+https://*)
  90                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
  91                 # We just remove svn+ here, so svn+http://... becomes http://...
  92                 # We also remove a trailing '/' to match what git-svn will do
  93                 svnurl="${url#svn+}"
  94                 svnurl="${svnurl%/}"
  95                 # We require svn info to succeed on the URL otherwise it's
  96                 # simply not a valid URL and without using -s on the init it
  97                 # will not otherwise be tested until the fetch
  98                 svn --non-interactive info "$svnurl" > /dev/null
  99                 # We initially use -s for the init which will possibly shorten
 100                 # the URL.  However, the shortening can fail if a password is
 101                 # not required for the longer version but is for the shorter,
 102                 # so try again without -s if the -s version fails.
 103                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 104                 # create a .git subdirectory!
 105                 GIT_DIR=. git svn init --prefix= -s "$svnurl" < /dev/null || \
 106                 GIT_DIR=. git svn init --prefix= "$svnurl" < /dev/null
 107                 # We need to remember this url so we can detect changes because
 108                 # ever so "helpful" git-svn may shorten it!
 109                 config_set svnurl "$svnurl"
 110                 # At this point, since we asked for a standard layout (-s) git-svn
 111                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 112                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 113                 # config items.  We could avoid this by not using the '-s' option
 114                 # but then we might not get all the history.  If, for example, we
 115                 # are cloning an http://svn.example.com/repos/public repository that
 116                 # early in its history moved trunk => public/trunk we would miss that
 117                 # earlier history without allowing the funky shorten+prefix behavior.
 118                 # So we read back the svn-remote.svn.fetch configuration and compute
 119                 # the prefix.  This way we are sure to get the correct prefix.
 120                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 121                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 122                 gitsvnprefix="${gitsvnfetch%%:*}"
 123                 gitsvnsuffix="${gitsvnprefix##*/}"
 124                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 125                 # Ask git-svn to store everything in the normal non-remote
 126                 # locations being careful to use the correct prefix
 127                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 128                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 129                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 130                 # look for additional non-standard directories to fetch
 131                 # check for standard layout at the same time
 132                 foundstd=
 133                 foundfile=
 134                 { svn --non-interactive ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 135                 { while read file; do case $file in
 136                         # skip the already-handled standard ones and any with a space or tab
 137                         *'      '*|*' '*) :;;
 138                         trunk/|branches/|tags/) foundstd=1;;
 139                         # only fetch extra directories from the $svnurl root (not any files)
 140                         *?/) git config --add svn-remote.svn.fetch \
 141                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 142                         *?) foundfile=1;;
 143                 esac; done
 144                 # if files found and no standard directories present use a simpler layout
 145                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 146                         git config --unset svn-remote.svn.branches
 147                         git config --unset svn-remote.svn.tags
 148                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 149                 fi; }
 150                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 151                 GIT_DIR=. git svn fetch --quiet < /dev/null
 152                 # git svn does not preserve group permissions in the svn subdirectory
 153                 chmod -R ug+rw,o+r svn
 154                 # git svn also leaves behind ref turds that end with @nnn
 155                 # We get rid of them now
 156                 git show-ref | \
 157                 { while read sha1 ref; do
 158                         case "$ref" in
 159                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 160                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 161                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 162                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 163                                 git update-ref -d "$ref"
 164                         esac
 165                 done; }
 166                 ;;
 167         darcs://*)
 168                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 169                 httpurl="http://${url#darcs://}"
 170                 # Remove any left-over .darcs dirs from a previous failed attempt
 171                 rm -rf *.darcs
 172                 # Remove any left-over export files from a previous failed attempt
 173                 rm -f dfe-marks gfi-marks gfi-packs
 174                 git_darcs_fetch "$httpurl"
 175                 ;;
 176         bzr://*)
 177                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 178                 # we just remove bzr:// here, a typical bzr url is just
 179                 # "lp:foo"
 180                 bzrurl="${url#bzr://}"
 181                 # Remove any left-over export files from a previous failed attempt
 182                 rm -f bfe-marks gfi-marks gfi-packs
 183                 git_bzr_fetch "$bzrurl"
 184                 ;;
 185         hg+http://* | hg+https://*)
 186                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 187                 # We just remove hg+ here, so hg+http://... becomes http://...
 188                 hgurl="${url#hg+}"
 189                 # Remove any left-over repo.hg dir from a previous failed attempt
 190                 rm -rf repo.hg
 191                 # Remove any left-over export files from a previous failed attempt
 192                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 193                 # Perform the initial hg clone
 194                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 195                 # Do the fast-export | fast-import
 196                 git_hg_fetch
 197                 ;;
 198         *)
 199                 git remote rm origin >/dev/null 2>&1 || :
 200                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 201                 # since we support older Git versions, just toss the warning
 202                 git remote add --mirror origin "$url" 2>/dev/null
 203                 GIT_SSL_NO_VERIFY=1 git remote update
 204                 GIT_SSL_NO_VERIFY=1 git remote prune origin
 205                 ;;
 206 esac
 207
 208 # The objects subdirectories permissions must be updated now.
 209 # In the case of a dumb http clone, the permissions will not be correct
 210 # (missing group write) despite the core.sharedrepository=1 setting!
 211 # The objects themselves seem to have the correct permissions.
 212 # This problem appears to have been fixed in the most recent git versions.
 213 perms=g+w
 214 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 215 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 216
 217 # Initialize gitweb.lastchange and info/lastactivity
 218 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 219 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 220         --count=1 refs/heads > info/lastactivity
 221 [ -s info/lastactivity ] || rm -f info/lastactivity
 222
 223 # The rest
 224 echo "Final touches..."
 225 git update-server-info
 226 trap "" EXIT
 227
 228 sizenote=
 229 ! is_gfi_mirror ||
 230 sizenote="
 231 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 232        size may be somewhat larger than necessary.  This will be corrected
 233        shortly.  If you intend to clone this repository you may want to
 234        wait up to 1 hour before doing so in order to receive the more
 235        compact final size.
 236 "
 237 [ -z "$mailaddrs" ] ||
 238 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 239 Congratulations! The clone of project $proj just completed.
 240
 241         * Source URL: $url
 242         * GitWeb interface: $cfg_gitweburl/$projdir
 243         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 244 $sizenote
 245 Have a lot of fun.
 246 EOT
 247
 248 echo "Mirroring finished successfuly!"
 249 # In case this is a re-mirror, lastgc could have been set already so clear it now
 250 git config --unset gitweb.lastgc || :
 251 rm .clone_in_progress
 252 echo "$sizenote@OVER@"