taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() {
   9         _err1=
  10         _err2=
  11         exec 3>&1
  12         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  13         $(
  14                 exec 4>&3 3>&1 1>&4 4>&-
  15                 {
  16                         _e1=0
  17                         "$cfg_basedir"/bin/darcs-fast-export \
  18                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  19                         echo $_e1 >&3
  20                 } | \
  21                 {
  22                         _e2=0
  23                         git fast-import \
  24                                 --export-marks="$(pwd)/gfi-marks" \
  25                                 --export-pack-edges="$(pwd)/gfi-packs" \
  26                                 --force 3>&- || _e2=$?
  27                         echo $_e2 >&3
  28                 }
  29         )
  30         EOT
  31         exec 3>&-
  32         [ "$_err1" = 0 -a "$_err2" = 0 ]
  33         return $?
  34 }
  35
  36 # bzr fast-export | git fast-import with error handling
  37 git_bzr_fetch() {
  38         _err1=
  39         _err2=
  40         exec 3>&1
  41         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  42         $(
  43                 exec 4>&3 3>&1 1>&4 4>&-
  44                 {
  45                         _e1=0
  46                         bzr fast-export \
  47                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  48                         echo $_e1 >&3
  49                 } | \
  50                 {
  51                         _e2=0
  52                         git fast-import \
  53                                 --export-marks="$(pwd)/gfi-marks" \
  54                                 --export-pack-edges="$(pwd)/gfi-packs" \
  55                                 --force 3>&- || _e2=$?
  56                         echo $_e2 >&3
  57                 }
  58         )
  59         EOT
  60         exec 3>&-
  61         [ "$_err1" = 0 -a "$_err2" = 0 ]
  62         return $?
  63 }
  64
  65 set -e
  66
  67 umask 002
  68 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  69
  70 projdir="$1"
  71 proj="${projdir%.git}"
  72
  73 cd "$cfg_reporoot/$projdir"
  74 trap "echo '@OVER@'; touch .clone_failed" EXIT
  75 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
  76 url="$(config_get baseurl || :)"
  77 case "$url" in *"       "*|*" "*|"")
  78         echo "Bad clone URL (\"$url\")"
  79         exit 1
  80 esac
  81
  82 if [ "$cfg_project_owners" = "source" ]; then
  83         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
  84 fi
  85
  86 mailaddrs="$(config_get owner || :)"
  87 [ -z "$cfg_admin" ] || \
  88 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
  89
  90 # Initial mirror
  91 echo "Initiating mirroring..."
  92 case "$url" in
  93         svn://* | svn+http://* | svn+https://*)
  94                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
  95                 # We just remove svn+ here, so svn+http://... becomes http://...
  96                 # We also remove a trailing '/' to match what git-svn will do
  97                 svnurl="${url#svn+}"
  98                 svnurl="${svnurl%/}"
  99                 # We require svn info to succeed on the URL otherwise it's
 100                 # simply not a valid URL and without using -s on the init it
 101                 # will not otherwise be tested until the fetch
 102                 svn --non-interactive info "$svnurl" > /dev/null
 103                 # We initially use -s for the init which will possibly shorten
 104                 # the URL.  However, the shortening can fail if a password is
 105                 # not required for the longer version but is for the shorter,
 106                 # so try again without -s if the -s version fails.
 107                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 108                 # create a .git subdirectory!
 109                 GIT_DIR=. git svn init --prefix "" -s "$svnurl" < /dev/null || \
 110                 GIT_DIR=. git svn init --prefix "" "$svnurl" < /dev/null
 111                 # We need to remember this url so we can detect changes because
 112                 # ever so "helpful" git-svn may shorten it!
 113                 config_set svnurl "$svnurl"
 114                 # At this point, since we asked for a standard layout (-s) git-svn
 115                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 116                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 117                 # config items.  We could avoid this by not using the '-s' option
 118                 # but then we might not get all the history.  If, for example, we
 119                 # are cloning an http://svn.example.com/repos/public repository that
 120                 # early in its history moved trunk => public/trunk we would miss that
 121                 # earlier history without allowing the funky shorten+prefix behavior.
 122                 # So we read back the svn-remote.svn.fetch configuration and compute
 123                 # the prefix.  This way we are sure to get the correct prefix.
 124                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 125                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 126                 gitsvnprefix="${gitsvnfetch%%:*}"
 127                 gitsvnsuffix="${gitsvnprefix##*/}"
 128                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 129                 # Ask git-svn to store everything in the normal non-remote
 130                 # locations being careful to use the correct prefix
 131                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 132                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 133                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 134                 # look for additional non-standard directories to fetch
 135                 # check for standard layout at the same time
 136                 foundstd=
 137                 foundfile=
 138                 { svn --non-interactive ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 139                 { while read file; do case $file in
 140                         # skip the already-handled standard ones and any with a space or tab
 141                         *'      '*|*' '*) :;;
 142                         trunk/|branches/|tags/) foundstd=1;;
 143                         # only fetch extra directories from the $svnurl root (not any files)
 144                         *?/) git config --add svn-remote.svn.fetch \
 145                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 146                         *?) foundfile=1;;
 147                 esac; done
 148                 # if files found and no standard directories present use a simpler layout
 149                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 150                         git config --unset svn-remote.svn.branches
 151                         git config --unset svn-remote.svn.tags
 152                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 153                 fi; }
 154                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 155                 GIT_DIR=. git svn fetch --quiet < /dev/null
 156                 # git svn does not preserve group permissions in the svn subdirectory
 157                 chmod -R ug+rw,o+r svn
 158                 # git svn also leaves behind ref turds that end with @nnn
 159                 # We get rid of them now
 160                 git show-ref | \
 161                 { while read sha1 ref; do
 162                         case "$ref" in
 163                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 164                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 165                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 166                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 167                                 git update-ref -d "$ref"
 168                         esac
 169                 done; }
 170                 ;;
 171         darcs://*)
 172                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 173                 httpurl="http://${url#darcs://}"
 174                 # Remove any left-over .darcs dirs from a previous failed attempt
 175                 rm -rf *.darcs
 176                 # Remove any left-over export files from a previous failed attempt
 177                 rm -f dfe-marks gfi-marks gfi-packs
 178                 git_darcs_fetch "$httpurl"
 179                 ;;
 180         bzr://*)
 181                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 182                 # we just remove bzr:// here, a typical bzr url is just
 183                 # "lp:foo"
 184                 bzrurl="${url#bzr://}"
 185                 # Remove any left-over export files from a previous failed attempt
 186                 rm -f bfe-marks gfi-marks gfi-packs
 187                 git_bzr_fetch "$bzrurl"
 188                 ;;
 189         hg+http://* | hg+https://*)
 190                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 191                 # We just remove hg+ here, so hg+http://... becomes http://...
 192                 hgurl="${url#hg+}"
 193                 # Remove any left-over repo.hg dir from a previous failed attempt
 194                 rm -rf repo.hg
 195                 # Remove any left-over export files from a previous failed attempt
 196                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 197                 # Perform the initial hg clone
 198                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 199                 # Do the fast-export | fast-import
 200                 git_hg_fetch
 201                 ;;
 202         *)
 203                 git remote rm origin >/dev/null 2>&1 || :
 204                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 205                 # since we support older Git versions, just toss the warning
 206                 git remote add --mirror origin "$url" 2>/dev/null
 207                 GIT_SSL_NO_VERIFY=1 git remote update
 208                 GIT_SSL_NO_VERIFY=1 git remote prune origin
 209                 ;;
 210 esac
 211
 212 # The objects subdirectories permissions must be updated now.
 213 # In the case of a dumb http clone, the permissions will not be correct
 214 # (missing group write) despite the core.sharedrepository=1 setting!
 215 # The objects themselves seem to have the correct permissions.
 216 # This problem appears to have been fixed in the most recent git versions.
 217 perms=g+w
 218 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 219 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 220
 221 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 222 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 223 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 224 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 225         --count=1 refs/heads > info/lastactivity
 226 [ -s info/lastactivity ] || rm -f info/lastactivity
 227
 228 # The rest
 229 echo "Final touches..."
 230 git update-server-info
 231 trap "" EXIT
 232
 233 sizenote=
 234 ! is_gfi_mirror ||
 235 sizenote="
 236 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 237        size may be somewhat larger than necessary.  This will be corrected
 238        shortly.  If you intend to clone this repository you may want to
 239        wait up to 1 hour before doing so in order to receive the more
 240        compact final size.
 241 "
 242 [ -z "$mailaddrs" ] ||
 243 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 244 Congratulations! The clone of project $proj just completed.
 245
 246         * Source URL: $url
 247         * GitWeb interface: $cfg_gitweburl/$projdir
 248         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 249 $sizenote
 250 Have a lot of fun.
 251 EOT
 252
 253 echo "Mirroring finished successfuly!"
 254 # In case this is a re-mirror, lastgc could have been set already so clear it now
 255 git config --unset gitweb.lastgc || :
 256 rm .clone_in_progress
 257 echo "$sizenote@OVER@"