taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() {
   9         _err1=
  10         _err2=
  11         exec 3>&1
  12         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  13         $(
  14                 exec 4>&3 3>&1 1>&4 4>&-
  15                 {
  16                         _e1=0
  17                         "$cfg_basedir"/bin/darcs-fast-export \
  18                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  19                         echo $_e1 >&3
  20                 } | \
  21                 {
  22                         _e2=0
  23                         git fast-import \
  24                                 --export-marks="$(pwd)/gfi-marks" \
  25                                 --export-pack-edges="$(pwd)/gfi-packs" \
  26                                 --force 3>&- || _e2=$?
  27                         echo $_e2 >&3
  28                 }
  29         )
  30         EOT
  31         exec 3>&-
  32         [ "$_err1" = 0 -a "$_err2" = 0 ]
  33         return $?
  34 }
  35
  36 # bzr fast-export | git fast-import with error handling
  37 git_bzr_fetch() {
  38         _err1=
  39         _err2=
  40         exec 3>&1
  41         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  42         $(
  43                 exec 4>&3 3>&1 1>&4 4>&-
  44                 {
  45                         _e1=0
  46                         bzr fast-export \
  47                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  48                         echo $_e1 >&3
  49                 } | \
  50                 {
  51                         _e2=0
  52                         git fast-import \
  53                                 --export-marks="$(pwd)/gfi-marks" \
  54                                 --export-pack-edges="$(pwd)/gfi-packs" \
  55                                 --force 3>&- || _e2=$?
  56                         echo $_e2 >&3
  57                 }
  58         )
  59         EOT
  60         exec 3>&-
  61         [ "$_err1" = 0 -a "$_err2" = 0 ]
  62         return $?
  63 }
  64
  65 set -e
  66
  67 umask 002
  68 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  69
  70 projdir="$1"
  71 proj="${projdir%.git}"
  72
  73 cd "$cfg_reporoot/$projdir"
  74 trap "echo '@OVER@'; touch .clone_failed" EXIT
  75 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
  76 url="$(config_get baseurl || :)"
  77 case "$url" in *"       "*|*" "*|"")
  78         echo "Bad mirror URL (\"$url\")"
  79         exit 1
  80 esac
  81 echo "Mirroring from URL \"$url\""
  82 echo ""
  83
  84 if [ "$cfg_project_owners" = "source" ]; then
  85         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
  86 fi
  87
  88 mailaddrs="$(config_get owner || :)"
  89 [ -z "$cfg_admin" ] || \
  90 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
  91
  92 # Initial mirror
  93 echo "Initiating mirroring..."
  94 case "$url" in
  95         svn://* | svn+http://* | svn+https://*)
  96                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
  97                 # We just remove svn+ here, so svn+http://... becomes http://...
  98                 # We also remove a trailing '/' to match what git-svn will do
  99                 svnurl="${url#svn+}"
 100                 svnurl="${svnurl%/}"
 101                 # We require svn info to succeed on the URL otherwise it's
 102                 # simply not a valid URL and without using -s on the init it
 103                 # will not otherwise be tested until the fetch
 104                 svn --non-interactive info "$svnurl" > /dev/null
 105                 # We initially use -s for the init which will possibly shorten
 106                 # the URL.  However, the shortening can fail if a password is
 107                 # not required for the longer version but is for the shorter,
 108                 # so try again without -s if the -s version fails.
 109                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 110                 # create a .git subdirectory!
 111                 GIT_DIR=. git svn init --prefix "" -s "$svnurl" < /dev/null || \
 112                 GIT_DIR=. git svn init --prefix "" "$svnurl" < /dev/null
 113                 # We need to remember this url so we can detect changes because
 114                 # ever so "helpful" git-svn may shorten it!
 115                 config_set svnurl "$svnurl"
 116                 # At this point, since we asked for a standard layout (-s) git-svn
 117                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 118                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 119                 # config items.  We could avoid this by not using the '-s' option
 120                 # but then we might not get all the history.  If, for example, we
 121                 # are cloning an http://svn.example.com/repos/public repository that
 122                 # early in its history moved trunk => public/trunk we would miss that
 123                 # earlier history without allowing the funky shorten+prefix behavior.
 124                 # So we read back the svn-remote.svn.fetch configuration and compute
 125                 # the prefix.  This way we are sure to get the correct prefix.
 126                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 127                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 128                 gitsvnprefix="${gitsvnfetch%%:*}"
 129                 gitsvnsuffix="${gitsvnprefix##*/}"
 130                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 131                 # Ask git-svn to store everything in the normal non-remote
 132                 # locations being careful to use the correct prefix
 133                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 134                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 135                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 136                 # look for additional non-standard directories to fetch
 137                 # check for standard layout at the same time
 138                 foundstd=
 139                 foundfile=
 140                 { svn --non-interactive ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 141                 { while read file; do case $file in
 142                         # skip the already-handled standard ones and any with a space or tab
 143                         *'      '*|*' '*) :;;
 144                         trunk/|branches/|tags/) foundstd=1;;
 145                         # only fetch extra directories from the $svnurl root (not any files)
 146                         *?/) git config --add svn-remote.svn.fetch \
 147                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 148                         *?) foundfile=1;;
 149                 esac; done
 150                 # if files found and no standard directories present use a simpler layout
 151                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 152                         git config --unset svn-remote.svn.branches
 153                         git config --unset svn-remote.svn.tags
 154                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 155                 fi; }
 156                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 157                 GIT_DIR=. git svn fetch --quiet < /dev/null
 158                 # git svn does not preserve group permissions in the svn subdirectory
 159                 chmod -R ug+rw,o+r svn
 160                 # git svn also leaves behind ref turds that end with @nnn
 161                 # We get rid of them now
 162                 git show-ref | \
 163                 { while read sha1 ref; do
 164                         case "$ref" in
 165                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 166                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 167                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 168                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 169                                 git update-ref -d "$ref"
 170                         esac
 171                 done; }
 172                 ;;
 173         darcs://*)
 174                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 175                 httpurl="http://${url#darcs://}"
 176                 # Remove any left-over .darcs dirs from a previous failed attempt
 177                 rm -rf *.darcs
 178                 # Remove any left-over export files from a previous failed attempt
 179                 rm -f dfe-marks gfi-marks gfi-packs
 180                 git_darcs_fetch "$httpurl"
 181                 ;;
 182         bzr://*)
 183                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 184                 # we just remove bzr:// here, a typical bzr url is just
 185                 # "lp:foo"
 186                 bzrurl="${url#bzr://}"
 187                 # Remove any left-over export files from a previous failed attempt
 188                 rm -f bfe-marks gfi-marks gfi-packs
 189                 git_bzr_fetch "$bzrurl"
 190                 ;;
 191         hg+http://* | hg+https://*)
 192                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 193                 # We just remove hg+ here, so hg+http://... becomes http://...
 194                 hgurl="${url#hg+}"
 195                 # Remove any left-over repo.hg dir from a previous failed attempt
 196                 rm -rf repo.hg
 197                 # Remove any left-over export files from a previous failed attempt
 198                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 199                 # Perform the initial hg clone
 200                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 201                 # Do the fast-export | fast-import
 202                 git_hg_fetch
 203                 ;;
 204         *)
 205                 git remote rm origin >/dev/null 2>&1 || :
 206                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 207                 # since we support older Git versions, just toss the warning
 208                 git remote add --mirror origin "$url" 2>/dev/null
 209                 GIT_SSL_NO_VERIFY=1 git remote update
 210                 GIT_SSL_NO_VERIFY=1 git remote prune origin
 211                 ;;
 212 esac
 213
 214 # The objects subdirectories permissions must be updated now.
 215 # In the case of a dumb http clone, the permissions will not be correct
 216 # (missing group write) despite the core.sharedrepository=1 setting!
 217 # The objects themselves seem to have the correct permissions.
 218 # This problem appears to have been fixed in the most recent git versions.
 219 perms=g+w
 220 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 221 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 222
 223 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 224 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 225 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 226 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 227         --count=1 refs/heads > info/lastactivity
 228 [ -s info/lastactivity ] || rm -f info/lastactivity
 229
 230 # The rest
 231 echo "Final touches..."
 232 git update-server-info
 233 trap "" EXIT
 234
 235 sizenote=
 236 ! is_gfi_mirror ||
 237 sizenote="
 238 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 239        size may be somewhat larger than necessary.  This will be corrected
 240        shortly.  If you intend to clone this repository you may want to
 241        wait up to 1 hour before doing so in order to receive the more
 242        compact final size.
 243 "
 244 [ -z "$mailaddrs" ] ||
 245 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 246 Congratulations! The clone of project $proj just completed.
 247
 248         * Source URL: $url
 249         * GitWeb interface: $cfg_gitweburl/$projdir
 250         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 251 $sizenote
 252 Have a lot of fun.
 253 EOT
 254
 255 echo "Mirroring finished successfuly!"
 256 # In case this is a re-mirror, lastgc could have been set already so clear it now
 257 git config --unset gitweb.lastgc || :
 258 rm .clone_in_progress
 259 echo "$sizenote@OVER@"