taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() (
   9         LC_ALL="$use_locale"
  10         export LC_ALL
  11         _err1=
  12         _err2=
  13         exec 3>&1
  14         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  15         $(
  16                 exec 4>&3 3>&1 1>&4 4>&-
  17                 {
  18                         _e1=0
  19                         "$cfg_basedir"/bin/darcs-fast-export \
  20                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  21                         echo $_e1 >&3
  22                 } | \
  23                 {
  24                         _e2=0
  25                         git fast-import \
  26                                 --export-marks="$(pwd)/gfi-marks" \
  27                                 --export-pack-edges="$(pwd)/gfi-packs" \
  28                                 --force 3>&- || _e2=$?
  29                         echo $_e2 >&3
  30                 }
  31         )
  32         EOT
  33         exec 3>&-
  34         [ "$_err1" = 0 -a "$_err2" = 0 ]
  35         return $?
  36 )
  37
  38 # bzr fast-export | git fast-import with error handling
  39 git_bzr_fetch() (
  40         LC_ALL="$use_locale"
  41         export LC_ALL
  42         _err1=
  43         _err2=
  44         exec 3>&1
  45         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  46         $(
  47                 exec 4>&3 3>&1 1>&4 4>&-
  48                 {
  49                         _e1=0
  50                         bzr fast-export \
  51                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  52                         echo $_e1 >&3
  53                 } | \
  54                 {
  55                         _e2=0
  56                         git fast-import \
  57                                 --export-marks="$(pwd)/gfi-marks" \
  58                                 --export-pack-edges="$(pwd)/gfi-packs" \
  59                                 --force 3>&- || _e2=$?
  60                         echo $_e2 >&3
  61                 }
  62         )
  63         EOT
  64         exec 3>&-
  65         [ "$_err1" = 0 -a "$_err2" = 0 ]
  66         return $?
  67 )
  68
  69 set -e
  70
  71 umask 002
  72 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  73
  74 projdir="$1"
  75 proj="${projdir%.git}"
  76
  77 cd "$cfg_reporoot/$projdir"
  78 trap "echo '@OVER@'; touch .clone_failed" EXIT
  79 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
  80 url="$(config_get baseurl || :)"
  81 case "$url" in *"       "*|*" "*|"")
  82         echo "Bad mirror URL (\"$url\")"
  83         exit 1
  84 esac
  85 echo "Mirroring from URL \"$url\""
  86 echo ""
  87
  88 if [ "$cfg_project_owners" = "source" ]; then
  89         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
  90 fi
  91
  92 mailaddrs="$(config_get owner || :)"
  93 [ -z "$cfg_admin" ] || \
  94 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
  95
  96 # Initial mirror
  97 echo "Initiating mirroring..."
  98 case "$url" in
  99         svn://* | svn+http://* | svn+https://*)
 100                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 101                 # We just remove svn+ here, so svn+http://... becomes http://...
 102                 # We also remove a trailing '/' to match what git-svn will do
 103                 svnurl="${url#svn+}"
 104                 svnurl="${svnurl%/}"
 105                 # We require svn info to succeed on the URL otherwise it's
 106                 # simply not a valid URL and without using -s on the init it
 107                 # will not otherwise be tested until the fetch
 108                 svn --non-interactive info "$svnurl" > /dev/null
 109                 # We initially use -s for the init which will possibly shorten
 110                 # the URL.  However, the shortening can fail if a password is
 111                 # not required for the longer version but is for the shorter,
 112                 # so try again without -s if the -s version fails.
 113                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 114                 # create a .git subdirectory!
 115                 GIT_DIR=. git svn init --prefix "" -s "$svnurl" < /dev/null || \
 116                 GIT_DIR=. git svn init --prefix "" "$svnurl" < /dev/null
 117                 # We need to remember this url so we can detect changes because
 118                 # ever so "helpful" git-svn may shorten it!
 119                 config_set svnurl "$svnurl"
 120                 # At this point, since we asked for a standard layout (-s) git-svn
 121                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 122                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 123                 # config items.  We could avoid this by not using the '-s' option
 124                 # but then we might not get all the history.  If, for example, we
 125                 # are cloning an http://svn.example.com/repos/public repository that
 126                 # early in its history moved trunk => public/trunk we would miss that
 127                 # earlier history without allowing the funky shorten+prefix behavior.
 128                 # So we read back the svn-remote.svn.fetch configuration and compute
 129                 # the prefix.  This way we are sure to get the correct prefix.
 130                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 131                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 132                 gitsvnprefix="${gitsvnfetch%%:*}"
 133                 gitsvnsuffix="${gitsvnprefix##*/}"
 134                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 135                 # Ask git-svn to store everything in the normal non-remote
 136                 # locations being careful to use the correct prefix
 137                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 138                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 139                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 140                 # look for additional non-standard directories to fetch
 141                 # check for standard layout at the same time
 142                 foundstd=
 143                 foundfile=
 144                 { svn --non-interactive ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 145                 { while read file; do case $file in
 146                         # skip the already-handled standard ones and any with a space or tab
 147                         *'      '*|*' '*) :;;
 148                         trunk/|branches/|tags/) foundstd=1;;
 149                         # only fetch extra directories from the $svnurl root (not any files)
 150                         *?/) git config --add svn-remote.svn.fetch \
 151                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 152                         *?) foundfile=1;;
 153                 esac; done
 154                 # if files found and no standard directories present use a simpler layout
 155                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 156                         git config --unset svn-remote.svn.branches
 157                         git config --unset svn-remote.svn.tags
 158                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 159                 fi; }
 160                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 161                 GIT_DIR=. git svn fetch --quiet < /dev/null
 162                 # git svn does not preserve group permissions in the svn subdirectory
 163                 chmod -R ug+rw,o+r svn
 164                 # git svn also leaves behind ref turds that end with @nnn
 165                 # We get rid of them now
 166                 git show-ref | \
 167                 { while read sha1 ref; do
 168                         case "$ref" in
 169                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 170                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 171                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 172                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 173                                 git update-ref -d "$ref"
 174                         esac
 175                 done; }
 176                 ;;
 177         darcs://*)
 178                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 179                 httpurl="http://${url#darcs://}"
 180                 # Remove any left-over .darcs dirs from a previous failed attempt
 181                 rm -rf *.darcs
 182                 # Remove any left-over export files from a previous failed attempt
 183                 rm -f dfe-marks gfi-marks gfi-packs
 184                 git_darcs_fetch "$httpurl"
 185                 ;;
 186         bzr://*)
 187                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 188                 # we just remove bzr:// here, a typical bzr url is just
 189                 # "lp:foo"
 190                 bzrurl="${url#bzr://}"
 191                 # Remove any left-over export files from a previous failed attempt
 192                 rm -f bfe-marks gfi-marks gfi-packs
 193                 git_bzr_fetch "$bzrurl"
 194                 ;;
 195         hg+http://* | hg+https://*)
 196                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 197                 # We just remove hg+ here, so hg+http://... becomes http://...
 198                 hgurl="${url#hg+}"
 199                 # Remove any left-over repo.hg dir from a previous failed attempt
 200                 rm -rf repo.hg
 201                 # Remove any left-over export files from a previous failed attempt
 202                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 203                 # Perform the initial hg clone
 204                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 205                 # Do the fast-export | fast-import
 206                 git_hg_fetch
 207                 ;;
 208         *)
 209                 git remote rm origin >/dev/null 2>&1 || :
 210                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 211                 # since we support older Git versions, just toss the warning
 212                 git remote add --mirror origin "$url" 2>/dev/null
 213                 GIT_SSL_NO_VERIFY=1 git remote update
 214                 GIT_SSL_NO_VERIFY=1 git remote prune origin
 215                 ;;
 216 esac
 217
 218 # The objects subdirectories permissions must be updated now.
 219 # In the case of a dumb http clone, the permissions will not be correct
 220 # (missing group write) despite the core.sharedrepository=1 setting!
 221 # The objects themselves seem to have the correct permissions.
 222 # This problem appears to have been fixed in the most recent git versions.
 223 perms=g+w
 224 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 225 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 226
 227 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 228 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 229 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 230 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 231         --count=1 refs/heads > info/lastactivity
 232 [ -s info/lastactivity ] || rm -f info/lastactivity
 233
 234 # The rest
 235 echo "Final touches..."
 236 git update-server-info
 237 trap "" EXIT
 238
 239 sizenote=
 240 ! is_gfi_mirror ||
 241 sizenote="
 242 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 243        size may be somewhat larger than necessary.  This will be corrected
 244        shortly.  If you intend to clone this repository you may want to
 245        wait up to 1 hour before doing so in order to receive the more
 246        compact final size.
 247 "
 248 [ -z "$mailaddrs" ] ||
 249 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 250 Congratulations! The clone of project $proj just completed.
 251
 252         * Source URL: $url
 253         * GitWeb interface: $cfg_gitweburl/$projdir
 254         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 255 $sizenote
 256 Have a lot of fun.
 257 EOT
 258
 259 echo "Mirroring finished successfuly!"
 260 # In case this is a re-mirror, lastgc could have been set already so clear it now
 261 git config --unset gitweb.lastgc || :
 262 rm .clone_in_progress
 263 echo "$sizenote@OVER@"