taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() (
   9         set_utf8_locale
  10         _err1=
  11         _err2=
  12         exec 3>&1
  13         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  14         $(
  15                 exec 4>&3 3>&1 1>&4 4>&-
  16                 {
  17                         _e1=0
  18                         "$cfg_basedir"/bin/darcs-fast-export \
  19                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  20                         echo $_e1 >&3
  21                 } | \
  22                 {
  23                         _e2=0
  24                         git fast-import \
  25                                 --export-marks="$(pwd)/gfi-marks" \
  26                                 --export-pack-edges="$(pwd)/gfi-packs" \
  27                                 --force 3>&- || _e2=$?
  28                         echo $_e2 >&3
  29                 }
  30         )
  31         EOT
  32         exec 3>&-
  33         [ "$_err1" = 0 -a "$_err2" = 0 ]
  34         return $?
  35 )
  36
  37 # bzr fast-export | git fast-import with error handling
  38 git_bzr_fetch() (
  39         set_utf8_locale
  40         BZR_LOG=/dev/null
  41         export BZR_LOG
  42         _err1=
  43         _err2=
  44         exec 3>&1
  45         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  46         $(
  47                 exec 4>&3 3>&1 1>&4 4>&-
  48                 {
  49                         _e1=0
  50                         bzr fast-export --plain \
  51                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  52                         echo $_e1 >&3
  53                 } | \
  54                 {
  55                         _e2=0
  56                         git fast-import \
  57                                 --export-marks="$(pwd)/gfi-marks" \
  58                                 --export-pack-edges="$(pwd)/gfi-packs" \
  59                                 --force 3>&- || _e2=$?
  60                         echo $_e2 >&3
  61                 }
  62         )
  63         EOT
  64         exec 3>&-
  65         [ "$_err1" = 0 -a "$_err2" = 0 ]
  66         return $?
  67 )
  68
  69 send_clone_failed() {
  70         trap "" EXIT
  71         # We must now close the .clonelog file that is open on stdout and stderr
  72         exec >/dev/null 2>&1
  73         failaddrs="$(config_get owner || :)"
  74         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  75         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  76         [ -z "$failaddrs" ] || \
  77         {
  78               cat <<EOT
  79 Condolences. The clone of project $proj just failed.
  80
  81         * Source URL: $url
  82         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
  83
  84 The project settings link may be used to adjust the settings
  85 and restart the clone in order to try the clone again.
  86 EOT
  87                 if [ -f .clonelog -a -r .clonelog ]; then
  88                         echo ""
  89                         echo "Log follows:"
  90                         echo ""
  91                         loglines=$(wc -l <.clonelog)
  92                         if [ $loglines -le 203 ]; then
  93                                 cat .clonelog
  94                         else
  95                                 head -n 100 .clonelog
  96                                 echo ""
  97                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
  98                                 echo ""
  99                                 tail -n 100 .clonelog
 100                         fi
 101                 fi
 102         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 103 }
 104
 105 set -e
 106
 107 umask 002
 108 [ "$cfg_permission_control" != "Hooks" ] || umask 000
 109
 110 proj="${1%.git}"
 111 cd "$cfg_reporoot/$proj.git"
 112 bang_reset
 113
 114 # Any pre-existing FETCH_HEAD from a previous clone failed or not is garbage
 115 rm -f FETCH_HEAD
 116
 117 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 118 echo "Project: $proj"
 119 echo ""
 120 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 121 url="$(config_get baseurl || :)"
 122 case "$url" in *"       "*|*" "*|"")
 123         echo "Bad mirror URL (\"$url\")"
 124         exit 1
 125 esac
 126 echo "Mirroring from URL \"$url\""
 127 echo ""
 128
 129 if [ "$cfg_project_owners" = "source" ]; then
 130         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 131 fi
 132
 133 mailaddrs="$(config_get owner || :)"
 134 [ -z "$cfg_admin" ] || \
 135 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 136
 137 # Initial mirror
 138 echo "Initiating mirroring..."
 139 headref=
 140 showheadwarn=
 141 warnempty=
 142 case "$url" in
 143         svn://* | svn+http://* | svn+https://* | svn+file://*)
 144                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 145                 # We just remove svn+ here, so svn+http://... becomes http://...
 146                 # We also remove a trailing '/' to match what git-svn will do
 147                 svnurl="${url#svn+}"
 148                 svnurl="${svnurl%/}"
 149                 # Remove any left-over svn-remote.svn config from a previous failed attempt
 150                 git config --remove-section svn-remote.svn 2>/dev/null || :
 151                 # Remove any left-over svn dir from a previous failed attempt
 152                 rm -rf svn
 153                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 154                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 155                 GIT_ASKPASS_PASSWORD=anonsvn
 156                 export GIT_ASKPASS_PASSWORD
 157                 # We require svn info to succeed on the URL otherwise it's
 158                 # simply not a valid URL and without using -s on the init it
 159                 # will not otherwise be tested until the fetch
 160                 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
 161                 # We initially use -s for the init which will possibly shorten
 162                 # the URL.  However, the shortening can fail if a password is
 163                 # not required for the longer version but is for the shorter,
 164                 # so try again without -s if the -s version fails.
 165                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 166                 # create a .git subdirectory!
 167                 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
 168                 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
 169                 # We need to remember this url so we can detect changes because
 170                 # ever so "helpful" git-svn may shorten it!
 171                 config_set svnurl "$svnurl"
 172                 # At this point, since we asked for a standard layout (-s) git-svn
 173                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 174                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 175                 # config items.  We could avoid this by not using the '-s' option
 176                 # but then we might not get all the history.  If, for example, we
 177                 # are cloning an http://svn.example.com/repos/public repository that
 178                 # early in its history moved trunk => public/trunk we would miss that
 179                 # earlier history without allowing the funky shorten+prefix behavior.
 180                 # So we read back the svn-remote.svn.fetch configuration and compute
 181                 # the prefix.  This way we are sure to get the correct prefix.
 182                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 183                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 184                 gitsvnprefix="${gitsvnfetch%%:*}"
 185                 gitsvnsuffix="${gitsvnprefix##*/}"
 186                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 187                 # Ask git-svn to store everything in the normal non-remote
 188                 # locations being careful to use the correct prefix
 189                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 190                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 191                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 192                 # look for additional non-standard directories to fetch
 193                 # check for standard layout at the same time
 194                 foundstd=
 195                 foundfile=
 196                 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 197                 { while read file; do case $file in
 198                         # skip the already-handled standard ones and any with a space or tab
 199                         *'      '*|*' '*) :;;
 200                         trunk/|branches/|tags/) foundstd=1;;
 201                         # only fetch extra directories from the $svnurl root (not any files)
 202                         *?/) git config --add svn-remote.svn.fetch \
 203                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 204                         *?) foundfile=1;;
 205                 esac; done
 206                 # if files found and no standard directories present use a simpler layout
 207                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 208                         git config --unset svn-remote.svn.branches
 209                         git config --unset svn-remote.svn.tags
 210                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 211                 fi; }
 212                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 213                 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 214                 # git svn does not preserve group permissions in the svn subdirectory
 215                 chmod -R ug+rw,o+r svn
 216                 # git svn also leaves behind ref turds that end with @nnn
 217                 # We get rid of them now
 218                 git for-each-ref --format='%(objectname) %(refname)' | \
 219                 { while read sha1 ref; do
 220                         case "$ref" in
 221                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 222                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 223                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 224                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 225                                 git update-ref -d "$ref"
 226                         esac
 227                 done; }
 228                 unset GIT_ASKPASS_PASSWORD
 229                 ;;
 230         darcs://*)
 231                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 232                 httpurl="http://${url#darcs://}"
 233                 # Remove any left-over .darcs dirs from a previous failed attempt
 234                 rm -rf *.darcs
 235                 # Remove any left-over export files from a previous failed attempt
 236                 rm -f dfe-marks gfi-marks gfi-packs
 237                 git_darcs_fetch "$httpurl"
 238                 ;;
 239         bzr://*)
 240                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 241                 # we just remove bzr:// here, a typical bzr url is just
 242                 # "lp:foo"
 243                 bzrurl="${url#bzr://}"
 244                 # Remove any left-over export files from a previous failed attempt
 245                 rm -f bfe-marks gfi-marks gfi-packs
 246                 git_bzr_fetch "$bzrurl"
 247                 ;;
 248         hg+http://* | hg+https://*)
 249                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 250                 # We just remove hg+ here, so hg+http://... becomes http://...
 251                 hgurl="${url#hg+}"
 252                 # Remove any left-over repo.hg dir from a previous failed attempt
 253                 rm -rf repo.hg
 254                 # Remove any left-over export files from a previous failed attempt
 255                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 256                 # Perform the initial hg clone
 257                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 258                 # Do the fast-export | fast-import
 259                 git_hg_fetch
 260                 ;;
 261         *)
 262                 git remote rm origin >/dev/null 2>&1 || :
 263                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 264                 # since we support older Git versions, just toss the warning
 265                 git remote add --mirror origin "$url" 2>/dev/null
 266                 # Set the correct HEAD symref by using ls-remote first
 267                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 268                 {
 269                         # Since everything was redirected, on failure there'd be no output,
 270                         # so let's make some failure output
 271                         cat .pkts-temp
 272                         echo ""
 273                         echo "git ls-remote \"$url\" failed"
 274                         exit 1
 275                 }
 276                 # Compensate for git() {} side effects
 277                 unset GIT_TRACE_PACKET
 278                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 279                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 280                 # out of the packet trace data.
 281                 if [ -s .refs-temp ]; then
 282                         # Nothing to do unless the remote repository has at least 1 ref
 283                         # See if we got a HEAD ref
 284                         head="$(grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | awk '{print $1}')"
 285                         # If the remote has HEAD set to a symbolic ref that does not exist
 286                         # then we will not receive a HEAD ref in the ls-remote output
 287                         headref=
 288                         showheadwarn=
 289                         symrefcap=
 290                         if [ -n "$head" ]; then
 291                                 symrefcap="$(sed -ne <.pkts-temp \
 292                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 293                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 294                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 295                                 # matches HEAD otherwise take the first refs/heads/... match
 296                                 matchcnt=0
 297                                 while read ref; do
 298                                         [ -n "$ref" ] || continue
 299                                         matchcnt=$(( $matchcnt + 1 ))
 300                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 301                                                 headref="$ref"
 302                                         fi
 303                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 304                                 done <<-EOT
 305                                 $(grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 306                                         awk '{print $2}')
 307                                 EOT
 308                                 # Warn if there was more than one match and $symrefcap is empty
 309                                 # or $symrefcap is not the same as $headref since our choice might
 310                                 # differ from the source repository's HEAD
 311                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 312                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 313                                         showheadwarn=1
 314                                 fi
 315                         fi
 316                         if [ -z "$headref" ]; then
 317                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 318                                 # if it exists otherwise take the first refs/heads/...
 319                                 # We do not support having a detached HEAD.
 320                                 # We always warn now because we will be setting HEAD differently
 321                                 # than the source repository had HEAD set
 322                                 showheadwarn=1
 323                                 while read ref; do
 324                                         [ -n "$ref" ] || continue
 325                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 326                                                 headref="$ref"
 327                                         fi
 328                                         [ "$headref" = "refs/heads/master" ] && break
 329                                 done <<-EOT
 330                                 $(grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 331                                         awk '{print $2}')
 332                                 EOT
 333                         fi
 334                         # If we STILL do not have a HEAD ref (perhaps the source repository
 335                         # contains only tags) then use refs/heads/master.  It will be invalid
 336                         # but is no worse than we used to do by default and we'll warn about
 337                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 338                         [ -n "$headref" ] || headref="refs/heads/master"
 339                         git symbolic-ref HEAD "$headref"
 340                         GIT_SSL_NO_VERIFY=1 git remote update --prune
 341                 else
 342                         warnempty=1
 343                         git symbolic-ref HEAD "refs/heads/master"
 344                 fi
 345                 rm -f .refs-temp .pkts-temp
 346                 ;;
 347 esac
 348
 349 # The objects subdirectories permissions must be updated now.
 350 # In the case of a dumb http clone, the permissions will not be correct
 351 # (missing group write) despite the core.sharedrepository=1 setting!
 352 # The objects themselves seem to have the correct permissions.
 353 # This problem appears to have been fixed in the most recent git versions.
 354 perms=g+w
 355 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 356 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 357
 358 # We may have just cloned a lot of refs and they will all be
 359 # individual files at this point.  Let's pack them now so we
 360 # can have better performance right from the start.
 361 git pack-refs --all
 362
 363 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 364 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 365 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 366 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 367         --count=1 refs/heads > info/lastactivity || :
 368 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 369
 370 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 371 rm -f FETCH_HEAD
 372
 373 # The rest
 374 echo "Final touches..."
 375 git update-server-info
 376 trap "" EXIT
 377
 378 emptynote=
 379 [ -z "$warnempty" ] ||
 380 emptynote="
 381 WARNING:  You have mirrored an empty repository.
 382 "
 383 headnote=
 384 [ -n "$showheadwarn" -a -n "$headref" ] &&
 385 headnote="
 386 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 387        Use the \"Project settings\" link to choose a different HEAD symref.
 388 "
 389 sizenote=
 390 ! is_gfi_mirror ||
 391 sizenote="
 392 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 393        size may be somewhat larger than necessary.  This will be corrected
 394        shortly.  If you intend to clone this repository you may want to
 395        wait up to 1 hour before doing so in order to receive the more
 396        compact final size.
 397 "
 398 [ -z "$mailaddrs" ] ||
 399 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 400 Congratulations! The clone of project $proj just completed.
 401
 402         * Source URL: $url
 403         * GitWeb interface: $cfg_gitweburl/$proj.git
 404         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 405 $emptynote$headnote$sizenote
 406 Have a lot of fun.
 407 EOT
 408
 409 echo "Mirroring finished successfuly!"
 410 # In case this is a re-mirror, lastgc could have been set already so clear it now
 411 git config --unset gitweb.lastgc || :
 412 rm .clone_in_progress
 413 echo "$sizenote@OVER@"