taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 set -e
   8
   9 umask 002
  10 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  11 clean_git_env
  12
  13 # darcs fast-export | git fast-import with error handling
  14 git_darcs_fetch() (
  15         set_utf8_locale
  16         _err1=
  17         _err2=
  18         exec 3>&1
  19         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  20         $(
  21                 exec 4>&3 3>&1 1>&4 4>&-
  22                 {
  23                         _e1=0
  24                         "$cfg_basedir"/bin/darcs-fast-export \
  25                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  26                         echo $_e1 >&3
  27                 } | \
  28                 {
  29                         _e2=0
  30                         git fast-import \
  31                                 --export-marks="$(pwd)/gfi-marks" \
  32                                 --export-pack-edges="$(pwd)/gfi-packs" \
  33                                 --force 3>&- || _e2=$?
  34                         echo $_e2 >&3
  35                 }
  36         )
  37         EOT
  38         exec 3>&-
  39         [ "$_err1" = 0 -a "$_err2" = 0 ]
  40         return $?
  41 )
  42
  43 # bzr fast-export | git fast-import with error handling
  44 git_bzr_fetch() (
  45         set_utf8_locale
  46         BZR_LOG=/dev/null
  47         export BZR_LOG
  48         _err1=
  49         _err2=
  50         exec 3>&1
  51         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  52         $(
  53                 exec 4>&3 3>&1 1>&4 4>&-
  54                 {
  55                         _e1=0
  56                         bzr fast-export --plain \
  57                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  58                         echo $_e1 >&3
  59                 } | \
  60                 {
  61                         _e2=0
  62                         git fast-import \
  63                                 --export-marks="$(pwd)/gfi-marks" \
  64                                 --export-pack-edges="$(pwd)/gfi-packs" \
  65                                 --force 3>&- || _e2=$?
  66                         echo $_e2 >&3
  67                 }
  68         )
  69         EOT
  70         exec 3>&-
  71         [ "$_err1" = 0 -a "$_err2" = 0 ]
  72         return $?
  73 )
  74
  75 send_clone_failed() {
  76         trap "" EXIT
  77         # We must now close the .clonelog file that is open on stdout and stderr
  78         exec >/dev/null 2>&1
  79         failaddrs="$(config_get owner || :)"
  80         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  81         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  82         [ -z "$failaddrs" ] || \
  83         {
  84               cat <<EOT
  85 Condolences. The clone of project $proj just failed.
  86
  87         * Source URL: $url
  88         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
  89
  90 The project settings link may be used to adjust the settings
  91 and restart the clone in order to try the clone again.
  92 EOT
  93                 if [ -f .clonelog -a -r .clonelog ]; then
  94                         echo ""
  95                         echo "Log follows:"
  96                         echo ""
  97                         loglines=$(LC_ALL=C wc -l <.clonelog)
  98                         if [ $loglines -le 203 ]; then
  99                                 cat .clonelog
 100                         else
 101                                 head -n 100 .clonelog
 102                                 echo ""
 103                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 104                                 echo ""
 105                                 tail -n 100 .clonelog
 106                         fi
 107                 fi
 108         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 109 }
 110
 111 # removes all leftovers from a previous failed clone attempt
 112 cleanup_failed_clone() {
 113
 114         # Remove any left-over svn-remote.svn or remote.origin config
 115         git config --remove-section svn-remote.svn 2>/dev/null || :
 116         git config --remove-section remote.origin 2>/dev/null || :
 117
 118         # If there is a remote-template.origin section, pre-seed the
 119         # remote.origin section with its contents
 120         git config --get-regexp '^remote-template\.origin\..' |
 121         while read name value; do
 122                 if [ -n "$name" -a -n "$value" ]; then
 123                         git config "remote${name#remote-template}" "$value"
 124                 fi
 125         done
 126
 127         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 128         # now garbage to be removed
 129         rm -f FETCH_HEAD
 130
 131         # Remove any left-over svn dir from a previous failed attempt
 132         rm -rf svn
 133
 134         # Remove any left-over .darcs dirs from a previous failed attempt
 135         rm -rf *.darcs
 136
 137         # Remove any left-over repo.hg dir from a previous failed attempt
 138         rm -rf repo.hg
 139
 140         # Remove any left-over import/export/temp files from a previous failed attempt
 141         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 142                 gfi-marks gfi-packs .pkts-temp .refs-temp
 143
 144         # We want a gc right after the clone, so re-enable that just in case.
 145         # There's a potential race where we could add it and gc.sh could remove
 146         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 147         [ -e .delaygc ] || >.delaygc
 148         git config --unset gitweb.lastgc 2>/dev/null || :
 149
 150         # Remove all pre-existing refs
 151         rm -f packed-refs
 152         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 153
 154         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 155         git symbolic-ref HEAD refs/heads/master
 156
 157         # HEAD is no longer "ok"
 158         git config --unset girocco.headok 2>/dev/null || :
 159
 160         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 161         # will get rid of any extras.  Also, if we're recloning the same thing, any
 162         # preexisting packs/loose objects containing what we're recloning will only
 163         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 164 }
 165
 166 proj="${1%.git}"
 167 cd "$cfg_reporoot/$proj.git"
 168 bang_reset
 169
 170 ! [ -e .delaygc ] || >.allowgc || :
 171
 172 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 173 echo "Project: $proj"
 174 echo ""
 175 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 176 url="$(config_get baseurl || :)"
 177 case "$url" in *"       "*|*" "*|"")
 178         echo "Bad mirror URL (\"$url\")"
 179         exit 1
 180 esac
 181
 182 cleanup_failed_clone
 183
 184 # Record original mirror type for use by update.sh
 185 mirror_type="$(get_url_mirror_type "$url")"
 186 git config girocco.mirrortype "$mirror_type"
 187
 188 echo "Mirroring from URL \"$url\""
 189 echo ""
 190
 191 if [ "$cfg_project_owners" = "source" ]; then
 192         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 193 fi
 194
 195 mailaddrs="$(config_get owner || :)"
 196 [ -z "$cfg_admin" ] || \
 197 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 198
 199 # Initial mirror
 200 echo "Initiating mirroring..."
 201 headref=
 202 showheadwarn=
 203 warnempty=
 204 case "$url" in
 205         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 206                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 207                 # We just remove svn+ here, so svn+http://... becomes http://...
 208                 # We also remove a trailing '/' to match what git-svn will do
 209                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 210                 svnurl="${svnurl%/}"
 211                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 212                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 213                 GIT_ASKPASS_PASSWORD=anonsvn
 214                 export GIT_ASKPASS_PASSWORD
 215                 # We require svn info to succeed on the URL otherwise it's
 216                 # simply not a valid URL and without using -s on the init it
 217                 # will not otherwise be tested until the fetch
 218                 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
 219                 # We initially use -s for the init which will possibly shorten
 220                 # the URL.  However, the shortening can fail if a password is
 221                 # not required for the longer version but is for the shorter,
 222                 # so try again without -s if the -s version fails.
 223                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 224                 # create a .git subdirectory!
 225                 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
 226                 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
 227                 # We need to remember this url so we can detect changes because
 228                 # ever so "helpful" git-svn may shorten it!
 229                 config_set svnurl "$svnurl"
 230                 # At this point, since we asked for a standard layout (-s) git-svn
 231                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 232                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 233                 # config items.  We could avoid this by not using the '-s' option
 234                 # but then we might not get all the history.  If, for example, we
 235                 # are cloning an http://svn.example.com/repos/public repository that
 236                 # early in its history moved trunk => public/trunk we would miss that
 237                 # earlier history without allowing the funky shorten+prefix behavior.
 238                 # So we read back the svn-remote.svn.fetch configuration and compute
 239                 # the prefix.  This way we are sure to get the correct prefix.
 240                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 241                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 242                 gitsvnprefix="${gitsvnfetch%%:*}"
 243                 gitsvnsuffix="${gitsvnprefix##*/}"
 244                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 245                 # Ask git-svn to store everything in the normal non-remote
 246                 # locations being careful to use the correct prefix
 247                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 248                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 249                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 250                 # look for additional non-standard directories to fetch
 251                 # check for standard layout at the same time
 252                 foundstd=
 253                 foundfile=
 254                 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 255                 { while read file; do case $file in
 256                         # skip the already-handled standard ones and any with a space or tab
 257                         *'      '*|*' '*) :;;
 258                         trunk/|branches/|tags/) foundstd=1;;
 259                         # only fetch extra directories from the $svnurl root (not any files)
 260                         *?/) git config --add svn-remote.svn.fetch \
 261                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 262                         *?) foundfile=1;;
 263                 esac; done
 264                 # if files found and no standard directories present use a simpler layout
 265                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 266                         git config --unset svn-remote.svn.branches
 267                         git config --unset svn-remote.svn.tags
 268                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 269                 fi; }
 270                 # remember the starting time so we can easily combine fetched loose objects
 271                 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
 272                 if ! [ -e .svnpack ]; then
 273                         rm -f .svnpack
 274                         >.svnpack
 275                         sleep 1
 276                 fi
 277                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 278                 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 279                 # git svn does not preserve group permissions in the svn subdirectory
 280                 chmod -R ug+rw,o+r svn
 281                 # git svn also leaves behind ref turds that end with @nnn
 282                 # We get rid of them now
 283                 git for-each-ref --format='%(refname)' |
 284                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 285                 git_updateref_stdin
 286                 unset GIT_ASKPASS_PASSWORD
 287                 ;;
 288         darcs://*)
 289                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 290                 httpurl="http://${url#darcs://}"
 291                 git_darcs_fetch "$httpurl"
 292                 ;;
 293         bzr://*)
 294                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 295                 # we just remove bzr:// here, a typical bzr url is just
 296                 # "lp:foo"
 297                 bzrurl="${url#bzr://}"
 298                 git_bzr_fetch "$bzrurl"
 299                 ;;
 300         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 301                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 302                 # We just remove hg+ here, so hg+http://... becomes http://...
 303                 hgurl="${url#hg+}"
 304                 # Perform the initial hg clone
 305                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 306                 # Do the fast-export | fast-import
 307                 git_hg_fetch
 308                 ;;
 309         *)
 310                 # We manually add remote.origin.url and remote.origin.fetch
 311                 # to simulate a `git remote add --mirror=fetch` since that's
 312                 # not available until Git 1.7.5 and this way we guarantee we
 313                 # always get exactly the intended configuration and nothing else.
 314                 git config remote.origin.url "$url"
 315                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 316                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 317                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 318                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 319                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 320                         git config --bool girocco.lastupdateclean true
 321                 else
 322                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 323                         git config --bool girocco.lastupdateclean false
 324                 fi
 325                 # Set the correct HEAD symref by using ls-remote first
 326                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 327                 {
 328                         # Since everything was redirected, on failure there'd be no output,
 329                         # so let's make some failure output
 330                         cat .pkts-temp
 331                         echo ""
 332                         echo "git ls-remote \"$url\" failed"
 333                         exit 1
 334                 }
 335                 # Compensate for git() {} side effects
 336                 unset GIT_TRACE_PACKET
 337                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 338                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 339                 # out of the packet trace data.
 340                 if [ -s .refs-temp ]; then
 341                         # Nothing to do unless the remote repository has at least 1 ref
 342                         # See if we got a HEAD ref
 343                         head="$(LC_ALL=C grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 344                         # If the remote has HEAD set to a symbolic ref that does not exist
 345                         # then we will not receive a HEAD ref in the ls-remote output
 346                         headref=
 347                         showheadwarn=
 348                         symrefcap=
 349                         if [ -n "$head" ]; then
 350                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 351                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 352                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 353                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 354                                 # matches HEAD otherwise take the first refs/heads/... match
 355                                 matchcnt=0
 356                                 while read ref; do
 357                                         [ -n "$ref" ] || continue
 358                                         matchcnt=$(( $matchcnt + 1 ))
 359                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 360                                                 headref="$ref"
 361                                         fi
 362                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 363                                 done <<-EOT
 364                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 365                                         LC_ALL=C awk '{print $2}')
 366                                 EOT
 367                                 # Warn if there was more than one match and $symrefcap is empty
 368                                 # or $symrefcap is not the same as $headref since our choice might
 369                                 # differ from the source repository's HEAD
 370                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 371                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 372                                         showheadwarn=1
 373                                 fi
 374                         fi
 375                         if [ -z "$headref" ]; then
 376                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 377                                 # if it exists otherwise take the first refs/heads/...
 378                                 # We do not support having a detached HEAD.
 379                                 # We always warn now because we will be setting HEAD differently
 380                                 # than the source repository had HEAD set
 381                                 showheadwarn=1
 382                                 while read ref; do
 383                                         [ -n "$ref" ] || continue
 384                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 385                                                 headref="$ref"
 386                                         fi
 387                                         [ "$headref" = "refs/heads/master" ] && break
 388                                 done <<-EOT
 389                                 $(LC_ALL=C grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 390                                         LC_ALL=C awk '{print $2}')
 391                                 EOT
 392                         fi
 393                         # If we STILL do not have a HEAD ref (perhaps the source repository
 394                         # contains only tags) then use refs/heads/master.  It will be invalid
 395                         # but is no worse than we used to do by default and we'll warn about
 396                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 397                         [ -n "$headref" ] || headref="refs/heads/master"
 398                         git symbolic-ref HEAD "$headref"
 399                         pruneopt=--prune
 400                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 401                         git_add_config 'fetch.unpackLimit=1'
 402                         # Note the git config documentation is wrong
 403                         # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 404                         git_add_config 'transfer.unpackLimit=1'
 405                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 406                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 407                         if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
 408                                 rm -f .gfipack
 409                                 >.gfipack
 410                                 sleep 1
 411                         fi
 412                         GIT_SSL_NO_VERIFY=1 git remote update $pruneopt
 413                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 414                                 find objects/pack -type f -newer .gfipack -name "pack-$octet20.pack" -print >>gfi-packs
 415                                 rm -f .gfipack
 416                         fi
 417                 else
 418                         warnempty=1
 419                         git symbolic-ref HEAD "refs/heads/master"
 420                 fi
 421                 rm -f .refs-temp .pkts-temp
 422                 ;;
 423 esac
 424
 425 # The objects subdirectories permissions must be updated now.
 426 # In the case of a dumb http clone, the permissions will not be correct
 427 # (missing group write) despite the core.sharedrepository=1 setting!
 428 # The objects themselves seem to have the correct permissions.
 429 # This problem appears to have been fixed in the most recent git versions.
 430 perms=g+w
 431 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 432 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 433
 434 # We may have just cloned a lot of refs and they will all be
 435 # individual files at this point.  Let's pack them now so we
 436 # can have better performance right from the start.
 437 git pack-refs --all
 438
 439 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 440 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 441 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 442 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 443         --count=1 refs/heads > info/lastactivity || :
 444 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 445
 446 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 447 rm -f FETCH_HEAD
 448
 449 # Last ditch attempt to get a valid HEAD for a non-git source
 450 check_and_set_head || :
 451
 452 # The rest
 453 echo "Final touches..."
 454 git update-server-info
 455 trap "" EXIT
 456
 457 # run gc now unless the clone is empty
 458 if [ -z "$warnempty" ]; then
 459         git config --unset gitweb.lastgc 2>/dev/null || :
 460         rm -f .delaygc .allowgc
 461 fi
 462
 463 emptynote=
 464 [ -z "$warnempty" ] ||
 465 emptynote="
 466 WARNING:  You have mirrored an empty repository.
 467 "
 468 headnote=
 469 [ -n "$showheadwarn" -a -n "$headref" ] &&
 470 headnote="
 471 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 472        Use the \"Project settings\" link to choose a different HEAD symref.
 473 "
 474 sizenote=
 475 ! is_gfi_mirror ||
 476 sizenote="
 477 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 478        size may be somewhat larger than necessary.  This will be corrected
 479        shortly.  If you intend to clone this repository you may want to
 480        wait up to 1 hour before doing so in order to receive the more
 481        compact final size.
 482 "
 483 [ -z "$mailaddrs" ] ||
 484 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 485 Congratulations! The clone of project $proj just completed.
 486
 487         * Source URL: $url
 488         * GitWeb interface: $cfg_gitweburl/$proj.git
 489         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 490 $emptynote$headnote$sizenote
 491 Have a lot of fun.
 492 EOT
 493
 494 echo "Mirroring finished successfuly!"
 495 # In case this is a re-mirror, lastgc could have been set already so clear it now
 496 git config --unset gitweb.lastgc || :
 497 rm .clone_in_progress
 498 echo "$sizenote@OVER@"