taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 set -e
   8
   9 umask 002
  10 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  11
  12 # darcs fast-export | git fast-import with error handling
  13 git_darcs_fetch() (
  14         set_utf8_locale
  15         _err1=
  16         _err2=
  17         exec 3>&1
  18         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  19         $(
  20                 exec 4>&3 3>&1 1>&4 4>&-
  21                 {
  22                         _e1=0
  23                         "$cfg_basedir"/bin/darcs-fast-export \
  24                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  25                         echo $_e1 >&3
  26                 } | \
  27                 {
  28                         _e2=0
  29                         git fast-import \
  30                                 --export-marks="$(pwd)/gfi-marks" \
  31                                 --export-pack-edges="$(pwd)/gfi-packs" \
  32                                 --force 3>&- || _e2=$?
  33                         echo $_e2 >&3
  34                 }
  35         )
  36         EOT
  37         exec 3>&-
  38         [ "$_err1" = 0 -a "$_err2" = 0 ]
  39         return $?
  40 )
  41
  42 # bzr fast-export | git fast-import with error handling
  43 git_bzr_fetch() (
  44         set_utf8_locale
  45         BZR_LOG=/dev/null
  46         export BZR_LOG
  47         _err1=
  48         _err2=
  49         exec 3>&1
  50         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  51         $(
  52                 exec 4>&3 3>&1 1>&4 4>&-
  53                 {
  54                         _e1=0
  55                         bzr fast-export --plain \
  56                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  57                         echo $_e1 >&3
  58                 } | \
  59                 {
  60                         _e2=0
  61                         git fast-import \
  62                                 --export-marks="$(pwd)/gfi-marks" \
  63                                 --export-pack-edges="$(pwd)/gfi-packs" \
  64                                 --force 3>&- || _e2=$?
  65                         echo $_e2 >&3
  66                 }
  67         )
  68         EOT
  69         exec 3>&-
  70         [ "$_err1" = 0 -a "$_err2" = 0 ]
  71         return $?
  72 )
  73
  74 send_clone_failed() {
  75         trap "" EXIT
  76         # We must now close the .clonelog file that is open on stdout and stderr
  77         exec >/dev/null 2>&1
  78         failaddrs="$(config_get owner || :)"
  79         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  80         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  81         [ -z "$failaddrs" ] || \
  82         {
  83               cat <<EOT
  84 Condolences. The clone of project $proj just failed.
  85
  86         * Source URL: $url
  87         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
  88
  89 The project settings link may be used to adjust the settings
  90 and restart the clone in order to try the clone again.
  91 EOT
  92                 if [ -f .clonelog -a -r .clonelog ]; then
  93                         echo ""
  94                         echo "Log follows:"
  95                         echo ""
  96                         loglines=$(LC_ALL=C wc -l <.clonelog)
  97                         if [ $loglines -le 203 ]; then
  98                                 cat .clonelog
  99                         else
 100                                 head -n 100 .clonelog
 101                                 echo ""
 102                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 103                                 echo ""
 104                                 tail -n 100 .clonelog
 105                         fi
 106                 fi
 107         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 108 }
 109
 110 # removes all leftovers from a previous failed clone attempt
 111 cleanup_failed_clone() {
 112
 113         # Remove any left-over svn-remote.svn or remote.origin config
 114         git config --remove-section svn-remote.svn 2>/dev/null || :
 115         git config --remove-section remote.origin 2>/dev/null || :
 116
 117         # If there is a remote-template.origin section, pre-seed the
 118         # remote.origin section with its contents
 119         git config --get-regexp '^remote-template\.origin\..' |
 120         while read name value; do
 121                 if [ -n "$name" -a -n "$value" ]; then
 122                         git config "remote${name#remote-template}" "$value"
 123                 fi
 124         done
 125
 126         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 127         # now garbage to be removed
 128         rm -f FETCH_HEAD
 129
 130         # Remove any left-over svn dir from a previous failed attempt
 131         rm -rf svn
 132
 133         # Remove any left-over .darcs dirs from a previous failed attempt
 134         rm -rf *.darcs
 135
 136         # Remove any left-over repo.hg dir from a previous failed attempt
 137         rm -rf repo.hg
 138
 139         # Remove any left-over import/export/temp files from a previous failed attempt
 140         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 141                 gfi-marks gfi-packs .pkts-temp .refs-temp
 142
 143         # We want a gc right after the clone, so re-enable that just in case.
 144         # There's a potential race where we could add it and gc.sh could remove
 145         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 146         [ -e .delaygc ] || >.delaygc
 147         git config --unset gitweb.lastgc 2>/dev/null || :
 148
 149         # Remove all pre-existing refs
 150         rm -f packed-refs
 151         eval "$(git for-each-ref --shell --format='git update-ref -d %(refname) || :')" 2>/dev/null || :
 152
 153         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 154         git symbolic-ref HEAD refs/heads/master
 155
 156         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 157         # will get rid of any extras.  Also, if we're recloning the same thing, any
 158         # preexisting packs/loose objects containing what we're recloning will only
 159         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 160 }
 161
 162 proj="${1%.git}"
 163 cd "$cfg_reporoot/$proj.git"
 164 bang_reset
 165
 166 ! [ -e .delaygc ] || >.allowgc || :
 167
 168 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 169 echo "Project: $proj"
 170 echo ""
 171 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 172 url="$(config_get baseurl || :)"
 173 case "$url" in *"       "*|*" "*|"")
 174         echo "Bad mirror URL (\"$url\")"
 175         exit 1
 176 esac
 177
 178 cleanup_failed_clone
 179
 180 # Record original mirror type for use by update.sh
 181 mirror_type="$(get_url_mirror_type "$url")"
 182 git config girocco.mirrortype "$mirror_type"
 183
 184 echo "Mirroring from URL \"$url\""
 185 echo ""
 186
 187 if [ "$cfg_project_owners" = "source" ]; then
 188         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 189 fi
 190
 191 mailaddrs="$(config_get owner || :)"
 192 [ -z "$cfg_admin" ] || \
 193 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 194
 195 # Initial mirror
 196 echo "Initiating mirroring..."
 197 headref=
 198 showheadwarn=
 199 warnempty=
 200 case "$url" in
 201         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 202                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 203                 # We just remove svn+ here, so svn+http://... becomes http://...
 204                 # We also remove a trailing '/' to match what git-svn will do
 205                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 206                 svnurl="${svnurl%/}"
 207                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 208                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 209                 GIT_ASKPASS_PASSWORD=anonsvn
 210                 export GIT_ASKPASS_PASSWORD
 211                 # We require svn info to succeed on the URL otherwise it's
 212                 # simply not a valid URL and without using -s on the init it
 213                 # will not otherwise be tested until the fetch
 214                 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
 215                 # We initially use -s for the init which will possibly shorten
 216                 # the URL.  However, the shortening can fail if a password is
 217                 # not required for the longer version but is for the shorter,
 218                 # so try again without -s if the -s version fails.
 219                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 220                 # create a .git subdirectory!
 221                 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
 222                 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
 223                 # We need to remember this url so we can detect changes because
 224                 # ever so "helpful" git-svn may shorten it!
 225                 config_set svnurl "$svnurl"
 226                 # At this point, since we asked for a standard layout (-s) git-svn
 227                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 228                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 229                 # config items.  We could avoid this by not using the '-s' option
 230                 # but then we might not get all the history.  If, for example, we
 231                 # are cloning an http://svn.example.com/repos/public repository that
 232                 # early in its history moved trunk => public/trunk we would miss that
 233                 # earlier history without allowing the funky shorten+prefix behavior.
 234                 # So we read back the svn-remote.svn.fetch configuration and compute
 235                 # the prefix.  This way we are sure to get the correct prefix.
 236                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 237                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 238                 gitsvnprefix="${gitsvnfetch%%:*}"
 239                 gitsvnsuffix="${gitsvnprefix##*/}"
 240                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 241                 # Ask git-svn to store everything in the normal non-remote
 242                 # locations being careful to use the correct prefix
 243                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 244                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 245                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 246                 # look for additional non-standard directories to fetch
 247                 # check for standard layout at the same time
 248                 foundstd=
 249                 foundfile=
 250                 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 251                 { while read file; do case $file in
 252                         # skip the already-handled standard ones and any with a space or tab
 253                         *'      '*|*' '*) :;;
 254                         trunk/|branches/|tags/) foundstd=1;;
 255                         # only fetch extra directories from the $svnurl root (not any files)
 256                         *?/) git config --add svn-remote.svn.fetch \
 257                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 258                         *?) foundfile=1;;
 259                 esac; done
 260                 # if files found and no standard directories present use a simpler layout
 261                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 262                         git config --unset svn-remote.svn.branches
 263                         git config --unset svn-remote.svn.tags
 264                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 265                 fi; }
 266                 # remember the starting time so we can easily combine fetched loose objects
 267                 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
 268                 if ! [ -e .svnpack ]; then
 269                         rm -f .svnpack
 270                         >.svnpack
 271                         sleep 1
 272                 fi
 273                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 274                 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 275                 # git svn does not preserve group permissions in the svn subdirectory
 276                 chmod -R ug+rw,o+r svn
 277                 # git svn also leaves behind ref turds that end with @nnn
 278                 # We get rid of them now
 279                 git for-each-ref --format='%(objectname) %(refname)' | \
 280                 { while read sha1 ref; do
 281                         case "$ref" in
 282                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 283                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 284                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 285                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 286                                 git update-ref -d "$ref"
 287                         esac
 288                 done; }
 289                 unset GIT_ASKPASS_PASSWORD
 290                 ;;
 291         darcs://*)
 292                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 293                 httpurl="http://${url#darcs://}"
 294                 git_darcs_fetch "$httpurl"
 295                 ;;
 296         bzr://*)
 297                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 298                 # we just remove bzr:// here, a typical bzr url is just
 299                 # "lp:foo"
 300                 bzrurl="${url#bzr://}"
 301                 git_bzr_fetch "$bzrurl"
 302                 ;;
 303         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 304                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 305                 # We just remove hg+ here, so hg+http://... becomes http://...
 306                 hgurl="${url#hg+}"
 307                 # Perform the initial hg clone
 308                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 309                 # Do the fast-export | fast-import
 310                 git_hg_fetch
 311                 ;;
 312         *)
 313                 # We manually add remote.origin.url and remote.origin.fetch
 314                 # to simulate a `git remote add --mirror=fetch` since that's
 315                 # not available until Git 1.7.5 and this way we guarantee we
 316                 # always get exactly the intended configuration and nothing else.
 317                 git config remote.origin.url "$url"
 318                 git config remote.origin.fetch "+refs/*:refs/*"
 319                 # Set the correct HEAD symref by using ls-remote first
 320                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 321                 {
 322                         # Since everything was redirected, on failure there'd be no output,
 323                         # so let's make some failure output
 324                         cat .pkts-temp
 325                         echo ""
 326                         echo "git ls-remote \"$url\" failed"
 327                         exit 1
 328                 }
 329                 # Compensate for git() {} side effects
 330                 unset GIT_TRACE_PACKET
 331                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 332                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 333                 # out of the packet trace data.
 334                 if [ -s .refs-temp ]; then
 335                         # Nothing to do unless the remote repository has at least 1 ref
 336                         # See if we got a HEAD ref
 337                         head="$(LC_ALL=C grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 338                         # If the remote has HEAD set to a symbolic ref that does not exist
 339                         # then we will not receive a HEAD ref in the ls-remote output
 340                         headref=
 341                         showheadwarn=
 342                         symrefcap=
 343                         if [ -n "$head" ]; then
 344                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 345                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 346                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 347                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 348                                 # matches HEAD otherwise take the first refs/heads/... match
 349                                 matchcnt=0
 350                                 while read ref; do
 351                                         [ -n "$ref" ] || continue
 352                                         matchcnt=$(( $matchcnt + 1 ))
 353                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 354                                                 headref="$ref"
 355                                         fi
 356                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 357                                 done <<-EOT
 358                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 359                                         LC_ALL=C awk '{print $2}')
 360                                 EOT
 361                                 # Warn if there was more than one match and $symrefcap is empty
 362                                 # or $symrefcap is not the same as $headref since our choice might
 363                                 # differ from the source repository's HEAD
 364                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 365                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 366                                         showheadwarn=1
 367                                 fi
 368                         fi
 369                         if [ -z "$headref" ]; then
 370                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 371                                 # if it exists otherwise take the first refs/heads/...
 372                                 # We do not support having a detached HEAD.
 373                                 # We always warn now because we will be setting HEAD differently
 374                                 # than the source repository had HEAD set
 375                                 showheadwarn=1
 376                                 while read ref; do
 377                                         [ -n "$ref" ] || continue
 378                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 379                                                 headref="$ref"
 380                                         fi
 381                                         [ "$headref" = "refs/heads/master" ] && break
 382                                 done <<-EOT
 383                                 $(LC_ALL=C grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 384                                         LC_ALL=C awk '{print $2}')
 385                                 EOT
 386                         fi
 387                         # If we STILL do not have a HEAD ref (perhaps the source repository
 388                         # contains only tags) then use refs/heads/master.  It will be invalid
 389                         # but is no worse than we used to do by default and we'll warn about
 390                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 391                         [ -n "$headref" ] || headref="refs/heads/master"
 392                         git symbolic-ref HEAD "$headref"
 393                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 394                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 395                         if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
 396                                 rm -f .gfipack
 397                                 >.gfipack
 398                                 sleep 1
 399                         fi
 400                         GIT_SSL_NO_VERIFY=1 git remote update --prune
 401                         if is_gfi_mirror_url "$url"; then
 402                                 find objects/pack -type f -newer .gfipack -name "pack-$octet20.pack" -print >>gfi-packs
 403                                 rm -f .gfipack
 404                         fi
 405                 else
 406                         warnempty=1
 407                         git symbolic-ref HEAD "refs/heads/master"
 408                 fi
 409                 rm -f .refs-temp .pkts-temp
 410                 ;;
 411 esac
 412
 413 # The objects subdirectories permissions must be updated now.
 414 # In the case of a dumb http clone, the permissions will not be correct
 415 # (missing group write) despite the core.sharedrepository=1 setting!
 416 # The objects themselves seem to have the correct permissions.
 417 # This problem appears to have been fixed in the most recent git versions.
 418 perms=g+w
 419 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 420 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 421
 422 # We may have just cloned a lot of refs and they will all be
 423 # individual files at this point.  Let's pack them now so we
 424 # can have better performance right from the start.
 425 git pack-refs --all
 426
 427 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 428 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 429 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 430 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 431         --count=1 refs/heads > info/lastactivity || :
 432 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 433
 434 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 435 rm -f FETCH_HEAD
 436
 437 # The rest
 438 echo "Final touches..."
 439 git update-server-info
 440 trap "" EXIT
 441
 442 # run gc now unless the clone is empty
 443 if [ -z "$warnempty" ]; then
 444         git config --unset gitweb.lastgc 2>/dev/null || :
 445         rm -f .delaygc .allowgc
 446 fi
 447
 448 emptynote=
 449 [ -z "$warnempty" ] ||
 450 emptynote="
 451 WARNING:  You have mirrored an empty repository.
 452 "
 453 headnote=
 454 [ -n "$showheadwarn" -a -n "$headref" ] &&
 455 headnote="
 456 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 457        Use the \"Project settings\" link to choose a different HEAD symref.
 458 "
 459 sizenote=
 460 ! is_gfi_mirror ||
 461 sizenote="
 462 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 463        size may be somewhat larger than necessary.  This will be corrected
 464        shortly.  If you intend to clone this repository you may want to
 465        wait up to 1 hour before doing so in order to receive the more
 466        compact final size.
 467 "
 468 [ -z "$mailaddrs" ] ||
 469 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 470 Congratulations! The clone of project $proj just completed.
 471
 472         * Source URL: $url
 473         * GitWeb interface: $cfg_gitweburl/$proj.git
 474         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 475 $emptynote$headnote$sizenote
 476 Have a lot of fun.
 477 EOT
 478
 479 echo "Mirroring finished successfuly!"
 480 # In case this is a re-mirror, lastgc could have been set already so clear it now
 481 git config --unset gitweb.lastgc || :
 482 rm .clone_in_progress
 483 echo "$sizenote@OVER@"