taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() (
   9         set_utf8_locale
  10         _err1=
  11         _err2=
  12         exec 3>&1
  13         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  14         $(
  15                 exec 4>&3 3>&1 1>&4 4>&-
  16                 {
  17                         _e1=0
  18                         "$cfg_basedir"/bin/darcs-fast-export \
  19                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  20                         echo $_e1 >&3
  21                 } | \
  22                 {
  23                         _e2=0
  24                         git fast-import \
  25                                 --export-marks="$(pwd)/gfi-marks" \
  26                                 --export-pack-edges="$(pwd)/gfi-packs" \
  27                                 --force 3>&- || _e2=$?
  28                         echo $_e2 >&3
  29                 }
  30         )
  31         EOT
  32         exec 3>&-
  33         [ "$_err1" = 0 -a "$_err2" = 0 ]
  34         return $?
  35 )
  36
  37 # bzr fast-export | git fast-import with error handling
  38 git_bzr_fetch() (
  39         set_utf8_locale
  40         BZR_LOG=/dev/null
  41         export BZR_LOG
  42         _err1=
  43         _err2=
  44         exec 3>&1
  45         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  46         $(
  47                 exec 4>&3 3>&1 1>&4 4>&-
  48                 {
  49                         _e1=0
  50                         bzr fast-export --plain \
  51                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  52                         echo $_e1 >&3
  53                 } | \
  54                 {
  55                         _e2=0
  56                         git fast-import \
  57                                 --export-marks="$(pwd)/gfi-marks" \
  58                                 --export-pack-edges="$(pwd)/gfi-packs" \
  59                                 --force 3>&- || _e2=$?
  60                         echo $_e2 >&3
  61                 }
  62         )
  63         EOT
  64         exec 3>&-
  65         [ "$_err1" = 0 -a "$_err2" = 0 ]
  66         return $?
  67 )
  68
  69 send_clone_failed() {
  70         trap "" EXIT
  71         # We must now close the .clonelog file that is open on stdout and stderr
  72         exec >/dev/null 2>&1
  73         failaddrs="$(config_get owner || :)"
  74         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  75         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  76         [ -z "$failaddrs" ] || \
  77         {
  78               cat <<EOT
  79 Condolences. The clone of project $proj just failed.
  80
  81         * Source URL: $url
  82         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
  83
  84 The project settings link may be used to adjust the settings
  85 and restart the clone in order to try the clone again.
  86 EOT
  87                 if [ -f .clonelog -a -r .clonelog ]; then
  88                         echo ""
  89                         echo "Log follows:"
  90                         echo ""
  91                         loglines=$(wc -l <.clonelog)
  92                         if [ $loglines -le 203 ]; then
  93                                 cat .clonelog
  94                         else
  95                                 head -n 100 .clonelog
  96                                 echo ""
  97                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
  98                                 echo ""
  99                                 tail -n 100 .clonelog
 100                         fi
 101                 fi
 102         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 103 }
 104
 105 set -e
 106
 107 umask 002
 108 [ "$cfg_permission_control" != "Hooks" ] || umask 000
 109
 110 proj="${1%.git}"
 111 cd "$cfg_reporoot/$proj.git"
 112 bang_reset
 113
 114 ! [ -e .delaygc ] || > .allowgc || :
 115
 116 # Any pre-existing FETCH_HEAD from a previous clone failed or not is garbage
 117 rm -f FETCH_HEAD
 118
 119 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 120 echo "Project: $proj"
 121 echo ""
 122 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 123 url="$(config_get baseurl || :)"
 124 case "$url" in *"       "*|*" "*|"")
 125         echo "Bad mirror URL (\"$url\")"
 126         exit 1
 127 esac
 128 echo "Mirroring from URL \"$url\""
 129 echo ""
 130
 131 if [ "$cfg_project_owners" = "source" ]; then
 132         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 133 fi
 134
 135 mailaddrs="$(config_get owner || :)"
 136 [ -z "$cfg_admin" ] || \
 137 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 138
 139 # Initial mirror
 140 echo "Initiating mirroring..."
 141 headref=
 142 showheadwarn=
 143 warnempty=
 144 case "$url" in
 145         svn://* | svn+http://* | svn+https://* | svn+file://*)
 146                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 147                 # We just remove svn+ here, so svn+http://... becomes http://...
 148                 # We also remove a trailing '/' to match what git-svn will do
 149                 svnurl="${url#svn+}"
 150                 svnurl="${svnurl%/}"
 151                 # Remove any left-over svn-remote.svn config from a previous failed attempt
 152                 git config --remove-section svn-remote.svn 2>/dev/null || :
 153                 # Remove any left-over svn dir from a previous failed attempt
 154                 rm -rf svn
 155                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 156                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 157                 GIT_ASKPASS_PASSWORD=anonsvn
 158                 export GIT_ASKPASS_PASSWORD
 159                 # We require svn info to succeed on the URL otherwise it's
 160                 # simply not a valid URL and without using -s on the init it
 161                 # will not otherwise be tested until the fetch
 162                 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
 163                 # We initially use -s for the init which will possibly shorten
 164                 # the URL.  However, the shortening can fail if a password is
 165                 # not required for the longer version but is for the shorter,
 166                 # so try again without -s if the -s version fails.
 167                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 168                 # create a .git subdirectory!
 169                 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
 170                 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
 171                 # We need to remember this url so we can detect changes because
 172                 # ever so "helpful" git-svn may shorten it!
 173                 config_set svnurl "$svnurl"
 174                 # At this point, since we asked for a standard layout (-s) git-svn
 175                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 176                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 177                 # config items.  We could avoid this by not using the '-s' option
 178                 # but then we might not get all the history.  If, for example, we
 179                 # are cloning an http://svn.example.com/repos/public repository that
 180                 # early in its history moved trunk => public/trunk we would miss that
 181                 # earlier history without allowing the funky shorten+prefix behavior.
 182                 # So we read back the svn-remote.svn.fetch configuration and compute
 183                 # the prefix.  This way we are sure to get the correct prefix.
 184                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 185                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 186                 gitsvnprefix="${gitsvnfetch%%:*}"
 187                 gitsvnsuffix="${gitsvnprefix##*/}"
 188                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 189                 # Ask git-svn to store everything in the normal non-remote
 190                 # locations being careful to use the correct prefix
 191                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 192                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 193                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 194                 # look for additional non-standard directories to fetch
 195                 # check for standard layout at the same time
 196                 foundstd=
 197                 foundfile=
 198                 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 199                 { while read file; do case $file in
 200                         # skip the already-handled standard ones and any with a space or tab
 201                         *'      '*|*' '*) :;;
 202                         trunk/|branches/|tags/) foundstd=1;;
 203                         # only fetch extra directories from the $svnurl root (not any files)
 204                         *?/) git config --add svn-remote.svn.fetch \
 205                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 206                         *?) foundfile=1;;
 207                 esac; done
 208                 # if files found and no standard directories present use a simpler layout
 209                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 210                         git config --unset svn-remote.svn.branches
 211                         git config --unset svn-remote.svn.tags
 212                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 213                 fi; }
 214                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 215                 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 216                 # git svn does not preserve group permissions in the svn subdirectory
 217                 chmod -R ug+rw,o+r svn
 218                 # git svn also leaves behind ref turds that end with @nnn
 219                 # We get rid of them now
 220                 git for-each-ref --format='%(objectname) %(refname)' | \
 221                 { while read sha1 ref; do
 222                         case "$ref" in
 223                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 224                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 225                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 226                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 227                                 git update-ref -d "$ref"
 228                         esac
 229                 done; }
 230                 unset GIT_ASKPASS_PASSWORD
 231                 ;;
 232         darcs://*)
 233                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 234                 httpurl="http://${url#darcs://}"
 235                 # Remove any left-over .darcs dirs from a previous failed attempt
 236                 rm -rf *.darcs
 237                 # Remove any left-over export files from a previous failed attempt
 238                 rm -f dfe-marks gfi-marks gfi-packs
 239                 git_darcs_fetch "$httpurl"
 240                 ;;
 241         bzr://*)
 242                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 243                 # we just remove bzr:// here, a typical bzr url is just
 244                 # "lp:foo"
 245                 bzrurl="${url#bzr://}"
 246                 # Remove any left-over export files from a previous failed attempt
 247                 rm -f bfe-marks gfi-marks gfi-packs
 248                 git_bzr_fetch "$bzrurl"
 249                 ;;
 250         hg+http://* | hg+https://*)
 251                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 252                 # We just remove hg+ here, so hg+http://... becomes http://...
 253                 hgurl="${url#hg+}"
 254                 # Remove any left-over repo.hg dir from a previous failed attempt
 255                 rm -rf repo.hg
 256                 # Remove any left-over export files from a previous failed attempt
 257                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 258                 # Perform the initial hg clone
 259                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 260                 # Do the fast-export | fast-import
 261                 git_hg_fetch
 262                 ;;
 263         *)
 264                 git remote rm origin >/dev/null 2>&1 || :
 265                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 266                 # since we support older Git versions, just toss the warning
 267                 git remote add --mirror origin "$url" 2>/dev/null
 268                 # Set the correct HEAD symref by using ls-remote first
 269                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 270                 {
 271                         # Since everything was redirected, on failure there'd be no output,
 272                         # so let's make some failure output
 273                         cat .pkts-temp
 274                         echo ""
 275                         echo "git ls-remote \"$url\" failed"
 276                         exit 1
 277                 }
 278                 # Compensate for git() {} side effects
 279                 unset GIT_TRACE_PACKET
 280                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 281                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 282                 # out of the packet trace data.
 283                 if [ -s .refs-temp ]; then
 284                         # Nothing to do unless the remote repository has at least 1 ref
 285                         # See if we got a HEAD ref
 286                         head="$(grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | awk '{print $1}')"
 287                         # If the remote has HEAD set to a symbolic ref that does not exist
 288                         # then we will not receive a HEAD ref in the ls-remote output
 289                         headref=
 290                         showheadwarn=
 291                         symrefcap=
 292                         if [ -n "$head" ]; then
 293                                 symrefcap="$(sed -ne <.pkts-temp \
 294                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 295                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 296                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 297                                 # matches HEAD otherwise take the first refs/heads/... match
 298                                 matchcnt=0
 299                                 while read ref; do
 300                                         [ -n "$ref" ] || continue
 301                                         matchcnt=$(( $matchcnt + 1 ))
 302                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 303                                                 headref="$ref"
 304                                         fi
 305                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 306                                 done <<-EOT
 307                                 $(grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 308                                         awk '{print $2}')
 309                                 EOT
 310                                 # Warn if there was more than one match and $symrefcap is empty
 311                                 # or $symrefcap is not the same as $headref since our choice might
 312                                 # differ from the source repository's HEAD
 313                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 314                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 315                                         showheadwarn=1
 316                                 fi
 317                         fi
 318                         if [ -z "$headref" ]; then
 319                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 320                                 # if it exists otherwise take the first refs/heads/...
 321                                 # We do not support having a detached HEAD.
 322                                 # We always warn now because we will be setting HEAD differently
 323                                 # than the source repository had HEAD set
 324                                 showheadwarn=1
 325                                 while read ref; do
 326                                         [ -n "$ref" ] || continue
 327                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 328                                                 headref="$ref"
 329                                         fi
 330                                         [ "$headref" = "refs/heads/master" ] && break
 331                                 done <<-EOT
 332                                 $(grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 333                                         awk '{print $2}')
 334                                 EOT
 335                         fi
 336                         # If we STILL do not have a HEAD ref (perhaps the source repository
 337                         # contains only tags) then use refs/heads/master.  It will be invalid
 338                         # but is no worse than we used to do by default and we'll warn about
 339                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 340                         [ -n "$headref" ] || headref="refs/heads/master"
 341                         git symbolic-ref HEAD "$headref"
 342                         GIT_SSL_NO_VERIFY=1 git remote update --prune
 343                 else
 344                         warnempty=1
 345                         git symbolic-ref HEAD "refs/heads/master"
 346                 fi
 347                 rm -f .refs-temp .pkts-temp
 348                 ;;
 349 esac
 350
 351 # The objects subdirectories permissions must be updated now.
 352 # In the case of a dumb http clone, the permissions will not be correct
 353 # (missing group write) despite the core.sharedrepository=1 setting!
 354 # The objects themselves seem to have the correct permissions.
 355 # This problem appears to have been fixed in the most recent git versions.
 356 perms=g+w
 357 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 358 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 359
 360 # We may have just cloned a lot of refs and they will all be
 361 # individual files at this point.  Let's pack them now so we
 362 # can have better performance right from the start.
 363 git pack-refs --all
 364
 365 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 366 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 367 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 368 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 369         --count=1 refs/heads > info/lastactivity || :
 370 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 371
 372 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 373 rm -f FETCH_HEAD
 374
 375 # The rest
 376 echo "Final touches..."
 377 git update-server-info
 378 trap "" EXIT
 379
 380 # allow gc now unless it's empty
 381 [ -n "$warnempty" ] || rm -f .delaygc .allowgc
 382
 383 emptynote=
 384 [ -z "$warnempty" ] ||
 385 emptynote="
 386 WARNING:  You have mirrored an empty repository.
 387 "
 388 headnote=
 389 [ -n "$showheadwarn" -a -n "$headref" ] &&
 390 headnote="
 391 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 392        Use the \"Project settings\" link to choose a different HEAD symref.
 393 "
 394 sizenote=
 395 ! is_gfi_mirror ||
 396 sizenote="
 397 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 398        size may be somewhat larger than necessary.  This will be corrected
 399        shortly.  If you intend to clone this repository you may want to
 400        wait up to 1 hour before doing so in order to receive the more
 401        compact final size.
 402 "
 403 [ -z "$mailaddrs" ] ||
 404 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 405 Congratulations! The clone of project $proj just completed.
 406
 407         * Source URL: $url
 408         * GitWeb interface: $cfg_gitweburl/$proj.git
 409         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 410 $emptynote$headnote$sizenote
 411 Have a lot of fun.
 412 EOT
 413
 414 echo "Mirroring finished successfuly!"
 415 # In case this is a re-mirror, lastgc could have been set already so clear it now
 416 git config --unset gitweb.lastgc || :
 417 rm .clone_in_progress
 418 echo "$sizenote@OVER@"