taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() (
   9         set_utf8_locale
  10         _err1=
  11         _err2=
  12         exec 3>&1
  13         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  14         $(
  15                 exec 4>&3 3>&1 1>&4 4>&-
  16                 {
  17                         _e1=0
  18                         "$cfg_basedir"/bin/darcs-fast-export \
  19                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  20                         echo $_e1 >&3
  21                 } | \
  22                 {
  23                         _e2=0
  24                         git fast-import \
  25                                 --export-marks="$(pwd)/gfi-marks" \
  26                                 --export-pack-edges="$(pwd)/gfi-packs" \
  27                                 --force 3>&- || _e2=$?
  28                         echo $_e2 >&3
  29                 }
  30         )
  31         EOT
  32         exec 3>&-
  33         [ "$_err1" = 0 -a "$_err2" = 0 ]
  34         return $?
  35 )
  36
  37 # bzr fast-export | git fast-import with error handling
  38 git_bzr_fetch() (
  39         set_utf8_locale
  40         BZR_LOG=/dev/null
  41         export BZR_LOG
  42         _err1=
  43         _err2=
  44         exec 3>&1
  45         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  46         $(
  47                 exec 4>&3 3>&1 1>&4 4>&-
  48                 {
  49                         _e1=0
  50                         bzr fast-export --plain \
  51                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  52                         echo $_e1 >&3
  53                 } | \
  54                 {
  55                         _e2=0
  56                         git fast-import \
  57                                 --export-marks="$(pwd)/gfi-marks" \
  58                                 --export-pack-edges="$(pwd)/gfi-packs" \
  59                                 --force 3>&- || _e2=$?
  60                         echo $_e2 >&3
  61                 }
  62         )
  63         EOT
  64         exec 3>&-
  65         [ "$_err1" = 0 -a "$_err2" = 0 ]
  66         return $?
  67 )
  68
  69 send_clone_failed() {
  70         trap "" EXIT
  71         # We must now close the .clonelog file that is open on stdout and stderr
  72         exec >/dev/null 2>&1
  73         failaddrs="$(config_get owner || :)"
  74         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  75         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  76         [ -z "$failaddrs" ] || \
  77         {
  78               cat <<EOT
  79 Condolences. The clone of project $proj just failed.
  80
  81         * Source URL: $url
  82         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
  83
  84 The project settings link may be used to adjust the settings
  85 and restart the clone in order to try the clone again.
  86 EOT
  87                 if [ -f .clonelog -a -r .clonelog ]; then
  88                         echo ""
  89                         echo "Log follows:"
  90                         echo ""
  91                         loglines=$(wc -l <.clonelog)
  92                         if [ $loglines -le 203 ]; then
  93                                 cat .clonelog
  94                         else
  95                                 head -n 100 .clonelog
  96                                 echo ""
  97                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
  98                                 echo ""
  99                                 tail -n 100 .clonelog
 100                         fi
 101                 fi
 102         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 103 }
 104
 105 set -e
 106
 107 umask 002
 108 [ "$cfg_permission_control" != "Hooks" ] || umask 000
 109
 110 proj="${1%.git}"
 111 cd "$cfg_reporoot/$proj.git"
 112
 113 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 114 echo "Project: $proj"
 115 echo ""
 116 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 117 url="$(config_get baseurl || :)"
 118 case "$url" in *"       "*|*" "*|"")
 119         echo "Bad mirror URL (\"$url\")"
 120         exit 1
 121 esac
 122 echo "Mirroring from URL \"$url\""
 123 echo ""
 124
 125 if [ "$cfg_project_owners" = "source" ]; then
 126         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 127 fi
 128
 129 mailaddrs="$(config_get owner || :)"
 130 [ -z "$cfg_admin" ] || \
 131 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 132
 133 # Initial mirror
 134 echo "Initiating mirroring..."
 135 headref=
 136 showheadwarn=
 137 warnempty=
 138 case "$url" in
 139         svn://* | svn+http://* | svn+https://*)
 140                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 141                 # We just remove svn+ here, so svn+http://... becomes http://...
 142                 # We also remove a trailing '/' to match what git-svn will do
 143                 svnurl="${url#svn+}"
 144                 svnurl="${svnurl%/}"
 145                 # Remove any left-over svn-remote.svn config from a previous failed attempt
 146                 git config --remove-section svn-remote.svn 2>/dev/null || :
 147                 # Remove any left-over svn dir from a previous failed attempt
 148                 rm -rf svn
 149                 # Use an 'anonsvn' username as is commonly used for anonymous svn
 150                 # Use an 'anonsvn' password as is commonly used for anonymous svn
 151                 GIT_ASKPASS_PASSWORD=anonsvn
 152                 export GIT_ASKPASS_PASSWORD
 153                 # We require svn info to succeed on the URL otherwise it's
 154                 # simply not a valid URL and without using -s on the init it
 155                 # will not otherwise be tested until the fetch
 156                 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
 157                 # We initially use -s for the init which will possibly shorten
 158                 # the URL.  However, the shortening can fail if a password is
 159                 # not required for the longer version but is for the shorter,
 160                 # so try again without -s if the -s version fails.
 161                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 162                 # create a .git subdirectory!
 163                 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
 164                 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
 165                 # We need to remember this url so we can detect changes because
 166                 # ever so "helpful" git-svn may shorten it!
 167                 config_set svnurl "$svnurl"
 168                 # At this point, since we asked for a standard layout (-s) git-svn
 169                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 170                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 171                 # config items.  We could avoid this by not using the '-s' option
 172                 # but then we might not get all the history.  If, for example, we
 173                 # are cloning an http://svn.example.com/repos/public repository that
 174                 # early in its history moved trunk => public/trunk we would miss that
 175                 # earlier history without allowing the funky shorten+prefix behavior.
 176                 # So we read back the svn-remote.svn.fetch configuration and compute
 177                 # the prefix.  This way we are sure to get the correct prefix.
 178                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 179                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 180                 gitsvnprefix="${gitsvnfetch%%:*}"
 181                 gitsvnsuffix="${gitsvnprefix##*/}"
 182                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 183                 # Ask git-svn to store everything in the normal non-remote
 184                 # locations being careful to use the correct prefix
 185                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 186                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 187                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 188                 # look for additional non-standard directories to fetch
 189                 # check for standard layout at the same time
 190                 foundstd=
 191                 foundfile=
 192                 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 193                 { while read file; do case $file in
 194                         # skip the already-handled standard ones and any with a space or tab
 195                         *'      '*|*' '*) :;;
 196                         trunk/|branches/|tags/) foundstd=1;;
 197                         # only fetch extra directories from the $svnurl root (not any files)
 198                         *?/) git config --add svn-remote.svn.fetch \
 199                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 200                         *?) foundfile=1;;
 201                 esac; done
 202                 # if files found and no standard directories present use a simpler layout
 203                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 204                         git config --unset svn-remote.svn.branches
 205                         git config --unset svn-remote.svn.tags
 206                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 207                 fi; }
 208                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 209                 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
 210                 # git svn does not preserve group permissions in the svn subdirectory
 211                 chmod -R ug+rw,o+r svn
 212                 # git svn also leaves behind ref turds that end with @nnn
 213                 # We get rid of them now
 214                 git for-each-ref --format='%(objectname) %(refname)' | \
 215                 { while read sha1 ref; do
 216                         case "$ref" in
 217                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 218                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 219                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 220                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 221                                 git update-ref -d "$ref"
 222                         esac
 223                 done; }
 224                 unset GIT_ASKPASS_PASSWORD
 225                 ;;
 226         darcs://*)
 227                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 228                 httpurl="http://${url#darcs://}"
 229                 # Remove any left-over .darcs dirs from a previous failed attempt
 230                 rm -rf *.darcs
 231                 # Remove any left-over export files from a previous failed attempt
 232                 rm -f dfe-marks gfi-marks gfi-packs
 233                 git_darcs_fetch "$httpurl"
 234                 ;;
 235         bzr://*)
 236                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 237                 # we just remove bzr:// here, a typical bzr url is just
 238                 # "lp:foo"
 239                 bzrurl="${url#bzr://}"
 240                 # Remove any left-over export files from a previous failed attempt
 241                 rm -f bfe-marks gfi-marks gfi-packs
 242                 git_bzr_fetch "$bzrurl"
 243                 ;;
 244         hg+http://* | hg+https://*)
 245                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 246                 # We just remove hg+ here, so hg+http://... becomes http://...
 247                 hgurl="${url#hg+}"
 248                 # Remove any left-over repo.hg dir from a previous failed attempt
 249                 rm -rf repo.hg
 250                 # Remove any left-over export files from a previous failed attempt
 251                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 252                 # Perform the initial hg clone
 253                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 254                 # Do the fast-export | fast-import
 255                 git_hg_fetch
 256                 ;;
 257         *)
 258                 git remote rm origin >/dev/null 2>&1 || :
 259                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 260                 # since we support older Git versions, just toss the warning
 261                 git remote add --mirror origin "$url" 2>/dev/null
 262                 # Set the correct HEAD symref by using ls-remote first
 263                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 264                 {
 265                         # Since everything was redirected, on failure there'd be no output,
 266                         # so let's make some failure output
 267                         cat .pkts-temp
 268                         echo ""
 269                         echo "git ls-remote \"$url\" failed"
 270                         exit 1
 271                 }
 272                 # Compensate for git() {} side effects
 273                 unset GIT_TRACE_PACKET
 274                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 275                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 276                 # out of the packet trace data.
 277                 if [ -s .refs-temp ]; then
 278                         # Nothing to do unless the remote repository has at least 1 ref
 279                         # See if we got a HEAD ref
 280                         head="$(grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | awk '{print $1}')"
 281                         # If the remote has HEAD set to a symbolic ref that does not exist
 282                         # then we will not receive a HEAD ref in the ls-remote output
 283                         headref=
 284                         showheadwarn=
 285                         symrefcap=
 286                         if [ -n "$head" ]; then
 287                                 symrefcap="$(sed -ne <.pkts-temp \
 288                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 289                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 290                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 291                                 # matches HEAD otherwise take the first refs/heads/... match
 292                                 matchcnt=0
 293                                 while read ref; do
 294                                         [ -n "$ref" ] || continue
 295                                         matchcnt=$(( $matchcnt + 1 ))
 296                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 297                                                 headref="$ref"
 298                                         fi
 299                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 300                                 done <<-EOT
 301                                 $(grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 302                                         awk '{print $2}')
 303                                 EOT
 304                                 # Warn if there was more than one match and $symrefcap is empty
 305                                 # or $symrefcap is not the same as $headref since our choice might
 306                                 # differ from the source repository's HEAD
 307                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 308                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 309                                         showheadwarn=1
 310                                 fi
 311                         fi
 312                         if [ -z "$headref" ]; then
 313                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 314                                 # if it exists otherwise take the first refs/heads/...
 315                                 # We do not support having a detached HEAD.
 316                                 # We always warn now because we will be setting HEAD differently
 317                                 # than the source repository had HEAD set
 318                                 showheadwarn=1
 319                                 while read ref; do
 320                                         [ -n "$ref" ] || continue
 321                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 322                                                 headref="$ref"
 323                                         fi
 324                                         [ "$headref" = "refs/heads/master" ] && break
 325                                 done <<-EOT
 326                                 $(grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
 327                                         awk '{print $2}')
 328                                 EOT
 329                         fi
 330                         # If we STILL do not have a HEAD ref (perhaps the source repository
 331                         # contains only tags) then use refs/heads/master.  It will be invalid
 332                         # but is no worse than we used to do by default and we'll warn about
 333                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 334                         [ -n "$headref" ] || headref="refs/heads/master"
 335                         git symbolic-ref HEAD "$headref"
 336                         GIT_SSL_NO_VERIFY=1 git remote update
 337                         GIT_SSL_NO_VERIFY=1 git remote prune origin
 338                 else
 339                         warnempty=1
 340                         git symbolic-ref HEAD "refs/heads/master"
 341                 fi
 342                 rm -f .refs-temp .pkts-temp
 343                 ;;
 344 esac
 345
 346 # The objects subdirectories permissions must be updated now.
 347 # In the case of a dumb http clone, the permissions will not be correct
 348 # (missing group write) despite the core.sharedrepository=1 setting!
 349 # The objects themselves seem to have the correct permissions.
 350 # This problem appears to have been fixed in the most recent git versions.
 351 perms=g+w
 352 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 353 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 354
 355 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 356 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 357 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 358 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 359         --count=1 refs/heads > info/lastactivity || :
 360 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 361
 362 # The rest
 363 echo "Final touches..."
 364 git update-server-info
 365 trap "" EXIT
 366
 367 emptynote=
 368 [ -z "$warnempty" ] ||
 369 emptynote="
 370 WARNING:  You have mirrored an empty repository.
 371 "
 372 headnote=
 373 [ -n "$showheadwarn" -a -n "$headref" ] &&
 374 headnote="
 375 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 376        Use the \"Project settings\" link to choose a different HEAD symref.
 377 "
 378 sizenote=
 379 ! is_gfi_mirror ||
 380 sizenote="
 381 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 382        size may be somewhat larger than necessary.  This will be corrected
 383        shortly.  If you intend to clone this repository you may want to
 384        wait up to 1 hour before doing so in order to receive the more
 385        compact final size.
 386 "
 387 [ -z "$mailaddrs" ] ||
 388 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 389 Congratulations! The clone of project $proj just completed.
 390
 391         * Source URL: $url
 392         * GitWeb interface: $cfg_gitweburl/$proj.git
 393         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 394 $emptynote$headnote$sizenote
 395 Have a lot of fun.
 396 EOT
 397
 398 echo "Mirroring finished successfuly!"
 399 # In case this is a re-mirror, lastgc could have been set already so clear it now
 400 git config --unset gitweb.lastgc || :
 401 rm .clone_in_progress
 402 echo "$sizenote@OVER@"