taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6
   7 # darcs fast-export | git fast-import with error handling
   8 git_darcs_fetch() (
   9         set_utf8_locale
  10         _err1=
  11         _err2=
  12         exec 3>&1
  13         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  14         $(
  15                 exec 4>&3 3>&1 1>&4 4>&-
  16                 {
  17                         _e1=0
  18                         "$cfg_basedir"/bin/darcs-fast-export \
  19                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  20                         echo $_e1 >&3
  21                 } | \
  22                 {
  23                         _e2=0
  24                         git fast-import \
  25                                 --export-marks="$(pwd)/gfi-marks" \
  26                                 --export-pack-edges="$(pwd)/gfi-packs" \
  27                                 --force 3>&- || _e2=$?
  28                         echo $_e2 >&3
  29                 }
  30         )
  31         EOT
  32         exec 3>&-
  33         [ "$_err1" = 0 -a "$_err2" = 0 ]
  34         return $?
  35 )
  36
  37 # bzr fast-export | git fast-import with error handling
  38 git_bzr_fetch() (
  39         set_utf8_locale
  40         _err1=
  41         _err2=
  42         exec 3>&1
  43         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  44         $(
  45                 exec 4>&3 3>&1 1>&4 4>&-
  46                 {
  47                         _e1=0
  48                         bzr fast-export \
  49                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  50                         echo $_e1 >&3
  51                 } | \
  52                 {
  53                         _e2=0
  54                         git fast-import \
  55                                 --export-marks="$(pwd)/gfi-marks" \
  56                                 --export-pack-edges="$(pwd)/gfi-packs" \
  57                                 --force 3>&- || _e2=$?
  58                         echo $_e2 >&3
  59                 }
  60         )
  61         EOT
  62         exec 3>&-
  63         [ "$_err1" = 0 -a "$_err2" = 0 ]
  64         return $?
  65 )
  66
  67 send_clone_failed() {
  68         trap "" EXIT
  69         # We must now close the .clonelog file that is open on stdout and stderr
  70         exec >/dev/null 2>&1
  71         failaddrs="$(config_get owner || :)"
  72         [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
  73         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  74         [ -z "$failaddrs" ] || \
  75         {
  76               cat <<EOT
  77 Condolences. The clone of project $proj just failed.
  78
  79         * Source URL: $url
  80         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
  81
  82 The project settings link may be used to adjust the settings
  83 and restart the clone in order to try the clone again.
  84 EOT
  85                 if [ -f .clonelog -a -r .clonelog ]; then
  86                         echo ""
  87                         echo "Log follows:"
  88                         echo ""
  89                         loglines=$(wc -l <.clonelog)
  90                         if [ $loglines -le 203 ]; then
  91                                 cat .clonelog
  92                         else
  93                                 head -n 100 .clonelog
  94                                 echo ""
  95                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
  96                                 echo ""
  97                                 tail -n 100 .clonelog
  98                         fi
  99                 fi
 100         } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 101 }
 102
 103 set -e
 104
 105 umask 002
 106 [ "$cfg_permission_control" != "Hooks" ] || umask 000
 107
 108 projdir="$1"
 109 proj="${projdir%.git}"
 110
 111 cd "$cfg_reporoot/$projdir"
 112 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 113 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 114 url="$(config_get baseurl || :)"
 115 case "$url" in *"       "*|*" "*|"")
 116         echo "Bad mirror URL (\"$url\")"
 117         exit 1
 118 esac
 119 echo "Mirroring from URL \"$url\""
 120 echo ""
 121
 122 if [ "$cfg_project_owners" = "source" ]; then
 123         config_set owner "$(stat -c %U "$url" 2>/dev/null)"
 124 fi
 125
 126 mailaddrs="$(config_get owner || :)"
 127 [ -z "$cfg_admin" ] || \
 128 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 129
 130 # Initial mirror
 131 echo "Initiating mirroring..."
 132 headref=
 133 showheadwarn=
 134 warnempty=
 135 case "$url" in
 136         svn://* | svn+http://* | svn+https://*)
 137                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 138                 # We just remove svn+ here, so svn+http://... becomes http://...
 139                 # We also remove a trailing '/' to match what git-svn will do
 140                 svnurl="${url#svn+}"
 141                 svnurl="${svnurl%/}"
 142                 # Remove any left-over svn-remote.svn config from a previous failed attempt
 143                 git config --remove-section svn-remote.svn 2>/dev/null || :
 144                 # Remove any left-over svn dir from a previous failed attempt
 145                 rm -rf svn
 146                 # We require svn info to succeed on the URL otherwise it's
 147                 # simply not a valid URL and without using -s on the init it
 148                 # will not otherwise be tested until the fetch
 149                 svn --non-interactive info "$svnurl" > /dev/null
 150                 # We initially use -s for the init which will possibly shorten
 151                 # the URL.  However, the shortening can fail if a password is
 152                 # not required for the longer version but is for the shorter,
 153                 # so try again without -s if the -s version fails.
 154                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 155                 # create a .git subdirectory!
 156                 GIT_DIR=. git svn init --prefix "" -s "$svnurl" < /dev/null || \
 157                 GIT_DIR=. git svn init --prefix "" "$svnurl" < /dev/null
 158                 # We need to remember this url so we can detect changes because
 159                 # ever so "helpful" git-svn may shorten it!
 160                 config_set svnurl "$svnurl"
 161                 # At this point, since we asked for a standard layout (-s) git-svn
 162                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 163                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 164                 # config items.  We could avoid this by not using the '-s' option
 165                 # but then we might not get all the history.  If, for example, we
 166                 # are cloning an http://svn.example.com/repos/public repository that
 167                 # early in its history moved trunk => public/trunk we would miss that
 168                 # earlier history without allowing the funky shorten+prefix behavior.
 169                 # So we read back the svn-remote.svn.fetch configuration and compute
 170                 # the prefix.  This way we are sure to get the correct prefix.
 171                 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
 172                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
 173                 gitsvnprefix="${gitsvnfetch%%:*}"
 174                 gitsvnsuffix="${gitsvnprefix##*/}"
 175                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 176                 # Ask git-svn to store everything in the normal non-remote
 177                 # locations being careful to use the correct prefix
 178                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 179                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 180                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 181                 # look for additional non-standard directories to fetch
 182                 # check for standard layout at the same time
 183                 foundstd=
 184                 foundfile=
 185                 { svn --non-interactive ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
 186                 { while read file; do case $file in
 187                         # skip the already-handled standard ones and any with a space or tab
 188                         *'      '*|*' '*) :;;
 189                         trunk/|branches/|tags/) foundstd=1;;
 190                         # only fetch extra directories from the $svnurl root (not any files)
 191                         *?/) git config --add svn-remote.svn.fetch \
 192                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 193                         *?) foundfile=1;;
 194                 esac; done
 195                 # if files found and no standard directories present use a simpler layout
 196                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 197                         git config --unset svn-remote.svn.branches
 198                         git config --unset svn-remote.svn.tags
 199                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 200                 fi; }
 201                 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 202                 GIT_DIR=. git svn fetch --quiet < /dev/null
 203                 # git svn does not preserve group permissions in the svn subdirectory
 204                 chmod -R ug+rw,o+r svn
 205                 # git svn also leaves behind ref turds that end with @nnn
 206                 # We get rid of them now
 207                 git show-ref | \
 208                 { while read sha1 ref; do
 209                         case "$ref" in
 210                         ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
 211                         ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
 212                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
 213                         ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
 214                                 git update-ref -d "$ref"
 215                         esac
 216                 done; }
 217                 ;;
 218         darcs://*)
 219                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 220                 httpurl="http://${url#darcs://}"
 221                 # Remove any left-over .darcs dirs from a previous failed attempt
 222                 rm -rf *.darcs
 223                 # Remove any left-over export files from a previous failed attempt
 224                 rm -f dfe-marks gfi-marks gfi-packs
 225                 git_darcs_fetch "$httpurl"
 226                 ;;
 227         bzr://*)
 228                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 229                 # we just remove bzr:// here, a typical bzr url is just
 230                 # "lp:foo"
 231                 bzrurl="${url#bzr://}"
 232                 # Remove any left-over export files from a previous failed attempt
 233                 rm -f bfe-marks gfi-marks gfi-packs
 234                 git_bzr_fetch "$bzrurl"
 235                 ;;
 236         hg+http://* | hg+https://*)
 237                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 238                 # We just remove hg+ here, so hg+http://... becomes http://...
 239                 hgurl="${url#hg+}"
 240                 # Remove any left-over repo.hg dir from a previous failed attempt
 241                 rm -rf repo.hg
 242                 # Remove any left-over export files from a previous failed attempt
 243                 rm -f gfi-packs hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
 244                 # Perform the initial hg clone
 245                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 246                 # Do the fast-export | fast-import
 247                 git_hg_fetch
 248                 ;;
 249         *)
 250                 git remote rm origin >/dev/null 2>&1 || :
 251                 # starting with Git 1.7.5 --mirror by itself will spew a warning
 252                 # since we support older Git versions, just toss the warning
 253                 git remote add --mirror origin "$url" 2>/dev/null
 254                 # Set the correct HEAD symref by using ls-remote first
 255                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
 256                 {
 257                         # Since everything was redirected, on failure there'd be no output,
 258                         # so let's make some failure output
 259                         cat .pkts-temp
 260                         echo ""
 261                         echo "git ls-remote \"$url\" failed"
 262                         exit 1
 263                 }
 264                 # Compensate for git() {} side effects
 265                 unset GIT_TRACE_PACKET
 266                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 267                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 268                 # out of the packet trace data.
 269                 if [ -s .refs-temp ]; then
 270                         # Nothing to do unless the remote repository has at least 1 ref
 271                         # See if we got a HEAD ref
 272                         head="$(grep -E "^$octet20[[:space:]]+HEAD\$" <.refs-temp | awk '{print $1}')"
 273                         # If the remote has HEAD set to a symbolic ref that does not exist
 274                         # then we will not receive a HEAD ref in the ls-remote output
 275                         headref=
 276                         showheadwarn=
 277                         symrefcap=
 278                         if [ -n "$head" ]; then
 279                                 symrefcap="$(sed -ne <.pkts-temp \
 280                                         '/packet:.*git<.*[[:space:]]symref=HEAD:refs\/heads\/[^[:space:]][^[:space:]]*/'"\
 281                                         "'{s/^.*[[:space:]]symref=HEAD:\(refs\/heads\/[^[:space:]][^[:space:]]*\).*$/\1/;p;}')"
 282                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 283                                 # matches HEAD otherwise take the first refs/heads/... match
 284                                 matchcnt=0
 285                                 while read ref; do
 286                                         [ -n "$ref" ] || continue
 287                                         matchcnt=$(( $matchcnt + 1 ))
 288                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 289                                                 headref="$ref"
 290                                         fi
 291                                         [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
 292                                 done <<-EOT
 293                                 $(grep -E "^$head[[:space:]]+refs/heads/[^[:space:]]+\$" <.refs-temp | \
 294                                         awk '{print $2}')
 295                                 EOT
 296                                 # Warn if there was more than one match and $symrefcap is empty
 297                                 # or $symrefcap is not the same as $headref since our choice might
 298                                 # differ from the source repository's HEAD
 299                                 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
 300                                    [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
 301                                         showheadwarn=1
 302                                 fi
 303                         fi
 304                         if [ -z "$headref" ]; then
 305                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 306                                 # if it exists otherwise take the first refs/heads/...
 307                                 # We do not support having a detached HEAD.
 308                                 # We always warn now because we will be setting HEAD differently
 309                                 # than the source repository had HEAD set
 310                                 showheadwarn=1
 311                                 while read ref; do
 312                                         [ -n "$ref" ] || continue
 313                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 314                                                 headref="$ref"
 315                                         fi
 316                                         [ "$headref" = "refs/heads/master" ] && break
 317                                 done <<-EOT
 318                                 $(grep -E "^$octet20[[:space:]]+refs/heads/[^[:space:]]+\$" <.refs-temp | \
 319                                         awk '{print $2}')
 320                                 EOT
 321                         fi
 322                         # If we STILL do not have a HEAD ref (perhaps the source repository
 323                         # contains only tags) then use refs/heads/master.  It will be invalid
 324                         # but is no worse than we used to do by default and we'll warn about
 325                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 326                         [ -n "$headref" ] || headref="refs/heads/master"
 327                         git symbolic-ref HEAD "$headref"
 328                         GIT_SSL_NO_VERIFY=1 git remote update
 329                         GIT_SSL_NO_VERIFY=1 git remote prune origin
 330                 else
 331                         warnempty=1
 332                         git symbolic-ref HEAD "refs/heads/master"
 333                 fi
 334                 rm -f .refs-temp .pkts-temp
 335                 ;;
 336 esac
 337
 338 # The objects subdirectories permissions must be updated now.
 339 # In the case of a dumb http clone, the permissions will not be correct
 340 # (missing group write) despite the core.sharedrepository=1 setting!
 341 # The objects themselves seem to have the correct permissions.
 342 # This problem appears to have been fixed in the most recent git versions.
 343 perms=g+w
 344 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 345 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
 346
 347 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 348 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 349 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 350 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 351         --count=1 refs/heads > info/lastactivity
 352 [ -s info/lastactivity ] || rm -f info/lastactivity
 353
 354 # The rest
 355 echo "Final touches..."
 356 git update-server-info
 357 trap "" EXIT
 358
 359 emptynote=
 360 [ -z "$warnempty" ] ||
 361 emptynote="
 362 WARNING:  You have mirrored an empty repository.
 363 "
 364 headnote=
 365 [ -n "$showheadwarn" -a -n "$headref" ] &&
 366 headnote="
 367 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 368        Use the \"Project settings\" link to choose a different HEAD symref.
 369 "
 370 sizenote=
 371 ! is_gfi_mirror ||
 372 sizenote="
 373 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 374        size may be somewhat larger than necessary.  This will be corrected
 375        shortly.  If you intend to clone this repository you may want to
 376        wait up to 1 hour before doing so in order to receive the more
 377        compact final size.
 378 "
 379 [ -z "$mailaddrs" ] ||
 380 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 381 Congratulations! The clone of project $proj just completed.
 382
 383         * Source URL: $url
 384         * GitWeb interface: $cfg_gitweburl/$projdir
 385         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | sed -e 's/[+]/%2B/g')
 386 $emptynote$headnote$sizenote
 387 Have a lot of fun.
 388 EOT
 389
 390 echo "Mirroring finished successfuly!"
 391 # In case this is a re-mirror, lastgc could have been set already so clear it now
 392 git config --unset gitweb.lastgc || :
 393 rm .clone_in_progress
 394 echo "$sizenote@OVER@"