projects: make sure HEAD is a valid symref
[girocco.git] / taskd / clone.sh
blobcbe3481ed8fa280284869cd3d14371809a09e453
1 #!/bin/sh
3 # Invoked from taskd/taskd.pl
5 . @basedir@/shlib.sh
7 set -e
9 umask 002
10 [ "$cfg_permission_control" != "Hooks" ] || umask 000
12 # darcs fast-export | git fast-import with error handling
13 git_darcs_fetch() (
14 set_utf8_locale
15 _err1=
16 _err2=
17 exec 3>&1
18 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
20 exec 4>&3 3>&1 1>&4 4>&-
22 _e1=0
23 "$cfg_basedir"/bin/darcs-fast-export \
24 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
25 echo $_e1 >&3
26 } | \
28 _e2=0
29 git fast-import \
30 --export-marks="$(pwd)/gfi-marks" \
31 --export-pack-edges="$(pwd)/gfi-packs" \
32 --force 3>&- || _e2=$?
33 echo $_e2 >&3
36 EOT
37 exec 3>&-
38 [ "$_err1" = 0 -a "$_err2" = 0 ]
39 return $?
42 # bzr fast-export | git fast-import with error handling
43 git_bzr_fetch() (
44 set_utf8_locale
45 BZR_LOG=/dev/null
46 export BZR_LOG
47 _err1=
48 _err2=
49 exec 3>&1
50 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
52 exec 4>&3 3>&1 1>&4 4>&-
54 _e1=0
55 bzr fast-export --plain \
56 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
57 echo $_e1 >&3
58 } | \
60 _e2=0
61 git fast-import \
62 --export-marks="$(pwd)/gfi-marks" \
63 --export-pack-edges="$(pwd)/gfi-packs" \
64 --force 3>&- || _e2=$?
65 echo $_e2 >&3
68 EOT
69 exec 3>&-
70 [ "$_err1" = 0 -a "$_err2" = 0 ]
71 return $?
74 send_clone_failed() {
75 trap "" EXIT
76 # We must now close the .clonelog file that is open on stdout and stderr
77 exec >/dev/null 2>&1
78 failaddrs="$(config_get owner || :)"
79 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
80 if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
81 [ -z "$failaddrs" ] || \
83 cat <<EOT
84 Condolences. The clone of project $proj just failed.
86 * Source URL: $url
87 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
89 The project settings link may be used to adjust the settings
90 and restart the clone in order to try the clone again.
91 EOT
92 if [ -f .clonelog -a -r .clonelog ]; then
93 echo ""
94 echo "Log follows:"
95 echo ""
96 loglines=$(LC_ALL=C wc -l <.clonelog)
97 if [ $loglines -le 203 ]; then
98 cat .clonelog
99 else
100 head -n 100 .clonelog
101 echo ""
102 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
103 echo ""
104 tail -n 100 .clonelog
107 } | mail -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
110 # removes all leftovers from a previous failed clone attempt
111 cleanup_failed_clone() {
113 # Remove any left-over svn-remote.svn or remote.origin config
114 git config --remove-section svn-remote.svn 2>/dev/null || :
115 git config --remove-section remote.origin 2>/dev/null || :
117 # If there is a remote-template.origin section, pre-seed the
118 # remote.origin section with its contents
119 git config --get-regexp '^remote-template\.origin\..' |
120 while read name value; do
121 if [ -n "$name" -a -n "$value" ]; then
122 git config "remote${name#remote-template}" "$value"
124 done
126 # Any pre-existing FETCH_HEAD from a previous clone failed or not is
127 # now garbage to be removed
128 rm -f FETCH_HEAD
130 # Remove any left-over svn dir from a previous failed attempt
131 rm -rf svn
133 # Remove any left-over .darcs dirs from a previous failed attempt
134 rm -rf *.darcs
136 # Remove any left-over repo.hg dir from a previous failed attempt
137 rm -rf repo.hg
139 # Remove any left-over import/export/temp files from a previous failed attempt
140 rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
141 gfi-marks gfi-packs .pkts-temp .refs-temp
143 # We want a gc right after the clone, so re-enable that just in case.
144 # There's a potential race where we could add it and gc.sh could remove
145 # it, but we'll reunset lastgc just before we remove .delaygc at the end.
146 [ -e .delaygc ] || >.delaygc
147 git config --unset gitweb.lastgc 2>/dev/null || :
149 # Remove all pre-existing refs
150 rm -f packed-refs
151 eval "$(git for-each-ref --shell --format='git update-ref -d %(refname) || :')" 2>/dev/null || :
153 # The initial state before a clone starts has HEAD as a symbolic-ref to master
154 git symbolic-ref HEAD refs/heads/master
156 # HEAD is no longer "ok"
157 git config --unset girocco.headok 2>/dev/null || :
159 # We, perhaps, ought to remove any packs/loose objects now, but the next gc
160 # will get rid of any extras. Also, if we're recloning the same thing, any
161 # preexisting packs/loose objects containing what we're recloning will only
162 # speed up the reclone by avoiding some disk writes. So we don't kill them.
165 proj="${1%.git}"
166 cd "$cfg_reporoot/$proj.git"
167 bang_reset
169 ! [ -e .delaygc ] || >.allowgc || :
171 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
172 echo "Project: $proj"
173 echo ""
174 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
175 url="$(config_get baseurl || :)"
176 case "$url" in *" "*|*" "*|"")
177 echo "Bad mirror URL (\"$url\")"
178 exit 1
179 esac
181 cleanup_failed_clone
183 # Record original mirror type for use by update.sh
184 mirror_type="$(get_url_mirror_type "$url")"
185 git config girocco.mirrortype "$mirror_type"
187 echo "Mirroring from URL \"$url\""
188 echo ""
190 if [ "$cfg_project_owners" = "source" ]; then
191 config_set owner "$(stat -c %U "$url" 2>/dev/null)"
194 mailaddrs="$(config_get owner || :)"
195 [ -z "$cfg_admin" ] || \
196 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
198 # Initial mirror
199 echo "Initiating mirroring..."
200 headref=
201 showheadwarn=
202 warnempty=
203 case "$url" in
204 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
205 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
206 # We just remove svn+ here, so svn+http://... becomes http://...
207 # We also remove a trailing '/' to match what git-svn will do
208 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
209 svnurl="${svnurl%/}"
210 # Use an 'anonsvn' username as is commonly used for anonymous svn
211 # Use an 'anonsvn' password as is commonly used for anonymous svn
212 GIT_ASKPASS_PASSWORD=anonsvn
213 export GIT_ASKPASS_PASSWORD
214 # We require svn info to succeed on the URL otherwise it's
215 # simply not a valid URL and without using -s on the init it
216 # will not otherwise be tested until the fetch
217 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null
218 # We initially use -s for the init which will possibly shorten
219 # the URL. However, the shortening can fail if a password is
220 # not required for the longer version but is for the shorter,
221 # so try again without -s if the -s version fails.
222 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
223 # create a .git subdirectory!
224 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null || \
225 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null
226 # We need to remember this url so we can detect changes because
227 # ever so "helpful" git-svn may shorten it!
228 config_set svnurl "$svnurl"
229 # At this point, since we asked for a standard layout (-s) git-svn
230 # may have been "helpful" and adjusted our $svnurl to a prefix and
231 # then glued the removed suffix onto the front of any svn-remote.svn.*
232 # config items. We could avoid this by not using the '-s' option
233 # but then we might not get all the history. If, for example, we
234 # are cloning an http://svn.example.com/repos/public repository that
235 # early in its history moved trunk => public/trunk we would miss that
236 # earlier history without allowing the funky shorten+prefix behavior.
237 # So we read back the svn-remote.svn.fetch configuration and compute
238 # the prefix. This way we are sure to get the correct prefix.
239 gitsvnurl="$(git config --get svn-remote.svn.url || :)"
240 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)"
241 gitsvnprefix="${gitsvnfetch%%:*}"
242 gitsvnsuffix="${gitsvnprefix##*/}"
243 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
244 # Ask git-svn to store everything in the normal non-remote
245 # locations being careful to use the correct prefix
246 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
247 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
248 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
249 # look for additional non-standard directories to fetch
250 # check for standard layout at the same time
251 foundstd=
252 foundfile=
253 { svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \
254 { while read file; do case $file in
255 # skip the already-handled standard ones and any with a space or tab
256 *' '*|*' '*) :;;
257 trunk/|branches/|tags/) foundstd=1;;
258 # only fetch extra directories from the $svnurl root (not any files)
259 *?/) git config --add svn-remote.svn.fetch \
260 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
261 *?) foundfile=1;;
262 esac; done
263 # if files found and no standard directories present use a simpler layout
264 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
265 git config --unset svn-remote.svn.branches
266 git config --unset svn-remote.svn.tags
267 git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
268 fi; }
269 # remember the starting time so we can easily combine fetched loose objects
270 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
271 if ! [ -e .svnpack ]; then
272 rm -f .svnpack
273 >.svnpack
274 sleep 1
276 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
277 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet < /dev/null
278 # git svn does not preserve group permissions in the svn subdirectory
279 chmod -R ug+rw,o+r svn
280 # git svn also leaves behind ref turds that end with @nnn
281 # We get rid of them now
282 git for-each-ref --format='%(objectname) %(refname)' | \
283 { while read sha1 ref; do
284 case "$ref" in
285 ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\
286 ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\
287 ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
288 ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
289 git update-ref -d "$ref"
290 esac
291 done; }
292 unset GIT_ASKPASS_PASSWORD
294 darcs://*)
295 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
296 httpurl="http://${url#darcs://}"
297 git_darcs_fetch "$httpurl"
299 bzr://*)
300 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
301 # we just remove bzr:// here, a typical bzr url is just
302 # "lp:foo"
303 bzrurl="${url#bzr://}"
304 git_bzr_fetch "$bzrurl"
306 hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
307 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
308 # We just remove hg+ here, so hg+http://... becomes http://...
309 hgurl="${url#hg+}"
310 # Perform the initial hg clone
311 hg clone -U "$hgurl" "$(pwd)/repo.hg"
312 # Do the fast-export | fast-import
313 git_hg_fetch
316 # We manually add remote.origin.url and remote.origin.fetch
317 # to simulate a `git remote add --mirror=fetch` since that's
318 # not available until Git 1.7.5 and this way we guarantee we
319 # always get exactly the intended configuration and nothing else.
320 git config remote.origin.url "$url"
321 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
322 git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
323 git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
324 git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
325 git config --bool girocco.lastupdateclean true
326 else
327 git config --replace-all remote.origin.fetch "+refs/*:refs/*"
328 git config --bool girocco.lastupdateclean false
330 # Set the correct HEAD symref by using ls-remote first
331 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp || \
333 # Since everything was redirected, on failure there'd be no output,
334 # so let's make some failure output
335 cat .pkts-temp
336 echo ""
337 echo "git ls-remote \"$url\" failed"
338 exit 1
340 # Compensate for git() {} side effects
341 unset GIT_TRACE_PACKET
342 # If the server is running at least Git 1.8.4.3 then it will send us the actual
343 # symref for HEAD. If we are running at least Git 1.7.5 then we can snarf that
344 # out of the packet trace data.
345 if [ -s .refs-temp ]; then
346 # Nothing to do unless the remote repository has at least 1 ref
347 # See if we got a HEAD ref
348 head="$(LC_ALL=C grep -E "^$octet20[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
349 # If the remote has HEAD set to a symbolic ref that does not exist
350 # then we will not receive a HEAD ref in the ls-remote output
351 headref=
352 showheadwarn=
353 symrefcap=
354 if [ -n "$head" ]; then
355 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
356 "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
357 {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
358 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
359 # matches HEAD otherwise take the first refs/heads/... match
360 matchcnt=0
361 while read ref; do
362 [ -n "$ref" ] || continue
363 matchcnt=$(( $matchcnt + 1 ))
364 if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
365 headref="$ref"
367 [ "$headref" = "${symrefcap:-refs/heads/master}" -a $matchcnt -gt 1 ] && break
368 done <<-EOT
369 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
370 LC_ALL=C awk '{print $2}')
372 # Warn if there was more than one match and $symrefcap is empty
373 # or $symrefcap is not the same as $headref since our choice might
374 # differ from the source repository's HEAD
375 if [ $matchcnt -ge 1 -a "$symrefcap" != "$headref" ] && \
376 [ -n "$symrefcap" -o $matchcnt -gt 1 ]; then
377 showheadwarn=1
380 if [ -z "$headref" ]; then
381 # If we still don't have a HEAD ref then prefer refs/heads/master
382 # if it exists otherwise take the first refs/heads/...
383 # We do not support having a detached HEAD.
384 # We always warn now because we will be setting HEAD differently
385 # than the source repository had HEAD set
386 showheadwarn=1
387 while read ref; do
388 [ -n "$ref" ] || continue
389 if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
390 headref="$ref"
392 [ "$headref" = "refs/heads/master" ] && break
393 done <<-EOT
394 $(LC_ALL=C grep -E "^$octet20[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp | \
395 LC_ALL=C awk '{print $2}')
398 # If we STILL do not have a HEAD ref (perhaps the source repository
399 # contains only tags) then use refs/heads/master. It will be invalid
400 # but is no worse than we used to do by default and we'll warn about
401 # it. We do not support a HEAD symref to anything other than refs/heads/...
402 [ -n "$headref" ] || headref="refs/heads/master"
403 git symbolic-ref HEAD "$headref"
404 pruneopt=--prune
405 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
406 git_add_config 'fetch.unpackLimit=1'
407 # Note the git config documentation is wrong
408 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
409 git_add_config 'transfer.unpackLimit=1'
410 # remember the starting time so we can easily detect new packs for fast-import mirrors
411 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
412 if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
413 rm -f .gfipack
414 >.gfipack
415 sleep 1
417 GIT_SSL_NO_VERIFY=1 git remote update $pruneopt
418 if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
419 find objects/pack -type f -newer .gfipack -name "pack-$octet20.pack" -print >>gfi-packs
420 rm -f .gfipack
422 else
423 warnempty=1
424 git symbolic-ref HEAD "refs/heads/master"
426 rm -f .refs-temp .pkts-temp
428 esac
430 # The objects subdirectories permissions must be updated now.
431 # In the case of a dumb http clone, the permissions will not be correct
432 # (missing group write) despite the core.sharedrepository=1 setting!
433 # The objects themselves seem to have the correct permissions.
434 # This problem appears to have been fixed in the most recent git versions.
435 perms=g+w
436 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
437 chmod $perms $(find objects -maxdepth 1 -type d) 2>/dev/null || :
439 # We may have just cloned a lot of refs and they will all be
440 # individual files at this point. Let's pack them now so we
441 # can have better performance right from the start.
442 git pack-refs --all
444 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
445 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
446 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
447 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
448 --count=1 refs/heads > info/lastactivity || :
449 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
451 # Don't leave a multi-megabyte useless FETCH_HEAD behind
452 rm -f FETCH_HEAD
454 # Last ditch attempt to get a valid HEAD for a non-git source
455 check_and_set_head || :
457 # The rest
458 echo "Final touches..."
459 git update-server-info
460 trap "" EXIT
462 # run gc now unless the clone is empty
463 if [ -z "$warnempty" ]; then
464 git config --unset gitweb.lastgc 2>/dev/null || :
465 rm -f .delaygc .allowgc
468 emptynote=
469 [ -z "$warnempty" ] ||
470 emptynote="
471 WARNING: You have mirrored an empty repository.
473 headnote=
474 [ -n "$showheadwarn" -a -n "$headref" ] &&
475 headnote="
476 NOTE: HEAD has been set to a symbolic ref to \"$headref\".
477 Use the \"Project settings\" link to choose a different HEAD symref.
479 sizenote=
480 ! is_gfi_mirror ||
481 sizenote="
482 NOTE: Since this is a mirror of a non-Git source, the initial repository
483 size may be somewhat larger than necessary. This will be corrected
484 shortly. If you intend to clone this repository you may want to
485 wait up to 1 hour before doing so in order to receive the more
486 compact final size.
488 [ -z "$mailaddrs" ] ||
489 mail -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
490 Congratulations! The clone of project $proj just completed.
492 * Source URL: $url
493 * GitWeb interface: $cfg_gitweburl/$proj.git
494 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
495 $emptynote$headnote$sizenote
496 Have a lot of fun.
499 echo "Mirroring finished successfuly!"
500 # In case this is a re-mirror, lastgc could have been set already so clear it now
501 git config --unset gitweb.lastgc || :
502 rm .clone_in_progress
503 echo "$sizenote@OVER@"