mail.sh: include copy/rename detection warnings
[girocco/readme.git] / taskd / clone.sh
blob35da82a5e5d50ea7e4cc1e399e15e82f5dfca49a
1 #!/bin/sh
3 # Invoked from taskd/taskd.pl
5 . @basedir@/shlib.sh
7 set -e
9 umask 002
10 [ "$cfg_permission_control" != "Hooks" ] || umask 000
11 clean_git_env
13 # darcs fast-export | git fast-import with error handling
14 git_darcs_fetch() (
15 set_utf8_locale
16 _err1=
17 _err2=
18 exec 3>&1
19 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
21 exec 4>&3 3>&1 1>&4 4>&-
23 _e1=0
24 "$cfg_basedir"/bin/darcs-fast-export \
25 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
26 echo $_e1 >&3
27 } |
29 _e2=0
30 git fast-import \
31 --export-marks="$(pwd)/gfi-marks" \
32 --export-pack-edges="$(pwd)/gfi-packs" \
33 --force 3>&- || _e2=$?
34 echo $_e2 >&3
37 EOT
38 exec 3>&-
39 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
40 return $?
43 # bzr fast-export | git fast-import with error handling
44 git_bzr_fetch() (
45 set_utf8_locale
46 BZR_LOG=/dev/null
47 export BZR_LOG
48 _err1=
49 _err2=
50 exec 3>&1
51 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
53 exec 4>&3 3>&1 1>&4 4>&-
55 _e1=0
56 bzr fast-export --plain \
57 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
58 echo $_e1 >&3
59 } |
61 _e2=0
62 git fast-import \
63 --export-marks="$(pwd)/gfi-marks" \
64 --export-pack-edges="$(pwd)/gfi-packs" \
65 --force 3>&- || _e2=$?
66 echo $_e2 >&3
69 EOT
70 exec 3>&-
71 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
72 return $?
75 send_clone_failed() {
76 trap "" EXIT
77 # We must now close the .clonelog file that is open on stdout and stderr
78 exec >/dev/null 2>&1
79 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
80 failaddrs="$(config_get owner)" || :
81 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
82 if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
83 [ -z "$failaddrs" ] ||
85 cat <<EOT
86 Condolences. The clone of project $proj just failed.
88 * Source URL: $url
89 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
91 The project settings link may be used to adjust the settings
92 and restart the clone in order to try the clone again.
93 EOT
94 if [ -f .clonelog ] && [ -r .clonelog ]; then
95 echo ""
96 echo "Log follows:"
97 echo ""
98 loglines=$(LC_ALL=C wc -l <.clonelog)
99 if [ $loglines -le 203 ]; then
100 cat .clonelog
101 else
102 head -n 100 .clonelog
103 echo ""
104 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
105 echo ""
106 tail -n 100 .clonelog
109 } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
112 # removes all leftovers from a previous failed clone attempt
113 cleanup_failed_clone() {
115 # Remove any left-over svn-remote.svn or remote.origin config
116 git config --remove-section svn-remote.svn 2>/dev/null || :
117 git config --remove-section remote.origin 2>/dev/null || :
119 # If there is a remote-template.origin section, pre-seed the
120 # remote.origin section with its contents
121 git config --get-regexp '^remote-template\.origin\..' |
122 while read name value; do
123 if [ -n "$name" ] && [ -n "$value" ]; then
124 git config "remote${name#remote-template}" "$value"
126 done
128 # Any pre-existing FETCH_HEAD from a previous clone failed or not is
129 # now garbage to be removed
130 rm -f FETCH_HEAD
132 # Remove any left-over svn dir from a previous failed attempt
133 rm -rf svn
135 # Remove any left-over .darcs dirs from a previous failed attempt
136 rm -rf *.darcs
138 # Remove any left-over repo.hg dir from a previous failed attempt
139 rm -rf repo.hg
141 # Remove any left-over import/export/temp files from a previous failed attempt
142 rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
143 gfi-marks gfi-packs .pkts-temp .refs-temp
145 # We want a gc right after the clone, so re-enable that just in case.
146 # There's a potential race where we could add it and gc.sh could remove
147 # it, but we'll reunset lastgc just before we remove .delaygc at the end.
148 [ -e .delaygc ] || >.delaygc
149 git config --unset gitweb.lastgc 2>/dev/null || :
151 # Remove all pre-existing refs
152 rm -f packed-refs
153 git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
155 # The initial state before a clone starts has HEAD as a symbolic-ref to master
156 git symbolic-ref HEAD refs/heads/master
158 # HEAD is no longer "ok"
159 git config --unset girocco.headok 2>/dev/null || :
161 # We, perhaps, ought to remove any packs/loose objects now, but the next gc
162 # will get rid of any extras. Also, if we're recloning the same thing, any
163 # preexisting packs/loose objects containing what we're recloning will only
164 # speed up the reclone by avoiding some disk writes. So we don't kill them.
167 proj="${1%.git}"
168 cd "$cfg_reporoot/$proj.git"
169 bang_reset
171 ! [ -e .delaygc ] || >.allowgc || :
173 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
174 echo "Project: $proj"
175 echo " Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
176 echo ""
177 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
178 url="$(config_get baseurl)" || :
179 case "$url" in *" "*|*" "*|"")
180 echo "Bad mirror URL (\"$url\")"
181 exit 1
182 esac
184 cleanup_failed_clone
186 # Record original mirror type for use by update.sh
187 mirror_type="$(get_url_mirror_type "$url")"
188 git config girocco.mirrortype "$mirror_type"
190 echo "Mirroring from URL \"$url\""
191 echo ""
193 if [ "$cfg_project_owners" = "source" ]; then
194 config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | awk '{print $3}')"
197 mailaddrs="$(config_get owner)" || :
198 [ -z "$cfg_admin" ] ||
199 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
201 # Make sure we don't get any unwanted loose objects
202 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
203 # tweak its configuration to prevent that
204 git_add_config 'fetch.unpackLimit=1'
205 # Note the git config documentation is wrong
206 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
207 git_add_config 'transfer.unpackLimit=1'
208 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
209 git_add_config 'fastimport.unpackLimit=0'
211 # Initial mirror
212 echo "Initiating mirroring..."
213 headref=
214 showheadwarn=
215 warnempty=
216 case "$url" in
217 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
218 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
219 # We just remove svn+ here, so svn+http://... becomes http://...
220 # We also remove a trailing '/' to match what git-svn will do
221 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
222 svnurl="${svnurl%/}"
223 # Use an 'anonsvn' username as is commonly used for anonymous svn
224 # Use an 'anonsvn' password as is commonly used for anonymous svn
225 GIT_ASKPASS_PASSWORD=anonsvn
226 export GIT_ASKPASS_PASSWORD
227 # We require svn info to succeed on the URL otherwise it's
228 # simply not a valid URL and without using -s on the init it
229 # will not otherwise be tested until the fetch
230 svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" >/dev/null
231 # We initially use -s for the init which will possibly shorten
232 # the URL. However, the shortening can fail if a password is
233 # not required for the longer version but is for the shorter,
234 # so try again without -s if the -s version fails.
235 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
236 # create a .git subdirectory!
237 GIT_DIR=. git svn init --username=anonsvn --prefix "" -s "$svnurl" </dev/null ||
238 GIT_DIR=. git svn init --username=anonsvn --prefix "" "$svnurl" </dev/null
239 # We need to remember this url so we can detect changes because
240 # ever so "helpful" git-svn may shorten it!
241 config_set svnurl "$svnurl"
242 # At this point, since we asked for a standard layout (-s) git-svn
243 # may have been "helpful" and adjusted our $svnurl to a prefix and
244 # then glued the removed suffix onto the front of any svn-remote.svn.*
245 # config items. We could avoid this by not using the '-s' option
246 # but then we might not get all the history. If, for example, we
247 # are cloning an http://svn.example.com/repos/public repository that
248 # early in its history moved trunk => public/trunk we would miss that
249 # earlier history without allowing the funky shorten+prefix behavior.
250 # So we read back the svn-remote.svn.fetch configuration and compute
251 # the prefix. This way we are sure to get the correct prefix.
252 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
253 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
254 gitsvnprefix="${gitsvnfetch%%:*}"
255 gitsvnsuffix="${gitsvnprefix##*/}"
256 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
257 # Ask git-svn to store everything in the normal non-remote
258 # locations being careful to use the correct prefix
259 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
260 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
261 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
262 # look for additional non-standard directories to fetch
263 # check for standard layout at the same time
264 foundstd=
265 foundfile=
266 svn --non-interactive --username anonsvn --password anonsvn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
267 { while read file; do case $file in
268 # skip the already-handled standard ones and any with a space or tab
269 *' '*|*' '*) :;;
270 trunk/|branches/|tags/) foundstd=1;;
271 # only fetch extra directories from the $svnurl root (not any files)
272 *?/) git config --add svn-remote.svn.fetch \
273 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
274 *?) foundfile=1;;
275 esac; done
276 # if files found and no standard directories present use a simpler layout
277 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
278 git config --unset svn-remote.svn.branches
279 git config --unset svn-remote.svn.tags
280 git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
281 fi; }
282 test $? -eq 0
283 # remember the starting time so we can easily combine fetched loose objects
284 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
285 if ! [ -e .svnpack ]; then
286 rm -f .svnpack
287 >.svnpack
288 sleep 1
290 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
291 GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet </dev/null
292 # git svn does not preserve group permissions in the svn subdirectory
293 chmod -R ug+rw,o+r svn
294 # git svn also leaves behind ref turds that end with @nnn
295 # We get rid of them now
296 git for-each-ref --format='%(refname)' |
297 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
298 git_updateref_stdin
299 unset GIT_ASKPASS_PASSWORD
301 darcs://*)
302 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
303 httpurl="http://${url#darcs://}"
304 git_darcs_fetch "$httpurl"
306 bzr://*)
307 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
308 # we just remove bzr:// here, a typical bzr url is just
309 # "lp:foo"
310 bzrurl="${url#bzr://}"
311 git_bzr_fetch "$bzrurl"
313 hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
314 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
315 # We just remove hg+ here, so hg+http://... becomes http://...
316 hgurl="${url#hg+}"
317 # Perform the initial hg clone
318 hg clone -U "$hgurl" "$(pwd)/repo.hg"
319 # Do the fast-export | fast-import
320 git_hg_fetch
323 # We manually add remote.origin.url and remote.origin.fetch
324 # to simulate a `git remote add --mirror=fetch` since that's
325 # not available until Git 1.7.5 and this way we guarantee we
326 # always get exactly the intended configuration and nothing else.
327 git config remote.origin.url "$url"
328 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
329 git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
330 git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
331 git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
332 git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
333 git config --bool girocco.lastupdateclean true
334 else
335 git config --replace-all remote.origin.fetch "+refs/*:refs/*"
336 git config --bool girocco.lastupdateclean false
338 # Set the correct HEAD symref by using ls-remote first
339 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
341 # Since everything was redirected, on failure there'd be no output,
342 # so let's make some failure output
343 cat .pkts-temp
344 echo ""
345 echo "git ls-remote \"$url\" failed"
346 exit 1
348 # Compensate for git() {} side effects
349 unset GIT_TRACE_PACKET
350 # If the server is running at least Git 1.8.4.3 then it will send us the actual
351 # symref for HEAD. If we are running at least Git 1.7.5 then we can snarf that
352 # out of the packet trace data.
353 if [ -s .refs-temp ]; then
354 # Nothing to do unless the remote repository has at least 1 ref
355 # See if we got a HEAD ref
356 head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
357 # If the remote has HEAD set to a symbolic ref that does not exist
358 # then we will not receive a HEAD ref in the ls-remote output
359 headref=
360 showheadwarn=
361 symrefcap=
362 if [ -n "$head" ]; then
363 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
364 "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
365 {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
366 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
367 # matches HEAD otherwise take the first refs/heads/... match
368 matchcnt=0
369 while read ref; do
370 [ -n "$ref" ] || continue
371 matchcnt=$(( $matchcnt + 1 ))
372 if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
373 headref="$ref"
375 if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
376 break
378 done <<-EOT
379 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
380 LC_ALL=C awk '{print $2}')
382 # Warn if there was more than one match and $symrefcap is empty
383 # or $symrefcap is not the same as $headref since our choice might
384 # differ from the source repository's HEAD
385 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
386 { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
387 showheadwarn=1
390 if [ -z "$headref" ]; then
391 # If we still don't have a HEAD ref then prefer refs/heads/master
392 # if it exists otherwise take the first refs/heads/...
393 # We do not support having a detached HEAD.
394 # We always warn now because we will be setting HEAD differently
395 # than the source repository had HEAD set
396 showheadwarn=1
397 while read ref; do
398 [ -n "$ref" ] || continue
399 if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
400 headref="$ref"
402 [ "$headref" != "refs/heads/master" ] || break
403 done <<-EOT
404 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
405 LC_ALL=C awk '{print $2}')
408 # If we STILL do not have a HEAD ref (perhaps the source repository
409 # contains only tags) then use refs/heads/master. It will be invalid
410 # but is no worse than we used to do by default and we'll warn about
411 # it. We do not support a HEAD symref to anything other than refs/heads/...
412 [ -n "$headref" ] || headref="refs/heads/master"
413 git symbolic-ref HEAD "$headref"
414 pruneopt=--prune
415 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
416 # remember the starting time so we can easily detect new packs for fast-import mirrors
417 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
418 if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
419 rm -f .gfipack
420 >.gfipack
421 sleep 1
423 GIT_SSL_NO_VERIFY=1 git remote update $pruneopt
424 if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
425 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
426 rm -f .gfipack
428 else
429 warnempty=1
430 git symbolic-ref HEAD "refs/heads/master"
432 rm -f .refs-temp .pkts-temp
434 esac
436 # The objects subdirectories permissions must be updated now.
437 # In the case of a dumb http clone, the permissions will not be correct
438 # (missing group write) despite the core.sharedrepository=1 setting!
439 # The objects themselves seem to have the correct permissions.
440 # This problem appears to have been fixed in the most recent git versions.
441 perms=g+w
442 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
443 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
445 # We may have just cloned a lot of refs and they will all be
446 # individual files at this point. Let's pack them now so we
447 # can have better performance right from the start.
448 git pack-refs --all
450 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
451 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
452 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
453 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
454 --count=1 refs/heads >info/lastactivity || :
455 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
457 # Don't leave a multi-megabyte useless FETCH_HEAD behind
458 rm -f FETCH_HEAD
460 # Last ditch attempt to get a valid HEAD for a non-git source
461 check_and_set_head || :
463 # The rest
464 echo "Final touches..."
465 git update-server-info
466 trap "" EXIT
468 # run gc now unless the clone is empty
469 if [ -z "$warnempty" ]; then
470 git config --unset gitweb.lastgc 2>/dev/null || :
471 rm -f .delaygc .allowgc
474 emptynote=
475 [ -z "$warnempty" ] ||
476 emptynote="
477 WARNING: You have mirrored an empty repository.
479 headnote=
480 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
481 headnote="
482 NOTE: HEAD has been set to a symbolic ref to \"$headref\".
483 Use the \"Project settings\" link to choose a different HEAD symref.
485 sizenote=
486 ! is_gfi_mirror ||
487 sizenote="
488 NOTE: Since this is a mirror of a non-Git source, the initial repository
489 size may be somewhat larger than necessary. This will be corrected
490 shortly. If you intend to clone this repository you may want to
491 wait up to 1 hour before doing so in order to receive the more
492 compact final size.
494 [ -z "$mailaddrs" ] ||
495 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
496 Congratulations! The clone of project $proj just completed.
498 * Source URL: $url
499 * GitWeb interface: $cfg_gitweburl/$proj.git
500 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
501 $emptynote$headnote$sizenote
502 Have a lot of fun.
505 echo "Mirroring finished successfuly!"
506 # In case this is a re-mirror, lastgc could have been set already so clear it now
507 git config --unset gitweb.lastgc || :
508 rm .clone_in_progress
509 echo "$sizenote@OVER@"