3 # Invoked from taskd/taskd.pl
10 [ "$cfg_permission_control" != "Hooks" ] ||
umask 000
13 # darcs fast-export | git fast-import with error handling
19 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
21 exec 4>&3 3>&1 1>&4 4>&-
24 "$cfg_basedir"/bin/darcs-fast-export \
25 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
31 --export-marks="$(pwd)/gfi-marks" \
32 --export-pack-edges="$(pwd)/gfi-packs" \
33 --force 3>&- || _e2=$?
39 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
43 # bzr fast-export | git fast-import with error handling
51 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
53 exec 4>&3 3>&1 1>&4 4>&-
56 bzr fast-export --plain \
57 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
63 --export-marks="$(pwd)/gfi-marks" \
64 --export-pack-edges="$(pwd)/gfi-packs" \
65 --force 3>&- || _e2=$?
71 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
77 # We must now close the .clonelog file that is open on stdout and stderr
79 failaddrs
="$(config_get owner)" ||
:
80 [ -z "$cfg_admincc" ] ||
[ "$cfg_admincc" = "0" ] ||
[ -z "$cfg_admin" ] ||
81 if [ -z "$failaddrs" ]; then failaddrs
="$cfg_admin"; else failaddrs
="$failaddrs,$cfg_admin"; fi
82 [ -z "$failaddrs" ] ||
85 Condolences. The clone of project $proj just failed.
88 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
90 The project settings link may be used to adjust the settings
91 and restart the clone in order to try the clone again.
93 if [ -f .clonelog
] && [ -r .clonelog
]; then
97 loglines
=$
(LC_ALL
=C
wc -l <.clonelog
)
98 if [ $loglines -le 203 ]; then
101 head -n 100 .clonelog
103 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
105 tail -n 100 .clonelog
108 } | mailref
"clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" ||
:
111 # removes all leftovers from a previous failed clone attempt
112 cleanup_failed_clone
() {
114 # Remove any left-over svn-remote.svn or remote.origin config
115 git config
--remove-section svn-remote.svn
2>/dev
/null ||
:
116 git config
--remove-section remote.origin
2>/dev
/null ||
:
118 # If there is a remote-template.origin section, pre-seed the
119 # remote.origin section with its contents
120 git config
--get-regexp '^remote-template\.origin\..' |
121 while read name value
; do
122 if [ -n "$name" ] && [ -n "$value" ]; then
123 git config
"remote${name#remote-template}" "$value"
127 # Any pre-existing FETCH_HEAD from a previous clone failed or not is
128 # now garbage to be removed
131 # Remove any left-over svn dir from a previous failed attempt
134 # Remove any left-over .darcs dirs from a previous failed attempt
137 # Remove any left-over repo.hg dir from a previous failed attempt
140 # Remove any left-over import/export/temp files from a previous failed attempt
141 rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks
* hg2git-state \
142 gfi-marks gfi-packs .pkts-temp .refs-temp
144 # We want a gc right after the clone, so re-enable that just in case.
145 # There's a potential race where we could add it and gc.sh could remove
146 # it, but we'll reunset lastgc just before we remove .delaygc at the end.
147 [ -e .delaygc
] ||
>.delaygc
148 git config
--unset gitweb.lastgc
2>/dev
/null ||
:
150 # Remove all pre-existing refs
152 git for-each-ref
--format='delete %(refname)' | git_updateref_stdin
2>/dev
/null ||
:
154 # The initial state before a clone starts has HEAD as a symbolic-ref to master
155 git symbolic-ref HEAD refs
/heads
/master
157 # HEAD is no longer "ok"
158 git config
--unset girocco.headok
2>/dev
/null ||
:
160 # We, perhaps, ought to remove any packs/loose objects now, but the next gc
161 # will get rid of any extras. Also, if we're recloning the same thing, any
162 # preexisting packs/loose objects containing what we're recloning will only
163 # speed up the reclone by avoiding some disk writes. So we don't kill them.
167 cd "$cfg_reporoot/$proj.git"
170 ! [ -e .delaygc
] ||
>.allowgc ||
:
172 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
173 echo "Project: $proj"
174 echo " Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
176 [ -n "$cfg_mirror" ] ||
{ echo "Mirroring is disabled" >&2; exit 1; }
177 url
="$(config_get baseurl)" ||
:
178 case "$url" in *" "*|
*" "*|
"")
179 echo "Bad mirror URL (\"$url\")"
185 # Record original mirror type for use by update.sh
186 mirror_type
="$(get_url_mirror_type "$url")"
187 git config girocco.mirrortype
"$mirror_type"
189 echo "Mirroring from URL \"$url\""
192 if [ "$cfg_project_owners" = "source" ]; then
193 config
set owner
"$(ls -ldH "${url#file://}" 2>/dev/null | awk '{print $3}')"
196 mailaddrs
="$(config_get owner)" ||
:
197 [ -z "$cfg_admin" ] ||
198 if [ -z "$mailaddrs" ]; then mailaddrs
="$cfg_admin"; else mailaddrs
="$mailaddrs,$cfg_admin"; fi
200 # Make sure we don't get any unwanted loose objects
201 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
202 # tweak its configuration to prevent that
203 git_add_config
'fetch.unpackLimit=1'
204 # Note the git config documentation is wrong
205 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
206 git_add_config
'transfer.unpackLimit=1'
207 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
208 git_add_config
'fastimport.unpackLimit=0'
211 echo "Initiating mirroring..."
216 svn
://* | svn
+http
://* | svn
+https
://* | svn
+file://* | svn
+ssh://*)
217 [ -n "$cfg_mirror_svn" ] ||
{ echo "Mirroring svn is disabled" >&2; exit 1; }
218 # We just remove svn+ here, so svn+http://... becomes http://...
219 # We also remove a trailing '/' to match what git-svn will do
220 case "$url" in svn
+ssh://*) svnurl
="$url";; *) svnurl
="${url#svn+}";; esac
222 # Use an 'anonsvn' username as is commonly used for anonymous svn
223 # Use an 'anonsvn' password as is commonly used for anonymous svn
224 GIT_ASKPASS_PASSWORD
=anonsvn
225 export GIT_ASKPASS_PASSWORD
226 # We require svn info to succeed on the URL otherwise it's
227 # simply not a valid URL and without using -s on the init it
228 # will not otherwise be tested until the fetch
229 svn
--non-interactive --username anonsvn
--password anonsvn info
"$svnurl" >/dev
/null
230 # We initially use -s for the init which will possibly shorten
231 # the URL. However, the shortening can fail if a password is
232 # not required for the longer version but is for the shorter,
233 # so try again without -s if the -s version fails.
234 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
235 # create a .git subdirectory!
236 GIT_DIR
=. git svn init
--username=anonsvn
--prefix "" -s "$svnurl" </dev
/null ||
237 GIT_DIR
=. git svn init
--username=anonsvn
--prefix "" "$svnurl" </dev
/null
238 # We need to remember this url so we can detect changes because
239 # ever so "helpful" git-svn may shorten it!
240 config_set svnurl
"$svnurl"
241 # At this point, since we asked for a standard layout (-s) git-svn
242 # may have been "helpful" and adjusted our $svnurl to a prefix and
243 # then glued the removed suffix onto the front of any svn-remote.svn.*
244 # config items. We could avoid this by not using the '-s' option
245 # but then we might not get all the history. If, for example, we
246 # are cloning an http://svn.example.com/repos/public repository that
247 # early in its history moved trunk => public/trunk we would miss that
248 # earlier history without allowing the funky shorten+prefix behavior.
249 # So we read back the svn-remote.svn.fetch configuration and compute
250 # the prefix. This way we are sure to get the correct prefix.
251 gitsvnurl
="$(git config --get svn-remote.svn.url)" ||
:
252 gitsvnfetch
="$(git config --get-all svn-remote.svn.fetch | tail -1)" ||
:
253 gitsvnprefix
="${gitsvnfetch%%:*}"
254 gitsvnsuffix
="${gitsvnprefix##*/}"
255 gitsvnprefix
="${gitsvnprefix%$gitsvnsuffix}"
256 # Ask git-svn to store everything in the normal non-remote
257 # locations being careful to use the correct prefix
258 git config
--replace-all svn-remote.svn.fetch
"${gitsvnprefix}trunk:refs/heads/master"
259 git config
--replace-all svn-remote.svn.branches
"${gitsvnprefix}branches/*:refs/heads/*"
260 git config
--replace-all svn-remote.svn.tags
"${gitsvnprefix}tags/*:refs/tags/*"
261 # look for additional non-standard directories to fetch
262 # check for standard layout at the same time
265 svn
--non-interactive --username anonsvn
--password anonsvn
ls "$gitsvnurl/${gitsvnprefix}" 2>/dev
/null |
266 { while read file; do case $file in
267 # skip the already-handled standard ones and any with a space or tab
269 trunk
/|branches
/|tags
/) foundstd
=1;;
270 # only fetch extra directories from the $svnurl root (not any files)
271 *?
/) git config
--add svn-remote.svn.fetch \
272 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
275 # if files found and no standard directories present use a simpler layout
276 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
277 git config
--unset svn-remote.svn.branches
278 git config
--unset svn-remote.svn.tags
279 git config
--replace-all svn-remote.svn.fetch
':refs/heads/master'
282 # remember the starting time so we can easily combine fetched loose objects
283 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
284 if ! [ -e .svnpack
]; then
289 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
290 GIT_DIR
=. git svn fetch
--log-window-size=$var_log_window_size --username=anonsvn
--quiet </dev
/null
291 # git svn does not preserve group permissions in the svn subdirectory
292 chmod -R ug
+rw
,o
+r svn
293 # git svn also leaves behind ref turds that end with @nnn
294 # We get rid of them now
295 git for-each-ref
--format='%(refname)' |
296 LC_ALL
=C
sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
298 unset GIT_ASKPASS_PASSWORD
301 [ -n "$cfg_mirror_darcs" ] ||
{ echo "Mirroring darcs is disabled" >&2; exit 1; }
302 httpurl
="http://${url#darcs://}"
303 git_darcs_fetch
"$httpurl"
306 [ -n "$cfg_mirror_bzr" ] ||
{ echo "Mirroring bzr is disabled" >&2; exit 1; }
307 # we just remove bzr:// here, a typical bzr url is just
309 bzrurl
="${url#bzr://}"
310 git_bzr_fetch
"$bzrurl"
312 hg
+http
://* | hg
+https
://* | hg
+file://* | hg
+ssh://*)
313 [ -n "$cfg_mirror_hg" ] ||
{ echo "Mirroring hg is disabled" >&2; exit 1; }
314 # We just remove hg+ here, so hg+http://... becomes http://...
316 # Perform the initial hg clone
317 hg clone
-U "$hgurl" "$(pwd)/repo.hg"
318 # Do the fast-export | fast-import
322 # We manually add remote.origin.url and remote.origin.fetch
323 # to simulate a `git remote add --mirror=fetch` since that's
324 # not available until Git 1.7.5 and this way we guarantee we
325 # always get exactly the intended configuration and nothing else.
326 git config remote.origin.url
"$url"
327 if ! is_gfi_mirror_url
"$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
328 git config
--replace-all remote.origin.fetch
"+refs/heads/*:refs/heads/*"
329 git config
--add remote.origin.fetch
"+refs/tags/*:refs/tags/*"
330 git config
--add remote.origin.fetch
"+refs/notes/*:refs/notes/*"
331 git config
--add remote.origin.fetch
"+refs/top-bases/*:refs/top-bases/*"
332 git config
--bool girocco.lastupdateclean true
334 git config
--replace-all remote.origin.fetch
"+refs/*:refs/*"
335 git config
--bool girocco.lastupdateclean false
337 # Set the correct HEAD symref by using ls-remote first
338 GIT_SSL_NO_VERIFY
=1 GIT_TRACE_PACKET
=1 git ls-remote origin
>.refs-temp
2>.pkts-temp ||
340 # Since everything was redirected, on failure there'd be no output,
341 # so let's make some failure output
344 echo "git ls-remote \"$url\" failed"
347 # Compensate for git() {} side effects
348 unset GIT_TRACE_PACKET
349 # If the server is running at least Git 1.8.4.3 then it will send us the actual
350 # symref for HEAD. If we are running at least Git 1.7.5 then we can snarf that
351 # out of the packet trace data.
352 if [ -s .refs-temp
]; then
353 # Nothing to do unless the remote repository has at least 1 ref
354 # See if we got a HEAD ref
355 head="$(LC_ALL=C grep -E "^
$octet20$hexdig*[ $tab]+HEAD\$
" <.refs-temp | LC_ALL=C awk '{print $1}')"
356 # If the remote has HEAD set to a symbolic ref that does not exist
357 # then we will not receive a HEAD ref in the ls-remote output
361 if [ -n "$head" ]; then
362 symrefcap
="$(LC_ALL=C sed -ne <.pkts-temp \
363 "/packet
:.
*git
<.
*[ $tab]symref
="'HEAD:refs\/heads\/'"[^
$tab]/\
364 {s
/^.
*[ $tab]symref
="'HEAD:\(refs\/heads\/'"[^
$tab][^
$tab]*"'\).*$/\1/;p;}')"
365 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
366 # matches HEAD otherwise take the first refs/heads/... match
369 [ -n "$ref" ] ||
continue
370 matchcnt
=$
(( $matchcnt + 1 ))
371 if [ -z "$headref" ] ||
[ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
374 if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
378 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
379 LC_ALL=C awk '{print $2}')
381 # Warn if there was more than one match and $symrefcap is empty
382 # or $symrefcap is not the same as $headref since our choice might
383 # differ from the source repository's HEAD
384 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
385 { [ -n "$symrefcap" ] ||
[ $matchcnt -gt 1 ]; }; then
389 if [ -z "$headref" ]; then
390 # If we still don't have a HEAD ref then prefer refs/heads/master
391 # if it exists otherwise take the first refs/heads/...
392 # We do not support having a detached HEAD.
393 # We always warn now because we will be setting HEAD differently
394 # than the source repository had HEAD set
397 [ -n "$ref" ] ||
continue
398 if [ -z "$headref" ] ||
[ "$ref" = "refs/heads/master" ]; then
401 [ "$headref" != "refs/heads/master" ] ||
break
403 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
404 LC_ALL=C awk '{print $2}')
407 # If we STILL do not have a HEAD ref (perhaps the source repository
408 # contains only tags) then use refs/heads/master. It will be invalid
409 # but is no worse than we used to do by default and we'll warn about
410 # it. We do not support a HEAD symref to anything other than refs/heads/...
411 [ -n "$headref" ] || headref
="refs/heads/master"
412 git symbolic-ref HEAD
"$headref"
414 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt
=
415 # remember the starting time so we can easily detect new packs for fast-import mirrors
416 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
417 if is_gfi_mirror_url
"$url" && ! [ -e .gfipack
]; then
422 GIT_SSL_NO_VERIFY
=1 git remote update
$pruneopt
423 if [ -e .gfipack
] && is_gfi_mirror_url
"$url"; then
424 find -L objects
/pack
-type f
-newer .gfipack
-name "pack-$octet20*.pack" -print >>gfi-packs
429 git symbolic-ref HEAD
"refs/heads/master"
431 rm -f .refs-temp .pkts-temp
435 # The objects subdirectories permissions must be updated now.
436 # In the case of a dumb http clone, the permissions will not be correct
437 # (missing group write) despite the core.sharedrepository=1 setting!
438 # The objects themselves seem to have the correct permissions.
439 # This problem appears to have been fixed in the most recent git versions.
441 [ "$cfg_permission_control" != "Hooks" ] || perms
=go
+w
442 chmod $perms $
(find -L objects
-maxdepth 1 -type d
) 2>/dev
/null ||
:
444 # We may have just cloned a lot of refs and they will all be
445 # individual files at this point. Let's pack them now so we
446 # can have better performance right from the start.
449 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
450 git config gitweb.lastreceive
"$(date '+%a, %d %b %Y %T %z')"
451 git config gitweb.lastchange
"$(date '+%a, %d %b %Y %T %z')"
452 git for-each-ref
--sort=-committerdate --format='%(committerdate:iso8601)' \
453 --count=1 refs
/heads
>info
/lastactivity ||
:
454 ! [ -d htmlcache
] ||
{ >htmlcache
/changed
; } 2>/dev
/null ||
:
456 # Don't leave a multi-megabyte useless FETCH_HEAD behind
459 # Last ditch attempt to get a valid HEAD for a non-git source
460 check_and_set_head ||
:
463 echo "Final touches..."
464 git update-server-info
467 # run gc now unless the clone is empty
468 if [ -z "$warnempty" ]; then
469 git config
--unset gitweb.lastgc
2>/dev
/null ||
:
470 rm -f .delaygc .allowgc
474 [ -z "$warnempty" ] ||
476 WARNING: You have mirrored an empty repository.
479 [ -z "$showheadwarn" ] ||
[ -z "$headref" ] ||
481 NOTE: HEAD has been set to a symbolic ref to \"$headref\".
482 Use the \"Project settings\" link to choose a different HEAD symref.
487 NOTE: Since this is a mirror of a non-Git source, the initial repository
488 size may be somewhat larger than necessary. This will be corrected
489 shortly. If you intend to clone this repository you may want to
490 wait up to 1 hour before doing so in order to receive the more
493 [ -z "$mailaddrs" ] ||
494 mailref
"clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
495 Congratulations! The clone of project $proj just completed.
498 * GitWeb interface: $cfg_gitweburl/$proj.git
499 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
500 $emptynote$headnote$sizenote
504 echo "Mirroring finished successfuly!"
505 # In case this is a re-mirror, lastgc could have been set already so clear it now
506 git config
--unset gitweb.lastgc ||
:
507 rm .clone_in_progress
508 echo "$sizenote@OVER@"