8 echo "Usage: update.sh projname" >&2
12 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
13 datefmt
='+%a, %d %b %Y %T %z'
15 # freshen_loose_objects full-sha ...
16 # if "$n" is a loose object, set its modification time to now
17 # otherwise silently do nothing with no error. To facilitate conversion
18 # of mirror projects to push projects we also add group write permission.
19 freshen_loose_objects
() {
24 _list
="$_list objects/$_shard/$_fn"
26 if [ -n "$_list" ]; then
27 chmod ug
+w
$_list 2>/dev
/null ||
:
28 touch -c $_list 2>/dev
/null ||
:
32 # darcs fast-export | git fast-import with error handling
38 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
40 exec 4>&3 3>&1 1>&4 4>&-
43 "$cfg_basedir"/bin/darcs-fast-export \
44 --export-marks="$(pwd)/dfe-marks" \
45 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
51 --export-marks="$(pwd)/gfi-marks" \
52 --export-pack-edges="$(pwd)/gfi-packs" \
53 --import-marks="$(pwd)/gfi-marks" \
54 --force 3>&- || _e2=$?
60 [ "$_err1" = 0 -a "$_err2" = 0 ]
64 # bzr fast-export | git fast-import with error handling
72 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
74 exec 4>&3 3>&1 1>&4 4>&-
77 bzr fast-export --plain \
78 --export-marks="$(pwd)/dfe-marks" \
79 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
85 --export-marks="$(pwd)/gfi-marks" \
86 --export-pack-edges="$(pwd)/gfi-packs" \
87 --import-marks="$(pwd)/gfi-marks" \
88 --force 3>&- || _e2=$?
94 [ "$_err1" = 0 -a "$_err2" = 0 ]
98 [ -n "$cfg_mirror" ] ||
{ echo "Mirroring is disabled" >&2; exit 0; }
101 [ "$cfg_permission_control" != "Hooks" ] ||
umask 000
105 cd "$cfg_reporoot/$proj.git"
107 trap 'if [ $? != 0 ]; then echo "update failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
111 if check_interval lastrefresh
$cfg_min_mirror_interval; then
112 progress
"= [$proj] update skip (last at $(config_get lastrefresh))"
115 progress
"+ [$proj] update (`date`)"
117 # Any pre-existing FETCH_HEAD from a previous fetch failed or not is garbage
120 # A previous failed update attempt can leave a huge tmp_pack_XXXXXX file behind.
121 # Since no pushes are allowed to mirrors, we know that any such files that exist
122 # at this point in time are garbage and can be safely deleted, we do not even
123 # need to check how old they are. A tmp_idx_XXXXXX file is also created during
124 # the later stages of the fetch process, so we kill any of those as well.
125 find objects
/pack
-maxdepth 1 -type f
-name "tmp_pack_?*" -print0 |
xargs -0 rm -f
126 find objects
/pack
-maxdepth 1 -type f
-name "tmp_idx_?*" -print0 |
xargs -0 rm -f
128 # Make sure we have a reflogs subdirectory and abort the update if not
129 # This should not count as a normal "bang" failure if unsuccessful
130 [ -d reflogs
] || mkdir
-p reflogs
>/dev
/null
2>&1 ||
:
138 # Since gitweb shows the .last_refresh date, it's safe to update
139 # gitweb.lastrefresh to throttle the updates w/o corrupting the
140 # last refresh date display on the gitweb summary page
141 # It's therefore important that we do NOT touch .last_refresh here
142 config_set lastrefresh
"$(date "$datefmt")"
146 bang
echo "Project: $proj"
148 mail="$(config_get owner || :)"
149 url
="$(config_get baseurl || :)"
150 case "$url" in *" "*|
*" "*|
"")
151 bang_eval
'echo "Bad mirror URL (\"$url\")"; ! :'
154 bang
echo "Mirroring from URL \"$url\""
156 statusok
="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
158 [ "$statusok" = "false" -o -z "$mail" ] || mailaddrs
="$mail"
159 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] || \
160 if [ -z "$mailaddrs" ]; then mailaddrs
="$cfg_admin"; else mailaddrs
="$mailaddrs,$cfg_admin"; fi
162 bang_eval
"git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
163 bang_eval
"LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-before"
164 ! [ -e .delaygc
] ||
>.allowgc ||
:
168 svn
://* | svn
+http
://* | svn
+https
://* | svn
+file://* | svn
+ssh://*)
169 [ -n "$cfg_mirror_svn" ] ||
{ echo "Mirroring svn is disabled" >&2; exit 0; }
170 # Use an 'anonsvn' username as is commonly used for anonymous svn
171 # Use an 'anonsvn' password as is commonly used for anonymous svn
172 GIT_ASKPASS_PASSWORD
=anonsvn
173 export GIT_ASKPASS_PASSWORD
174 # Update the git svn url to match baseurl but be cognizant of any
175 # needed prefix changes. See the comments in taskd/clone.sh about
176 # why we need to put up with a prefix in the first place.
177 case "$url" in svn
+ssh://*) svnurl
="$url";; *) svnurl
="${url#svn+}";; esac
179 svnurlold
="$(config_get svnurl || :)"
180 if [ "$svnurl" != "$svnurlold" ]; then
181 # We better already have an svn-remote.svn.fetch setting
182 bang
test -n "$(git config --get-all svn-remote.svn.fetch || :)"
183 # the only way to truly know what the proper prefix is
184 # is to attempt a fresh git-svn init -s on the new url
185 rm -rf svn-new-url ||
:
186 # We require svn info to succeed on the URL otherwise it's
187 # simply not a valid URL and without using -s on the init it
188 # will not otherwise be tested until the fetch
189 bang
eval 'svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" > /dev/null'
190 bang mkdir svn-new-url
191 GIT_DIR
=svn-new-url bang git init
--bare --quiet
192 # We initially use -s for the init which will possibly shorten
193 # the URL. However, the shortening can fail if a password is
194 # not required for the longer version but is for the shorter,
195 # so try again without -s if the -s version fails.
196 cmdstr
='git svn init --username=anonsvn --prefix "" -s "$svnurl" < /dev/null > /dev/null 2>&1 || '
197 cmdstr
="$cmdstr"'git svn init --username=anonsvn --prefix "" "$svnurl" < /dev/null > /dev/null 2>&1'
198 GIT_DIR
=svn-new-url bang
eval "$cmdstr"
199 gitsvnurl
="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url || :)"
200 gitsvnfetch
="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch || :)"
201 gitsvnprefixnew
="${gitsvnfetch%%:*}"
202 gitsvnsuffixnew
="${gitsvnprefixnew##*/}"
203 gitsvnprefixnew
="${gitsvnprefixnew%$gitsvnsuffixnew}"
204 rm -rf svn-new-url ||
:
205 # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
207 if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
208 # The url has been changed.
209 # We must update the url and replace the prefix on all config items
210 gitsvnfetch
="$(git config --get-all svn-remote.svn.fetch | head -1 || :)"
211 gitsvnprefixold
="${gitsvnfetch%%:*}"
212 gitsvnsuffixold
="${gitsvnprefixold##*/}"
213 gitsvnprefixold
="${gitsvnprefixold%$gitsvnsuffixold}"
214 git config
--remove-section 'svn-remote.svnnew' 2>/dev
/null ||
:
215 git config
'svn-remote.svnnew.url' "$gitsvnurl"
216 { git config
--get-regexp '^svn-remote\.svn\.' ||
:; } | \
217 { while read sname sval
; do
219 svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags
)
220 sname
="${sname#svn-remote.svn.}"
221 sval
="${sval#$gitsvnprefixold}"
222 bang git config
--add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
225 bang git config
-f svn
/.metadata svn-remote.svn.reposRoot
"$gitsvnurl"
226 bang git config
--remove-section svn-remote.svn
227 bang git config
--rename-section svn-remote.svnnew svn-remote.svn
229 bang config_set svnurl
"$svnurl"
231 # remove any stale *.lock files greater than 1 hour old in case
232 # git-svn was killed on the last update because it took too long
233 find svn
-type f
-name '*.lock' -mmin +60 -print0 2>/dev
/null |
xargs -0 rm -f
234 # remember the starting time so we can easily combine fetched loose objects
235 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
236 if ! [ -e .svnpack
]; then
242 GIT_DIR
=. bang git svn fetch
--log-window-size=$var_log_window_size --username=anonsvn
--quiet < /dev
/null
243 # git svn does not preserve group permissions in the svn subdirectory
244 chmod -R ug
+rw
,o
+r svn
245 # git svn also leaves behind ref turds that end with @nnn
246 # We get rid of them now
247 git for-each-ref
--format='%(objectname) %(refname)' | \
248 { while read sha1 ref
; do
250 ?
*@
[1-9]|?
*@
[1-9][0-9]|?
*@
[1-9][0-9][0-9]|?
*@
[1-9][0-9][0-9][0-9]|\
251 ?
*@
[1-9][0-9][0-9][0-9][0-9]|?
*@
[1-9][0-9][0-9][0-9][0-9][0-9]|\
252 ?
*@
[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\
253 ?
*@
[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])
254 git update-ref
-d "$ref"
257 unset GIT_ASKPASS_PASSWORD
260 [ -n "$cfg_mirror_darcs" ] ||
{ echo "Mirroring darcs is disabled" >&2; exit 0; }
261 httpurl
="http://${url#darcs://}"
262 # remove any stale lock files greater than 1 hour old in case
263 # darcs_fast_export was killed on the last update because it took too long
264 find *.darcs
-maxdepth 2 -type f
-name 'lock' -mmin +60 -print0 2>/dev
/null |
xargs -0 rm -f
265 bang git_darcs_fetch
"$httpurl"
268 [ -n "$cfg_mirror_bzr" ] ||
{ echo "Mirroring bzr is disabled" >&2; exit 0; }
269 bzrurl
="${url#bzr://}"
270 bang git_bzr_fetch
"$bzrurl"
272 hg
+http
://* | hg
+https
://* | hg
+file://* | hg
+ssh://*)
273 [ -n "$cfg_mirror_hg" ] ||
{ echo "Mirroring hg is disabled" >&2; exit 0; }
274 # We just remove hg+ here, so hg+http://... becomes http://...
276 # Fetch any new updates
277 bang hg
-R "$(pwd)/repo.hg" pull
278 # Do the fast-export | fast-import
282 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url
"$url"
284 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt
=
285 git_add_config
'fetch.unpackLimit=1'
286 # Note the git config documentation is wrong
287 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
288 git_add_config
'transfer.unpackLimit=1'
289 if ! is_gfi_mirror_url
"$url"; then
291 [ "$(git config --bool girocco.lastupdateclean 2>/dev/null || :)" != "true" ] || lastwasclean
=1
293 [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" != "true" ] || nextisclean
=1
294 if [ "$nextisclean" != "$lastwasclean" ]; then
295 if [ -n "$nextisclean" ]; then
296 git config
--replace-all remote.origin.fetch
"+refs/heads/*:refs/heads/*"
297 git config
--add remote.origin.fetch
"+refs/tags/*:refs/tags/*"
298 git config
--add remote.origin.fetch
"+refs/notes/*:refs/notes/*"
300 git config
--replace-all remote.origin.fetch
"+refs/*:refs/*"
304 # remember the starting time so we can easily detect new packs for fast-import mirrors
305 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
306 if is_gfi_mirror_url
"$url" && [ ! -e .gfipack
]; then
311 GIT_SSL_NO_VERIFY
=1 bang git remote update
$pruneopt
312 if ! is_gfi_mirror_url
"$url" && [ "$nextisclean" != "$lastwasclean" ]; then
313 if [ -n "$nextisclean" ]; then
314 # We must manually purge the unclean refs now as even prune won't do it
315 git for-each-ref
--format='%(refname)' |
316 while read -r aref
; do
317 case "$aref" in refs
/heads
/*|refs
/tags
/*|refs
/notes
/*) :;; *)
318 git update-ref
-d "$aref"
322 git config
--bool girocco.lastupdateclean
${nextisclean:-0}
324 if [ -e .gfipack
] && is_gfi_mirror_url
"$url"; then
325 find objects
/pack
-type f
-newer .gfipack
-name "pack-$octet20.pack" -print >>gfi-packs
331 # The objects subdirectories permissions must be updated now.
332 # In the case of a dumb http clone, the permissions will not be correct
333 # (missing group write) despite the core.sharedrepository=1 setting!
334 # The objects themselves seem to have the correct permissions.
335 # This problem appears to have been fixed in the most recent git versions.
337 [ "$cfg_permission_control" != "Hooks" ] || perms
=go
+w
338 chmod $perms $
(find objects
-maxdepth 1 -type d
) 2>/dev
/null ||
:
340 bang git update-server-info
342 # We maintain the last refresh date in two places deliberately
343 # so that it's available as part of the config data and also
344 # as a standalone file timestamp that can be accessed without git.
345 bang config_set lastrefresh
"$(date "$datefmt")"
346 { >.last_refresh
; } 2>/dev
/null ||
:
348 # Check to see if any refs changed
349 bang_eval
"git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
350 bang_eval
"LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-after"
352 cmp -s .refs-before .refs-after || refschanged
=1
354 # There's no way to get git svn to fetch packs, so we always need to run
355 # a mini-gc afterwards if svn actually fetched anything, but that's okay
356 # because it will be very quick
357 if [ -z "$refschanged" -a -n "$svnpackcreated" ]; then
358 # we created the .svnpack but didn't actually fetch anything
359 # so remove it to avoid forcing a mini-gc if not necessary
362 if [ -e .svnpack
] && ! [ -e .needsgc
]; then
366 # Force a mini-gc if $Girocco::Config::delay_gfi_redelta is false and there's
367 # at least one gfi pack present now
368 if [ -z "$cfg_delay_gfi_redelta" ] && ! [ -e .needsgc
] && \
369 [ -f gfi-packs
-a -s gfi-packs
] && is_gfi_mirror_url
"$url"; then
373 # Activate a mini-gc if there are at least 20 packs present now
374 if ! [ -e .needsgc
]; then
376 { packs
="$(list_packs --quiet --count --exclude-no-idx objects/pack || :)" ||
:; } 2>/dev
/null
377 if [ -n "$packs" ] && [ "$packs" -ge 20 ]; then
382 # Look at which refs changed and trigger ref-change for these
383 sockpath
="$cfg_chroot/etc/taskd.socket"
384 if [ -n "$refschanged" ]; then
385 bang config_set lastreceive
"$(date '+%a, %d %b %Y %T %z')"
386 # We always use UTC for the log timestamp so that chroot and non-chroot match up.
387 # We don't have to worry about multiple log files since only one update runs
388 lognamets
="$(TZ=UTC date '+%Y%m%d_%H%M%S')"
389 loghhmmss
="${lognamets##*_}"
390 logname
="reflogs/${lognamets%%_*}"
391 # We freshen the mod time to now on any old or new ref that is a loose object
392 # For old refs we do it so we will be able to keep them around for 1 day
393 # For new refs we do it in case we are about to run gc and the new ref
394 # actually points to an oldish loose object that had been unreachable
395 # We probably do not need to do it for new refs as Git tries to do that,
396 # but since we're already doing it for old refs (which Git does not do),
397 # it's almost no extra work for new refs, just in case.
399 echo "ref-changes %$proj% $proj"
400 LC_ALL
=C
join .refs-before .refs-after |
401 LC_ALL
=C
sed -e '/^[^ ][^ ]* \([^ ][^ ]*\) \1$/d' |
402 while read ref old new
; do
403 echo "$loghhmmss $old $new $ref" >&3
404 freshen_loose_objects
"$old" "$new"
405 echo "$old $new $ref"
407 LC_ALL
=C
join -v 1 .refs-before .refs-after |
408 while read ref old
; do
409 echo "$loghhmmss $old 0000000000000000000000000000000000000000 $ref" >&3
410 freshen_loose_objects
"$old"
411 echo "$old 0000000000000000000000000000000000000000 $ref"
413 LC_ALL
=C
join -v 2 .refs-before .refs-after |
414 while read ref new
; do
415 echo "$loghhmmss 0000000000000000000000000000000000000000 $new $ref" >&3
416 freshen_loose_objects
"$new"
417 echo "0000000000000000000000000000000000000000 $new $ref"
419 git for-each-ref
--format='%(objectname) %(objectname) %(refname)' refs
/heads
420 } >.refs-temp
3>>"$logname"
421 if [ -S "$sockpath" ]; then
422 nc_openbsd
-w 15 -U "$sockpath" <.refs-temp ||
:
424 bang config_set lastchange
"$(date '+%a, %d %b %Y %T %z')"
425 bang_eval
"git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
426 --count=1 refs/heads > info/lastactivity"
427 ! [ -d htmlcache
] ||
{ >htmlcache
/changed
; } 2>/dev
/null ||
:
428 # While unlikely, it is conceivable that several ref updates have occurred that
429 # did not actually create any packs. In that case we could build up a large
430 # number of log files so request gc if there are more than 50 of them now.
431 # In the case of a mirror this is a practical impossibility but we check anyway.
432 if ! [ -e .needsgc
]; then
434 { logfiles
="$(($(find reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" ||
:; } 2>/dev
/null
435 if [ -n "$logfiles" ] && [ "$logfiles" -ge 50 ]; then
439 rm -f .delaygc .allowgc
442 # If the repository does not yet have a valid HEAD symref try to set one
443 # If an empty repository was cloned and then later becomes unempty you just
444 # lose out on the fancy "symref=HEAD:" logic and get this version instead
445 check_and_set_head ||
:
447 rm -f .refs-before .refs-after .refs-temp FETCH_HEAD
450 [ -z "$mailaddrs" ] ||
! was_banged_message_sent ||
452 echo "$proj update succeeded - failure recovery"
453 echo "this status message may be disabled on the project admin page"
454 } |
mail -s "[$cfg_name] $proj update succeeded" "$mailaddrs" ||
:
458 progress
"- [$proj] update (`date`)"