3 # NOTE: additional options can be passed to git repack by specifying
4 # them after the project name, for example:
12 echo "Usage: gc.sh projname [extra-repack-args]" >&2
17 packopts
="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
18 quiet
=; [ -n "$show_progress" ] || quiet
=-q
21 [ "$cfg_permission_control" != "Hooks" ] ||
umask 000
25 if _result
="$(kill -0 "$1" 2>&1)"; then
26 # process exists and we have permission to signal it
29 case "$_result" in *"not permitted"*)
30 # we do not have permission to signal the process
33 # process does not exist
38 # A .lock file should only exist for much less than a second.
39 # If we see a stale lock file (> 1h old), remove it and then,
40 # just in case, wait 30 seconds for any process whose .lock
41 # we might have just removed (it's racy) to finish doing what
42 # should take much less than a second to do.
43 _stalelock
="$(find "$1.lock
" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
44 if [ -n "$_stalelock" ]; then
49 if (set -C; > "$1.lock") 2>/dev
/null
; then
54 [ "$_try" != "p" ] ||
sleep 1
56 # cannot create lock file
60 # The pre-receive script creates one ref log file per push but we want them to
61 # be coalesced into one ref log file per day. We are guaranteed that any files
62 # we find to coalesce are NOT currently being written to since they are always
63 # written first as temporary files and then moved into place. We attempt to
64 # transfer the most recent modification time to the coalesced log file which
65 # would step on its mod time if it were being written to directly, but if we
66 # find per-process ref log files then it must be a push project and the only
67 # thing that would write directly to the main per-day log file would be a
68 # mirror project so there's actually no conflict.
70 [ -d reflogs
] ||
return 0
72 find reflogs
-maxdepth 1 -type f
-name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*" -print | LC_ALL
=C
sort |
73 while read -r rname
; do
74 logname
="${rname%%_*}"
75 cat "$rname" >>"$logname"
76 touch -r "$rname" "$logname"
78 if [ -e "$rname" ]; then
80 echo "! [$proj] failed to remove $rname" >&2
81 exit 1 # will only exit subshell created by "|"
87 # Remove any files in reflogs that are older than $cfg_reflogs_lifetime days
89 [ -d reflogs
] ||
return 0
90 exp
="$(( ${cfg_reflogs_lifetime:-1} * 1440 ))"
91 [ $exp -gt 0 ] || exp
=1440
92 [ $exp -le 43200 ] || exp
=43200
93 find reflogs
-maxdepth 1 -type f
-mmin "+$exp" -print0 |
xargs -0 rm -f
96 # return true if there's more than one objects/pack-<sha>.pack file or
97 # ANY sha-1 files in objects
99 _packs
=$
(find objects
/pack
-type f
-name "pack-$octet20.pack" -print |
head -n 2 | LC_ALL
=C
wc -l)
100 if [ $_packs != 1 ] && [ $_packs != 0 ]; then
103 _objs
=$
(find objects
/$octet -type f
-name "$octet19" -print 2>/dev
/null |
head -n 1 | LC_ALL
=C
wc -l)
107 # make sure combine-packs uses the correct Git executable
108 run_combine_packs
() {
109 PATH
="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@
/jobd
/combine-packs.sh
"$@"
112 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
113 # input pack names are read from standard input one per line delimited by the first
114 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
115 # all arguments, if any, are passed to pack-objects as additional options
116 # returns non-zero on failure AND creates .gc_failed in that case
119 find objects
/pack
-maxdepth 1 -type f
-name '*.zap*' -print0 |
xargs -0 rm -f
120 run_combine_packs
--replace "$@" $packopts --all-progress-implied $quiet --non-empty ||
{
127 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
129 [ -n "$gfi_mirror" ] ||
return 0
130 [ -d objects
/pack
] ||
{ rm -f gfi-packs
; return 0; }
131 progress
"~ [$proj] redeltifying poor quality git fast-import packs"
132 combine_packs
--ignore-missing --no-reuse-delta <gfi-packs
137 # combine small packs into larger pack(s)
138 # we avoid any keep, bndl or bitmap packs
139 # if the optional argument is non-empty even a single small pack will be redeltad
140 combine_small_packs
() {
143 if [ -n "$1" ] && [ -n "$noreusedeltaopt" ]; then
146 _lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
147 _lpo
="$_lpo --object-limit $var_redelta_threshold objects/pack"
149 _cnt
="$(list_packs --count $_lpo || :)"
150 test "${_cnt:-0}" -ge $_minsmallpacks
152 [ -n "$_didprogress" ] ||
{
153 progress
"~ [$proj] combining small packs into a single larger pack"
156 _newp
="$(list_packs $_lpo | combine_packs --names $noreusedeltaopt)"
157 _newc
="$(echo $(echo "$_newp" | LC_ALL=C wc -w))"
158 # be paranoid and exit the loop if we haven't reduced the number of packs
159 [ $_newc -lt $_cnt ] ||
break
165 # Unfortunately git-svn lacks the ability to store newly fetched revisions as a pack.
166 # However, the fetch code conveniently sets .svnpack just before it runs git-svn fetch
167 # so that it's easy to find all the objects that have been fetched by git-svn and
168 # combine them into a pack. The --no-reuse-delta option is meaningless here since
169 # everything to be packed is a loose object and therefore not a delta so deltification
170 # will always take place.
172 [ -f .svnpack
] && [ -n "$svn_mirror" ] ||
return 0
174 mv -f .svnpack .svnpackgc
175 progress
"~ [$proj] combining loose git-svn objects into a pack"
176 _newp
="$(find objects/$octet -maxdepth 1 -type f -newer .svnpackgc -name "$octet19" -print 2>/dev/null |
177 LC_ALL=C awk -F / '{print $2 $3}' |
178 run_combine_packs --objects --names $packopts --incremental --all-progress-implied $quiet --non-empty)" ||
{
179 mv -f .svnpackgc .svnpack
183 if [ -n "$_newp" ]; then
184 # remove the now-redundant loose objects -- this is always safe
185 # even during a concurrent push because a reprepare_packed_git
186 # will be triggered if an object that should be there is not
187 # found thereby finding it in the new pack instead
188 git prune-packed
$quiet
193 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
195 # Must have a matching .idx file and a non-empty packed-refs file
196 [ -s "${1%.pack}.idx" ] ||
return 1
197 [ -s "$2" ] ||
return 1
203 "ref: refs/"?
*|
"ref:refs/"?
*|
"refs/"?
*)
204 _headmatch
="${3#ref:}"
205 _headmatch
="${_headmatch# }"
206 _headmatchpat
="$(echo "$_headmatch" | LC_ALL=C sed -e 's/\([.$]\)/\\\1/g')"
207 _headsha
="$(LC_ALL=C grep -e "^
$octet20 $_headmatchpat\$
" < "$2" | \
208 LC_ALL=C cut -d ' ' -f 1)"
209 case "$_headsha" in $octet20) :;; *)
217 rm -rf pack_is_complete_test
218 mkdir pack_is_complete_test
219 mkdir pack_is_complete_test
/refs
220 mkdir pack_is_complete_test
/objects
221 mkdir pack_is_complete_test
/objects
/pack
222 echo "$_headsha" > pack_is_complete_test
/HEAD
223 ln -s "$1" pack_is_complete_test
/objects
/pack
/
224 ln -s "${1%.pack}.idx" pack_is_complete_test
/objects
/pack
/
225 ln -s "$2" pack_is_complete_test
/packed-refs
226 _count
="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
227 rm -rf pack_is_complete_test
228 [ -n "$_count" ] ||
return 1
229 [ "$_count" -gt 0 ] 2>/dev
/null ||
return 1
233 # On return a "$lockf" will have been created that must be removed when gc is done
235 # be compatibile with gc.pid file from newer Git releases
239 if [ "$(createlock "$lockf")" ]; then
241 # 1) less than 12 hours old
242 # 2) contains two fields (pid hostname) NO trailing NL
243 # 3) the hostname is different OR the pid is still alive
244 # then we exit as another active process is holding the lock
245 if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
248 read -r apid ahost ajunk
< "$lockf" ||
:
249 if [ "$apid" ] && [ "$ahost" ]; then
250 if [ "$ahost" != "$hn" ] || pidactive
"$apid"; then
256 echo >&2 "[$proj] unable to create gc.pid.lock file"
259 if [ -n "$active" ]; then
261 echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
264 printf "%s %s" "$$" "$hn" > "$lockf.lock"
265 chmod 0664 "$lockf.lock"
266 mv -f "$lockf.lock" "$lockf"
269 # Remove any crud that's been left behind by interrupted operations
270 # that did not clean up after themselves
272 # Remove any existing FETCH_HEAD
273 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
274 # receiving pushes (those never create a FETCH_HEAD).
275 # And if we're fetching because we're a mirror, we know we're not fetching right
276 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
277 # Therefore any existing FETCH_HEAD is junk. And it may be many megabytes if
278 # there were a lot of refs.
281 # Remove any stale pack remnants that are more than an hour old.
282 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
283 # .pack AND the corresponding .pack DOES NOT exist. A bunch of stale
284 # pack-<sha1>.idx files without their corresponding .pack files are worthless
285 # and just waste space. Normally there shouldn't be any remnants but actually
286 # this can happen when things are interrupted at just the wrong time.
287 # Note that the objects/pack directory is created by git init and should
289 find objects
/pack
-maxdepth 1 -type f
-mmin +60 -name "pack-$octet20.?*" -print | \
290 LC_ALL
=C
sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL
=C
sort -u | \
291 while read packsha
; do
292 [ ! -e "objects/pack/pack-$packsha.pack" ] ||
continue
293 rm -f "objects/pack/pack-$packsha".?
*
296 # Remove any stale tmp reflogs files that are more than one hour old.
297 # Since they are created only while the pre-receive hook is running and
298 # all it does is process a bunch of refs passed to it on standard input
299 # it's inconceivable that it would ever take as much as an hour to run.
300 if [ -d reflogs
]; then
301 find reflogs
-maxdepth 1 -type f
-mmin +60 -name "tmp_*" -print0 |
xargs -0 rm -f
304 # Remove any stale pack .keep files that are more than 12 hours old.
305 # We don't do anything to create any permanent pack .keep files, so they must
306 # be remnants from some failed push or something. Removing the .keep will
307 # allow the pack to be properly repacked.
308 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "pack-$octet20.keep" -print0 |
xargs -0 rm -f
310 # Remove any stale tmp_pack_*, tmp_idx_*, tmp_bitmap_*, packtmp-* or .tmp-*-pack files
311 # that are more than 12 hours old.
312 find objects
/pack
-maxdepth 1 -type f
-mmin +720 \
( \
313 -name "tmp_pack_?*" -o -name "tmp_idx_?*" -o -name "tmp_bitmap_?*" -o \
314 -name "packtmp-?*" -o -name ".tmp-?*-pack" \
315 \
) -print0 |
xargs -0 rm -f
317 # Remove any stale shallow_* files that are more than 12 hours old.
318 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
319 # requests a shallow clone.
320 find .
-maxdepth 1 -type f
-mmin +720 -name "shallow_?*" -print0 |
xargs -0 rm -f
322 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
323 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
324 find objects
-type f
-mmin +720 -name "*.temp" -print0 |
xargs -0 rm -f
326 # Remove any stale *.lock files in the htmlcache area that might have been left
327 # behind after an abnormal exit during an attempt to update a cached file and
328 # are more than 1 hour old.
329 ! [ -d htmlcache
] ||
find htmlcache
-type f
-mmin +60 -name "*.lock" -print0 |
xargs -0 rm -f
331 # Remove any stale git-svn temp files that are more than 12 hours old.
332 # The git-svn process creates temp files with random 10 character names
333 # in the root of $GIT_DIR. Unfortunately they do not have a recognizable
334 # prefix, so we just have to kill any files with a 10-character name. We
335 # do this only for git-svn mirrors. All characters are chosen from
336 # [A-Za-z0-9_] so we can at least check that and fortunately the only
337 # collision is 'FETCH_HEAD' but that shouldn't matter.
338 # There may also be temp files with a Git_ prefix as well.
339 if [ -n "$svn_mirror" ]; then
340 _randchar
='[A-Za-z0-9_]'
341 _randchar2
="$_randchar$_randchar"
342 _randchar4
="$_randchar2$_randchar2"
343 _randchar10
="$_randchar4$_randchar4$_randchar2"
344 find .
-maxdepth 1 -type f
-mmin +720 -name "$_randchar10" -print0 |
xargs -0 rm -f
345 find .
-maxdepth 1 -type f
-mmin +720 -name "Git_*" -print0 |
xargs -0 rm -f
348 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
349 if [ -n "$gfi_mirror" ]; then
350 find .
-maxdepth 1 -type f
-mmin +4320 -name "fast_import_crash_?*" -print0 |
xargs -0 rm -f
355 ## Garbage Collection Types
357 ## There are two kinds of possible garbage collection (gc) operations:
359 ## 1. A normal, full gc
362 ## If the full garbage collection interval has expired (or gc has never been
363 ## run), then a normal, full gc will take place. Otherwise, a "mini" gc will
364 ## take place if the file .needsgc exists.
366 ## A "mini" gc is similar to "git gc --auto" in that it may not end up actually
367 ## doing anything unless the right conditions are present so it's not a burden
368 ## to run it often. If the file .needsgc exists, a "mini" gc will occur at
369 ## the next opportunity.
371 ## Note, however, that the .nogc file suppresses ALL gc activity (normal or mini).
376 cd "$cfg_reporoot/$proj.git"
377 [ -d objects
/pack
] ||
{ rm -f gfi-packs
; mkdir
-p objects
/pack
; }
378 mirror_url
="$(get_mirror_url)"
380 ! is_svn_mirror_url
"$mirror_url" || svn_mirror
=1
382 if [ -f gfi-packs
-a -s gfi-packs
] && is_gfi_mirror_url
"$mirror_url"; then
386 # If git config --bool --get girocco.redelta is explicitly false then automatic
387 # redelta when there are less than $var_redelta_threshold objects will be suppressed.
388 # On the other hand, if git config --get girocco.redelta is "always" then, on a full
389 # gc only, for the final repack, deltas will always be recomputed.
390 # This can be set on a per-project basis to avoid unusual pathological gc behavior.
391 # Setting this will hurt efficiency of the affected repository.
392 # Note that fast-import packs ALWAYS get new deltas regardless of this setting.
393 noreusedeltaopt
="--no-reuse-delta"
394 [ "$(git config --bool --get girocco.redelta 2>/dev/null || :)" != "false" ] || noreusedeltaopt
=
396 [ "$(git config --get girocco.redelta 2>/dev/null || :)" != "always" ] || alwaysredelta
=1
398 trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
402 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
403 datefmt
='+%a, %d %b %Y %T %z'
406 if check_interval lastgc
$cfg_min_gc_interval; then
407 if [ -e .needsgc
]; then
410 progress
"= [$proj] garbage check skip (last at $(config_get lastgc))"
414 if [ -e .nogc
]; then
415 progress
"x [$proj] garbage check disabled"
419 if [ -n "$isminigc" ]; then
420 # Perform a "mini" gc
421 # Note that .delaygc is ignored here as that's only intended for full gc
423 rm -f .allowgc .needsgc
428 if [ -f .svnpack
] && [ -n "$svn_mirror" ]; then
430 progress
"+ [$proj] mini garbage check (`date`)"
433 if [ -z "$cfg_delay_gfi_redelta" ] && [ -n "$gfi_mirror" ]; then
434 # $Girocco::Config::delay_gfi_redelta is false, force redeltification now
435 if [ -z "$miniactive" ]; then
437 progress
"+ [$proj] mini garbage check (`date`)"
441 # If there aren't at least 10 non-keep, non-bitmap, non-bndl packs then
442 # don't actually process them yet
443 lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
444 packcnt
="$(list_packs --count $lpo objects/pack || :)"
445 if [ "${packcnt:-0}" -ge 10 ]; then
446 if [ -z "$miniactive" ]; then
448 progress
"+ [$proj] mini garbage check (`date`)"
450 if [ -n "$gfi_mirror" ]; then
452 packcnt
="$(list_packs --count $lpo objects/pack || :)"
454 # if repack_gfi_packs dropped the pack count to < 10 don't combine
455 if [ "${packcnt:-0}" -ge 10 ]; then
457 packcnt
="$(list_packs --count $lpo objects/pack || :)"
459 # if we still have more than 10 packs trigger a full gc
460 if [ "${packcnt:-0}" -ge 10 ]; then
461 # We shouldn't be in a .delaygc state at this point, but if
462 # we are then nuke it because we really need a full gc now
464 git config
--unset gitweb.lastgc
466 progress
"- [$proj] mini garbage check triggering full gc too many packs (`date`)"
471 if [ -n "$miniactive" ]; then
472 git update-server-info
473 progress
"- [$proj] mini garbage check (`date`)"
475 progress
"= [$proj] mini garbage check nothing but crud removal to do (`date`)"
480 # Avoid unnecessary garbage collections:
481 # 1. If lastreceive is set and is older than lastgc
483 # 2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
485 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
486 # If we are a fork and lastparentgc is NOT set we MUST run gc
488 # If the repo is dirty after removing any crud we MUST run gc
490 gcstart
="$(date "$datefmt")"
493 [ -s objects
/info
/alternates
] && isfork
=1
495 [ -n "$isfork" ] && lastparentgcsecs
="$(config_get_date_seconds lastparentgc || :)"
497 if lastreceivesecs
="$(config_get_date_seconds lastreceive)" && \
498 lastgcsecs
="$(config_get_date_seconds lastgc)" && \
499 [ $lastreceivesecs -lt $lastgcsecs ]; then
500 # We've run gc since we last received, so maybe we can skip,
501 # check if not fork or fork and lastparentgc < lastgc
502 if [ -n "$isfork" ]; then
503 if [ -n "$lastparentgcsecs" ] && \
504 [ $lastparentgcsecs -lt $lastgcsecs ]; then
505 # We've run gc since our parent ran gc so we can skip
509 # We don't have any alternates (we're not a forK) so we can skip
514 # Prevent any other simultaneous gc operations
517 # At this point, if .allowgc exists, it's now crud to be removed
520 # Ideally we would do this in post-receive, but that would mean duplicating the
521 # logic so it's available in the chroot jail and that's highly undesirable
522 # Instead, since the first gc will be triggered immediately following the first
523 # push, we do the check here as it's quick and harmless if HEAD is already valid
524 check_and_set_head ||
:
526 # Always get rid of crud
529 # Always perform reflogs maintenance
533 # Run 'git svn gc' now for svn mirrors
534 if [ -n "$svn_mirror" ]; then
538 # Skip the actual gc if .delaygc is set
539 if [ -e .delaygc
]; then
540 progress
"x [$proj] garbage check delayed (except for crud removal)"
545 # Do not skip gc if the repo is dirty
546 if [ -n "$skipgc" ] && ! is_dirty
; then
547 progress
"= [$proj] garbage check nothing but crud removal to do (`date`)"
548 config_set lastgc
"$gcstart"
554 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
555 # set lastparentgc and then update gcstart to be at least 1 second later
556 config_set lastparentgc
"$gcstart"
559 if [ -z "$lastreceivesecs" ]; then
560 # set lastreceive and then update gcstart to be at least 1 second later
561 config_set lastreceive
"$gcstart"
564 if [ -n "$bumptime" ]; then
566 gcstart
="$(date "$datefmt")"
569 progress
"+ [$proj] garbage check (`date`)"
572 [ -z "$alwaysredelta" ] || newdeltas
=-f
573 if [ -z "$newdeltas" ] && [ -n "$gfi_mirror" ]; then
574 if [ $
(list_packs
--exclude-no-idx --count objects
/pack
) -le \
575 $
(list_packs
--exclude-no-idx --count --quiet --only gfi-packs
) ]; then
576 # Don't bother with repack_gfi_packs since everything's being repacked
580 if [ -z "$newdeltas" ] && [ -n "$noreusedeltaopt" ] && \
581 [ $
(list_packs
--exclude-no-idx --count-objects objects
/pack
) -le $var_redelta_threshold ]; then
582 # There aren't enough objects to worry about so just redelta to get the best pack
585 if [ -z "$newdeltas" ]; then
586 # Since we're not going to recompute deltas overall, we need to do the
587 # "mini" maintenance so that we can get more optimal deltas
588 [ -z "$noreusedeltaopt" ] || make_svn_pack
590 force_single_pack_redelta
=
591 [ -n "$gfi_mirror" ] ||
[ -n "$svn_mirror" ] || force_single_pack_redelta
=1
592 [ -z "$noreusedeltaopt" ] || combine_small_packs
$force_single_pack_redelta
596 ## Safe Pruning In Forks
598 ## We are about to perform garbage collection. We do NOT use the "git gc"
599 ## command directly as it does not provide enough control over the fine details
600 ## that we require. However, we DO maintain a "gc.pid" file during our garbage
601 ## collection so that a simultaneous "git gc" by an administrator will be
602 ## blocked (and similarly we refuse to start garbage collection if we cannot
603 ## create the "gc.pid" file). When we say "gc" in the below description we are
604 ## referring to our "gc.sh" script, NOT the "git gc" command.
606 ## If the project we are running garbage collection (gc) on has any forks we
607 ## must be careful not to remove any objects that while no longer referenced by
608 ## this project (the parent) are still referenced by one or more forks (the
609 ## children) otherwise the children will become corrupt and we can't abide
612 ## One way to accomplish this is to simply hard-link all currently existing
613 ## loose objects and packs in the parent into all the children that refer to the
614 ## parent (via a line in their objects/info/alternates file) before beginning
615 ## the gc operation and then relying on a subsequent gc in the child to clean up
616 ## any excess objects/packs. We used to use this strategy but it's very
617 ## inefficient because:
619 ## 1. The disk space used by the old pack(s)/object(s) will not be reclaimed
620 ## until all children (and their children, if any) run gc by which time
621 ## it's quite possible the topmost parent will have run gc again and
622 ## hard-linked yet another old pack down to its children (not to mention
625 ## 2. As we are now using the "-A" option with "git repack", any new objects
626 ## in the parent that are not referenced by children will continually get
627 ## exploded out of the hard-linked pack in the children whenever the
630 ## 3. To avoid suboptimal and/or unnecessarily many packs being hard-linked
631 ## into child forks, we must run the "mini" gc maintenance before we
632 ## perform the hard-linking into the children which provides yet another
633 ## source of inefficiency.
635 ## Since we are using the "-A" option to "git repack" (that was not always the
636 ## case) to guarantee we can access old ref values for long enough to send out
637 ## a meaningful mail.sh notification, we now have another, more efficient,
638 ## option available to prevent corruption of child forks that continue to refer
639 ## to objects that are no longer reachable from any ref in the parent.
641 ## The only things that need be copied (or hard-linked) into the child fork(s)
642 ## are those objects that have become unreachable from any ref in the parent.
643 ## They are the only things that could ever be removed by "git prune" and
644 ## therefore the only things we need to prevent the loss of in order to avoid
645 ## corruption of the child fork(s).
647 ## Therefore we now use the following strategy instead to avoid excessive disk
648 ## use and lots of unnecessary loose objects in child forks:
650 ## 1. Run "git repack -A -d -l" in the parent BEFORE doing anything about
653 ## 2. Hard-link all remaining existing loose objects in the parent into the
654 ## immediate child forks.
656 ## 3. Now run "git prune" in the parent.
658 ## With this new strategy we avoid the need to run any "mini" gc maintenance
659 ## before copying (or hard-linking) anything down to the child forks.
660 ## Furthermore, only when the parent performs a non-fast-forward update will
661 ## anything ever be transferred to the children leaving them unperturbed in the
662 ## vast majority of cases. Finally, even if the parent references objects the
663 ## children do not, those objects will no longer continually end up in the
664 ## children as unreachable loose objects after the children run gc.
668 touch .gc_in_progress
669 rm -f .gc_failed bundles
/*
670 rm -f objects
/pack
/pack-
*.bndl
671 # We use the -A option with git repack so that unreachable objects can live
672 # on for a time as loose objects. This is particularly helpful if we just
673 # happen to be in the process of sending out a ref update for a ref that was
674 # force updated and the old ref value would have otherwise been removed by
675 # repack because it was now unreachable. Admittedly the window for gc to run
676 # and do that before we manage to send out the ref update is not large, but
677 # it would not be difficult to create such a situation. Unfortunately, when
678 # Git unpacks these unreachable objects it will give them the modification
679 # time of the *.pack file they came out of. This could be very, very old.
680 # If that happens, the subsequent git prune --expire some_time_ago will still
681 # remove the object(s) and our pending ref update will still lose out.
682 # To prevent this from happening and to get the behavior we want, we now
683 # touch the modification time of all pack-<sha>.pack files so that any
684 # loosened objects get a current time. Git does not provide any other
685 # mechanism to do this. We do not want to just touch all loose objects
686 # left after the repack because that would cause objects that were loosened
687 # previously to live on which we definitely do not want.
688 list_packs
--exclude-no-idx objects
/pack |
xargs touch -c 2>/dev
/null ||
:
689 # We wish to keep deltas from our last full pack so if we're not redeltaing
690 # then make sure the .pack associated with the .bitmap has a newer mod time
691 # (If there is no .bitmap then touch the pack with the most objects instead.)
692 if [ -z "$newdeltas" ]; then
693 bmpack
="$(list_packs --exclude-no-bitmap --exclude-no-idx --max-matches 1 objects/pack)"
694 [ -n "$bmpack" ] || bmpack
="$(list_packs --exclude-no-idx --max-matches 1 --object-limit -1 --include-boundary objects/pack)"
695 if [ -n "$bmpack" ] && [ -f "$bmpack" -a -s "$bmpack" ]; then
697 touch -c "$bmpack" 2>/dev
/null ||
:
700 # The git repack command only supports bitmaps if all objects are being packed.
701 # While it is theoretically possible that a project with a non-empty alternates
702 # file ends up packing all objects (because it does not actually use any of the
703 # objects found in the alternates), it's very unlikely. And, in the unlikely
704 # event that did occur, clients would see a message about only using one bitmap
705 # because Git can only use one bitmap at a time and at least one of the
706 # alternates is bound to have a bitmap. Therefore if we see a non-empty
707 # alternates file, we disable writing bitmaps which avoids the warning and any
708 # possibility of a client warning as well.
710 [ -z "$var_have_git_172" ] ||
! [ -s objects
/info
/alternates
] || \
711 nobm
='-c repack.writebitmaps=false -c pack.writebitmaps=false'
712 progress
"~ [$proj] running full gc repack${nobm:+ (bitmaps disabled)}"
713 git
$nobm repack
$packopts -A -d -l $quiet $newdeltas $@
714 [ ! -e .gc_failed
] ||
exit 1
715 # These, if they exist, are now meaningless and need to be removed
716 rm -f gfi-packs .needsgc .svnpack .svnpackgc
717 allpacks
="$(echo objects/pack/pack-$octet20.pack)"
718 curhead
="$(cat HEAD)"
720 [ ! -e packed-refs
] || pkrf
=packed-refs
721 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | LC_ALL=C awk '{print $1}') | \
722 LC_ALL=C sed -e 's/ / + /g') ))"
723 git update-server-info
724 # The -A option to `git repack` may have caused some loose objects to pop
725 # out of their packs. We must make these objects group writable so that they
726 # can be freshened by other pushers. Technically we need only do this for
727 # push projects but to enable mirror projects to be more easily converted to
728 # push projects, we go ahead and do it for all projects.
729 { find objects
/$octet -type f
-name "$octet19" -print0 |
xargs -0 chmod ug
+w ||
:; } 2>/dev
/null
731 if has_forks
"$proj"; then
732 progress
"~ [$proj] hard-linking loose objects into immediate child forks"
733 # We have to update the lastparentgc time in the child forks even if they do not get any
734 # new "loose objects" because they need to run gc just in case the parent now has some
735 # objects that used to only be in the child so they can be removed from the child.
736 # For example, a "patch" might be developed first in a fork and then later accepted into
737 # the parent in which case the objects making up the patch in the child fork are now
738 # redundant (since they're now in the parent as well) and need to be removed from the
739 # child fork which can only happen if the child fork runs gc.
741 # It is enough to copy objects just one level down and get_repo_list
742 # takes a regular expression (which is automatically prefixed with '^')
743 # so we can easily match forks exactly one level down from this project
744 get_repo_list
"$forkdir/[^/]*:" |
746 # Ignore forks that do not exist or are symbolic links
747 [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
749 # Or do not have a non-zero length alternates file
750 [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
752 # Match objects in parent project
753 for d
in objects
/$octet; do
754 [ "$d" != "objects/$octet" ] ||
continue
755 mkdir
-p "$cfg_reporoot/$fork.git/$d"
756 find "$d" -maxdepth 1 -type f
-name "$octet19" -print0 |
757 xargs -0 "$var_sh_bin" -c 'ln -f "$@" '"'$cfg_reporoot/$fork.git/$d/'" sh ||
:
759 # Update the fork's lastparentgc date (must be current, not $gcstart)
760 git
--git-dir="$cfg_reporoot/$fork.git" config \
761 gitweb.lastparentgc
"$(date "$datefmt")"
765 # The git prune command does not take a -q or --quiet but started outputting
766 # 'Checking connectivity' progress messages in v1.7.9. However, we can
767 # suppress those by piping through cat as it only activates the progress
768 # messages when stderr is a tty. We only expire loose objects older than one
769 # day just in case there's some pending action (such as sending out a ref
770 # update) in progress that might want to examine them. This may leave us with
771 # loose objects. That's okay because at the next gc interval, we will always
772 # run gc if we see any loose objects regardless of whether or not we've seen
773 # any updates or we've received new linked objects from our parent. Note that
774 # in order to keep loose objects that just recently became unreferenced but
775 # have a very old modification date around we rely on some help from both the
776 # update.sh and hooks/pre-receive scripts. Furthermore, since Git v2.2.0
777 # (d3038d22 prune: keep objects reachable from recent objects) an unreachable
778 # object that would otherwise be pruned (because it's too old) will be kept
779 # alive by an unreachable object that refers to it that's not old enough to
781 prunecmd
='git prune --expire 1_day_ago'
782 [ -n "$show_progress" ] || \
783 prunecmd
="{ $prunecmd 2>&1 || touch .gc_failed; } | cat"
784 progress
"~ [$proj] pruning expired unreachable loose objects"
786 [ ! -e .gc_failed
] ||
exit 1
788 # darcs:// mirrors have a xxx.log file that will grow endlessly
789 # if this is a mirror and the file exists, shorten it to 10000 lines
790 # also take this opportunity to optimize the darcs repo
791 if [ ! -e .nofetch
] && [ -n "$cfg_mirror" ]; then
792 url
="$(config_get baseurl || :)"
793 case "$url" in darcs
://*)
794 if [ -n "$cfg_mirror_darcs" ]; then
796 basedarcs
="$(basename "${url#darcs:/}")"
797 if [ -f "$basedarcs.log" ]; then
798 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
799 mv -f "$basedarcs.log.$$" "$basedarcs.log"
801 if [ -d "$basedarcs.darcs" ]; then
803 cd "$basedarcs.darcs"
804 # without show_progress suppress non-error output
805 [ -n "$show_progress" ] ||
exec >/dev
/null
806 # Note that this does not optimize _darcs/inventories/ :(
814 # Create a matching .bndl header file for the all-in-one pack we just created
815 # but only if we're not a fork (otherwise the bundle would not be complete)
816 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
817 if [ ! -s objects
/info
/alternates
] && [ -n "$var_have_git_172" ]; then
818 # There should only be one pack in $allpacks but if there was a
819 # simultaneous push...
820 # The one we just created will have a .idx and will NOT have a .keep
821 progress
"~ [$proj] creating downloadble bundle header"
824 for pk
in $allpacks; do
825 [ -s "$pk" ] ||
continue
827 [ -s "$pkbase.idx" ] ||
continue
828 [ ! -e "$pkbase.keep" ] ||
continue
829 if pkhead
="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs
" "$curhead")"; then
834 if [ -n "$pkfound" -a -n "$pkhead" ]; then
836 echo "# v2 git bundle"
837 LC_ALL
=C
sed -ne "/^$octet20 refs\/[^ $tab]*\$/ p" < packed-refs
841 bndletag
="$("$cfg_basedir/bin
/rangecgi
" --etag -m 1 "$pkbase.bndl
" "$pkbase.pack
" || :)"
842 bndlsha
="$(printf '%s' "$bndletag" | git hash-object --stdin || :)"
843 if [ -n "$bndletag" ]; then
844 case "$bndlsha" in $octet20)
845 bndlshatrailer
="${bndlsha#????????}"
846 bndlshaprefix
="${bndlsha%$bndlshatrailer}"
847 bndlname
="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
848 [ -d bundles
] || mkdir bundles
849 echo "${pkbase#objects/pack/}.bndl" > "bundles/$bndlname"
850 echo "${pkbase#objects/pack/}.pack" >> "bundles/$bndlname"
851 ln -s -f -n "$bndlname" bundles
/latest
857 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
858 config_set_raw girocco.reposizek
"${reposizek:-0}"
860 # We use $gcstart here to avoid a race where a push occurs during the gc itself
861 # and the next future gc could be incorrectly skipped if we used the current
862 # timestamp here instead
863 config_set lastgc
"$gcstart"
866 progress
"- [$proj] garbage check (`date`)"