3 # NOTE: additional options can be passed to git repack by specifying
4 # them after the project name, for example:
12 echo "Usage: gc.sh projname [extra-repack-args]" >&2
17 packopts
="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
20 [ "$cfg_permission_control" != "Hooks" ] ||
umask 000
23 if _result
="$(kill -0 "$1" 2>&1)"; then
24 # process exists and we have permission to signal it
27 case "$_result" in *"not permitted"*)
28 # we do not have permission to signal the process
31 # process does not exist
36 # A .lock file should only exist for much less than a second.
37 # If we see a stale lock file (> 1h old), remove it and then,
38 # just in case, wait 30 seconds for any process whose .lock
39 # we might have just removed (it's racy) to finish doing what
40 # should take much less than a second to do.
41 _stalelock
="$(find "$1.lock
" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
42 if [ -n "$_stalelock" ]; then
47 if (set -C; > "$1.lock") 2>/dev
/null
; then
52 [ "$_try" != "p" ] ||
sleep 1
54 # cannot create lock file
58 # return true if there's more than one objects/pack-<sha>.pack file or
59 # ANY sha-1 files in objects
61 _packs
=$
(find objects
/pack
-type f
-name "pack-$octet20.pack" -print |
head -n 2 |
wc -l)
62 if [ $_packs != 1 ] && [ $_packs != 0 ]; then
65 _objs
=$
(find objects
/$octet -type f
-name "$octet19" -print 2>/dev
/null |
head -n 1 |
wc -l)
69 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
71 is_gfi_mirror ||
return 0
72 [ -s gfi-packs
] ||
return 0
73 while IFS
=': ' read -r _pack _junk
; do
74 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
75 git show-index
< "${_pack%.pack}.idx" | cut
-d ' ' -f 2
78 git pack-objects
$packopts --no-reuse-delta --delta-base-offset \
79 --non-empty $quiet objects
/pack
/packtmp | \
80 while read -r _newpack
; do
81 rm -f objects
/pack
/pack-
$_newpack.
*
82 ln -f objects
/pack
/packtmp-
$_newpack.pack objects
/pack
/pack-
$_newpack.pack
83 ln -f objects
/pack
/packtmp-
$_newpack.idx objects
/pack
/pack-
$_newpack.idx
84 rm -f objects
/pack
/packtmp-
$_newpack.
*
89 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
91 # Must have a matching .idx file and a non-empty packed-refs file
92 [ -s "${1%.pack}.idx" ] ||
return 1
93 [ -s "$2" ] ||
return 1
99 "ref: refs/"?
*|
"ref:refs/"?
*|
"refs/"?
*)
100 _headmatch
="${3#ref:}"
101 _headmatch
="${_headmatch# }"
102 _headmatchpat
="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
103 _headsha
="$(grep -e "^
$octet20 $_headmatchpat\$
" < "$2" | \
105 case "$_headsha" in $octet20) :;; *)
113 rm -rf pack_is_complete_test
114 mkdir pack_is_complete_test
115 mkdir pack_is_complete_test
/refs
116 mkdir pack_is_complete_test
/objects
117 mkdir pack_is_complete_test
/objects
/pack
118 echo "$_headsha" > pack_is_complete_test
/HEAD
119 ln -s "$1" pack_is_complete_test
/objects
/pack
/
120 ln -s "${1%.pack}.idx" pack_is_complete_test
/objects
/pack
/
121 ln -s "$2" pack_is_complete_test
/packed-refs
122 _count
="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
123 rm -rf pack_is_complete_test
124 [ -n "$_count" ] ||
return 1
125 [ "$_count" -gt 0 ] 2>/dev
/null ||
return 1
131 cd "$cfg_reporoot/$proj.git"
133 trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
137 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
138 datefmt
='+%a, %d %b %Y %T %z'
140 if check_interval lastgc
$cfg_min_gc_interval; then
141 progress
"= [$proj] garbage check skip (last at $(config_get lastgc))"
144 if [ -e .nogc
]; then
145 progress
"x [$proj] garbage check disabled"
149 # Avoid unnecessary garbage collections:
150 # 1. If lastreceive is set and is older than lastgc
152 # 2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
154 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
155 # If we are a fork and lastparentgc is NOT set we MUST run gc
157 # If the repo is dirty after removing any crud we MUST run gc
159 gcstart
="$(date "$datefmt")"
162 [ -s objects
/info
/alternates
] && isfork
=1
164 [ -n "$isfork" ] && lastparentgcsecs
="$(config_get_date_seconds lastparentgc || :)"
166 if lastreceivesecs
="$(config_get_date_seconds lastreceive)" && \
167 lastgcsecs
="$(config_get_date_seconds lastgc)" && \
168 [ $lastreceivesecs -lt $lastgcsecs ]; then
169 # We've run gc since we last received, so maybe we can skip,
170 # check if not fork or fork and lastparentgc < lastgc
171 if [ -n "$isfork" ]; then
172 if [ -n "$lastparentgcsecs" ] && \
173 [ $lastparentgcsecs -lt $lastgcsecs ]; then
174 # We've run gc since our parent ran gc so we can skip
178 # We don't have any alternates (we're not a forK) so we can skip
183 # be compatibile with gc.pid file from newer Git releases
188 if [ "$(createlock "$lockf")" ]; then
190 # 1) less than 12 hours old
191 # 2) contains two fields (pid hostname) NO trailing NL
192 # 3) the hostname is different OR the pid is still alive
193 # then we exit as another active process is holding the lock
194 if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
197 read -r apid ahost ajunk
< "$lockf" ||
:
198 if [ "$apid" ] && [ "$ahost" ]; then
199 if [ "$ahost" != "$hn" ] || pidactive
"$apid"; then
205 echo >&2 "[$proj] unable to create gc.pid.lock file"
208 if [ -n "$active" ]; then
210 echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
213 printf "%s %s" "$$" "$hn" > "$lockf.lock"
214 chmod 0664 "$lockf.lock"
215 mv -f "$lockf.lock" "$lockf"
217 # At this point, if .allowgc exists, it's now crud to be removed
220 # Remove any existing FETCH_HEAD
221 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
222 # receiving pushes (those never create a FETCH_HEAD).
223 # And if we're fetching because we're a mirror, we know we're not fetching right
224 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
225 # Therefore any existing FETCH_HEAD is junk. And it may be many megabytes if
226 # there were a lot of refs.
229 # Remove any stale pack remnants that are more than an hour old.
230 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
231 # .pack AND the corresponding .pack DOES NOT exist. A bunch of stale
232 # pack-<sha1>.idx files without their corresponding .pack files are worthless
233 # and just waste space. Normally there shouldn't be any remnants but actually
234 # this can happen when things are interrupted at just the wrong time.
235 # Note that the objects/pack directory is created by git init and should
237 find objects
/pack
-maxdepth 1 -type f
-mmin +60 -name "pack-$octet20.?*" -print | \
238 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL
=C
sort -u | \
239 while read packsha
; do
240 [ ! -e "objects/pack/pack-$packsha.pack" ] ||
continue
241 rm -f "objects/pack/pack-$packsha".?
*
244 # Remove any stale pack .keep files that are more than 12 hours old.
245 # We don't do anything to create any permanent pack .keep files, so they must
246 # be remnants from some failed push or something. Removing the .keep will
247 # allow the pack to be properly repacked.
248 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "pack-$octet20.keep" -print0 |
xargs -0 rm -f
250 # Remove any stale tmp_pack_* or tmp_idx_* or tmp_bitmap_* or packtmp-* files
251 # that are more than 12 hours old.
252 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "tmp_pack_?*" -print0 |
xargs -0 rm -f
253 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "tmp_idx_?*" -print0 |
xargs -0 rm -f
254 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "tmp_bitmap_?*" -print0 |
xargs -0 rm -f
255 find objects
/pack
-maxdepth 1 -type f
-mmin +720 -name "packtmp-?*" -print0 |
xargs -0 rm -f
257 # Remove any stale shallow_* files that are more than 12 hours old.
258 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
259 # requests a shallow clone.
260 find .
-maxdepth 1 -type f
-mmin +720 -name "shallow_?*" -print0 |
xargs -0 rm -f
262 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
263 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
264 find objects
-type f
-mmin +720 -name "*.temp" -print0 |
xargs -0 rm -f
266 # Remove any stale *.lock files in the htmlcache area that might have been left
267 # behind after an abnormal exit during an attempt to update a cached file and
268 # are more than 1 hour old.
269 ! [ -d htmlcache
] ||
find htmlcache
-type f
-mmin +60 -name "*.lock" -print0 |
xargs -0 rm -f
271 # Remove any stale git-svn temp files that are more than 12 hours old.
272 # The git-svn process creates temp files with random 10 character names
273 # in the root of $GIT_DIR. Unfortunately they do not have a recognizable
274 # prefix, so we just have to kill any files with a 10-character name. We
275 # do this only for git-svn mirrors. All characters are chosen from
276 # [A-Za-z0-9_] so we can at least check that and fortunately the only
277 # collision is 'FETCH_HEAD' but that shouldn't matter.
278 # There may also be temp files with a Git_ prefix as well.
279 # We also take this opportunity to run 'git svn gc'
280 if is_svn_mirror
; then
281 _randchar
='[A-Za-z0-9_]'
282 _randchar2
="$_randchar$_randchar"
283 _randchar4
="$_randchar2$_randchar2"
284 _randchar10
="$_randchar4$_randchar4$_randchar2"
285 find .
-maxdepth 1 -type f
-mmin +720 -name "$_randchar10" -print0 |
xargs -0 rm -f
286 find .
-maxdepth 1 -type f
-mmin +720 -name "Git_*" -print0 |
xargs -0 rm -f
290 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
291 if is_gfi_mirror
; then
292 find .
-maxdepth 1 -type f
-mmin +4320 -name "fast_import_crash_?*" -print0 |
xargs -0 rm -f
295 # Skip the actual gc if .delaygc is set
296 if [ -e .delaygc
]; then
297 progress
"x [$proj] garbage check delayed (except for crud removal)"
302 # Do not skip gc if the repo is dirty
303 if [ -n "$skipgc" ] && ! is_dirty
; then
304 progress
"= [$proj] garbage check nothing but crud removal to do (`date`)"
305 config_set lastgc
"$gcstart"
311 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
312 # set lastparentgc and then update gcstart to be at least 1 second later
313 config_set lastparentgc
"$gcstart"
316 if [ -z "$lastreceivesecs" ]; then
317 # set lastreceive and then update gcstart to be at least 1 second later
318 config_set lastreceive
"$gcstart"
321 if [ -n "$bumptime" ]; then
323 gcstart
="$(date "$datefmt")"
326 progress
"+ [$proj] garbage check (`date`)"
328 # safe pruning: we put all our objects in all forks, then we can
329 # safely get rid of extra ones; repacks in forks will get rid of
330 # the redundant ones again then; we carefully grab only loose
331 # objects and pack .idx and .pack files
333 if [ -d "../${forkdir##*/}" ]; then
334 # It is enough to copy objects just one level down and get_repo_list
335 # takes a regular expression (which is automatically prefixed with '^')
336 # so we can easily match forks exactly one level down from this project
337 get_repo_list
"$forkdir/[^/]*:" |
339 # Ignore forks that do not exist or are symbolic links
340 [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
342 # Or do not have a non-zero length alternates file
343 [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
345 # Match objects in parent project
346 for d
in objects
/??
; do
347 [ "$d" != "objects/??" ] ||
continue
348 mkdir
-p "$cfg_reporoot/$fork.git/$d"
349 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" ||
:
351 # Match packs in parent project
352 mkdir
-p "$cfg_reporoot/$fork.git/objects/pack"
353 if [ "$(echo objects/pack/pack-*.idx)" != \
354 "objects/pack/pack-*.idx" ]; then
355 ln -f objects
/pack
/pack-
*.pack
"$cfg_reporoot/$fork.git/objects/pack" ||
:
356 ln -f objects
/pack
/pack-
*.idx
"$cfg_reporoot/$fork.git/objects/pack" ||
:
358 # Update the fork's lastparentgc date (must be current, not $gcstart)
359 GIT_DIR
="$cfg_reporoot/$fork.git" git config \
360 gitweb.lastparentgc
"$(date "$datefmt")"
364 quiet
=; [ -n "$show_progress" ] || quiet
=-q
368 touch .gc_in_progress
369 rm -f .gc_failed bundles
/*
370 rm -f objects
/pack
/pack-
*.bndl
371 # We use the -A option with git repack so that unreachable objects can live
372 # on for a time as loose objects. This is particularly helpful if we just
373 # happen to be in the process of sending out a ref update for a ref that was
374 # force updated and the old ref value would have otherwise been removed by
375 # repack because it was now unreachable. Admittedly the window for gc to run
376 # and do that before we manage to send out the ref update is not large, but
377 # it would not be difficult to create such a situation. Unfortunately, when
378 # Git unpacks these unreachable objects it will give them the modification
379 # time of the *.pack file they came out of. This could be very, very old.
380 # If that happens, the subsequent git prune --expire some_time_ago will still
381 # remove the object(s) and our pending ref update will still lose out.
382 # To prevent this from happening and to get the behavior we want, we now
383 # touch the modification time of all pack-<sha>.pack files so that any
384 # loosened objects get a current time. Git does not provide any other
385 # mechanism to do this. We do not want to just touch all loose objects
386 # left after the repack because that would cause objects that were loosened
387 # previously to live on which we definitely do not want.
388 touch -c -m objects
/pack
/pack-
$octet20.pack
389 # The git repack command may issue a 'disabling bitmap' warning for some
390 # repositories. This is perfectly normal and should be suppressed unless
391 # show_progress is set. Unfortunately that means we have to grep -v the
392 # output. And furthermore, since it's a translated message, we have to
393 # force the language to english to be sure we do it.
394 repackcmd
="git repack $packopts -A -d -l $quiet $@"
395 [ -n "$show_progress" ] || \
396 repackcmd
="{ LC_ALL=C $repackcmd 2>&1 || touch .gc_failed; } | grep -v 'disabling bitmap' || :"
398 [ ! -e .gc_failed
] ||
exit 1
399 allpacks
="$(echo objects/pack/pack-$octet20.pack)"
400 curhead
="$(cat HEAD)"
402 [ ! -e packed-refs
] || pkrf
=packed-refs
403 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | awk '{print $1}') | \
404 sed -e 's/ / + /g') ))"
405 # The git prune command does not take a -q or --quiet but started outputting
406 # 'Checking connectivity' progress messages in v1.7.9. However, we can
407 # suppress those by piping through cat as it only activates the progress
408 # messages when stderr is a tty. We only expire loose objects older than one
409 # day just in case there's some pending action (such as sending out a ref
410 # update) in progress that might want to examine them. This may leave us with
411 # loose objects. That's okay because at the next gc interval, we will always
412 # run gc if we see any loose objects regardless of whether or not we've seen
413 # any updates or we've received new linked objects from our parent. Note that
414 # in order to keep loose objects that just recently became unreferenced but
415 # have a very old modification date around we rely on some help from both the
416 # update.sh and hooks/pre-receive scripts. Furthermore, since Git v2.2.0
417 # (d3038d22 prune: keep objects reachable from recent objects) an unreachable
418 # object that would otherwise be pruned (because it's too old) will be kept
419 # alive by an unreachable object that refers to it that's not old enough to
421 prunecmd
='git prune --expire 1_day_ago'
422 [ -n "$show_progress" ] || \
423 prunecmd
="{ $prunecmd 2>&1 || touch .gc_failed; } | cat"
425 [ ! -e .gc_failed
] ||
exit 1
426 git update-server-info
428 # darcs:// mirrors have a xxx.log file that will grow endlessly
429 # if this is a mirror and the file exists, shorten it to 10000 lines
430 # also take this opportunity to optimize the darcs repo
431 if [ ! -e .nofetch
] && [ -n "$cfg_mirror" ]; then
432 url
="$(config_get baseurl || :)"
433 case "$url" in darcs
://*)
434 if [ -n "$cfg_mirror_darcs" ]; then
436 basedarcs
="$(basename "${url#darcs:/}")"
437 if [ -f "$basedarcs.log" ]; then
438 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
439 mv -f "$basedarcs.log.$$" "$basedarcs.log"
441 if [ -d "$basedarcs.darcs" ]; then
443 cd "$basedarcs.darcs"
444 # Note that this does not optimize _darcs/inventories/ :(
452 # Create a matching .bndl header file for the all-in-one pack we just created
453 # but only if we're not a fork (otherwise the bundle would not be complete)
454 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
455 if [ ! -s objects
/info
/alternates
] && [ -n "$var_have_git_172" ]; then
456 # There should only be one pack in $allpacks but if there was a
457 # simultaneous push...
458 # The one we just created will have a .idx and will NOT have a .keep
461 for pk
in $allpacks; do
462 [ -s "$pk" ] ||
continue
464 [ -s "$pkbase.idx" ] ||
continue
465 [ ! -e "$pkbase.keep" ] ||
continue
466 if pkhead
="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs
" "$curhead")"; then
471 if [ -n "$pkfound" -a -n "$pkhead" ]; then
473 echo "# v2 git bundle"
474 sed -ne "/^$octet20 refs\/[^ $tab]*\$/ p" < packed-refs
478 bndletag
="$("$cfg_basedir/bin
/rangecgi
" --etag "$pkbase.bndl
" "$pkbase.pack
" || :)"
479 bndlsha
="$(printf '%s' "$bndletag" | git hash-object --stdin || :)"
480 if [ -n "$bndletag" ]; then
481 case "$bndlsha" in $octet20)
482 bndlshatrailer
="${bndlsha#????????}"
483 bndlshaprefix
="${bndlsha%$bndlshatrailer}"
484 bndlname
="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
485 [ -d bundles
] || mkdir bundles
486 echo "${pkbase#objects/pack/}.bndl" > "bundles/$bndlname"
487 echo "${pkbase#objects/pack/}.pack" >> "bundles/$bndlname"
488 ln -s -f -n "$bndlname" bundles
/latest
494 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
495 config_set_raw girocco.reposizek
"${reposizek:-0}"
497 # We use $gcstart here to avoid a race where a push occurs during the gc itself
498 # and the next future gc could be incorrectly skipped if we used the current
499 # timestamp here instead
500 config_set lastgc
"$gcstart"
503 progress
"- [$proj] garbage check (`date`)"