gc.sh: make repack_gfi_packs more robust and scalable
[girocco.git] / jobd / gc.sh
blobab93cac2ee6f50976329dd2519f7298ad7fa4405
1 #!/bin/sh
3 # NOTE: additional options can be passed to git repack by specifying
4 # them after the project name, for example:
5 # gc.sh my-project -f
7 . @basedir@/shlib.sh
9 set -e
11 if [ $# -lt 1 ]; then
12 echo "Usage: gc.sh projname [extra-repack-args]" >&2
13 exit 1
16 # packing options
17 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
19 umask 002
20 [ "$cfg_permission_control" != "Hooks" ] || umask 000
22 pidactive() {
23 if _result="$(kill -0 "$1" 2>&1)"; then
24 # process exists and we have permission to signal it
25 return 0
27 case "$_result" in *"not permitted"*)
28 # we do not have permission to signal the process
29 return 0
30 esac
31 # process does not exist
32 return 1
35 createlock() {
36 # A .lock file should only exist for much less than a second.
37 # If we see a stale lock file (> 1h old), remove it and then,
38 # just in case, wait 30 seconds for any process whose .lock
39 # we might have just removed (it's racy) to finish doing what
40 # should take much less than a second to do.
41 _stalelock="$(find "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
42 if [ -n "$_stalelock" ]; then
43 rm -f "$_stalelock"
44 sleep 30
46 for _try in p p n; do
47 if (set -C; > "$1.lock") 2>/dev/null; then
48 echo "$1.lock"
49 return 0
51 # delay and try again
52 [ "$_try" != "p" ] || sleep 1
53 done
54 # cannot create lock file
55 return 1
58 # return true if there's more than one objects/pack-<sha>.pack file or
59 # ANY sha-1 files in objects
60 is_dirty() {
61 _packs=$(find objects/pack -type f -name "pack-$octet20.pack" -print | head -n 2 | wc -l)
62 if [ $_packs != 1 ] && [ $_packs != 0 ]; then
63 return 0
65 _objs=$(find objects/$octet -type f -name "$octet19" -print 2>/dev/null | head -n 1 | wc -l)
66 [ $_objs -ne 0 ]
69 # add "old" prefix to passed in existing files, but be careful to hard-link
70 # ALL the files to be renamed to the renamed name BEFORE removing anything
71 move_aside() {
72 for _f; do
73 [ -f "$_f" ] && ln -f "$_f" "old$_f"
74 done
75 for _f; do
76 [ -f "$_f" ] && rm -f "$_f"
77 done
78 return 0
81 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
82 repack_gfi_packs() {
83 is_gfi_mirror || return 0
84 [ -d objects/pack ] || { rm -f gfi-packs; return 0; }
85 [ -s gfi-packs ] || return 0
86 rm -f .gc_failed
87 find objects/pack -maxdepth 1 -type f -name '*.zap' -print0 | xargs -0 rm -f
88 while IFS=': ' read -r _pack _junk; do
89 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
90 > "${_pack%.pack}.zap"
91 git show-index < "${_pack%.pack}.idx"
93 done < gfi-packs | cut -d ' ' -f 2 | \
94 { git pack-objects $packopts --no-reuse-delta --delta-base-offset \
95 --non-empty $quiet objects/pack/packtmp || touch .gc_failed; } |
96 while read -r _newpack; do
97 move_aside objects/pack/pack-$_newpack.*
98 ln -f objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
99 ln -f objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
100 rm -f objects/pack/packtmp-$_newpack.*
101 done
102 [ ! -e .gc_failed ] || return 1
103 find objects/pack -maxdepth 1 -type f -name '*.zap' -print |
104 while read -r _remove; do
105 rm -f "${_remove%.zap}".*
106 done
107 rm -f gfi-packs
108 return 0
111 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
112 pack_is_complete() {
113 # Must have a matching .idx file and a non-empty packed-refs file
114 [ -s "${1%.pack}.idx" ] || return 1
115 [ -s "$2" ] || return 1
116 _headsha=
117 case "$3" in
118 $octet20)
119 _headsha="$3"
121 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
122 _headmatch="${3#ref:}"
123 _headmatch="${_headmatch# }"
124 _headmatchpat="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
125 _headsha="$(grep -e "^$octet20 $_headmatchpat\$" < "$2" | \
126 cut -d ' ' -f 1)"
127 case "$_headsha" in $octet20) :;; *)
128 return 1
129 esac
132 # bad HEAD
133 return 1
134 esac
135 rm -rf pack_is_complete_test
136 mkdir pack_is_complete_test
137 mkdir pack_is_complete_test/refs
138 mkdir pack_is_complete_test/objects
139 mkdir pack_is_complete_test/objects/pack
140 echo "$_headsha" > pack_is_complete_test/HEAD
141 ln -s "$1" pack_is_complete_test/objects/pack/
142 ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
143 ln -s "$2" pack_is_complete_test/packed-refs
144 _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
145 rm -rf pack_is_complete_test
146 [ -n "$_count" ] || return 1
147 [ "$_count" -gt 0 ] 2>/dev/null || return 1
148 echo "$_headsha"
151 proj="${1%.git}"
152 shift
153 cd "$cfg_reporoot/$proj.git"
155 trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
156 trap 'exit 130' INT
157 trap 'exit 143' TERM
159 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
160 datefmt='+%a, %d %b %Y %T %z'
162 if check_interval lastgc $cfg_min_gc_interval; then
163 progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
164 exit 0
166 if [ -e .nogc ]; then
167 progress "x [$proj] garbage check disabled"
168 exit 0
171 # Avoid unnecessary garbage collections:
172 # 1. If lastreceive is set and is older than lastgc
173 # -AND-
174 # 2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
176 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
177 # If we are a fork and lastparentgc is NOT set we MUST run gc
179 # If the repo is dirty after removing any crud we MUST run gc
181 gcstart="$(date "$datefmt")"
182 skipgc=
183 isfork=
184 [ -s objects/info/alternates ] && isfork=1
185 lastparentgcsecs=
186 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
187 lastreceivesecs=
188 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
189 lastgcsecs="$(config_get_date_seconds lastgc)" && \
190 [ $lastreceivesecs -lt $lastgcsecs ]; then
191 # We've run gc since we last received, so maybe we can skip,
192 # check if not fork or fork and lastparentgc < lastgc
193 if [ -n "$isfork" ]; then
194 if [ -n "$lastparentgcsecs" ] && \
195 [ $lastparentgcsecs -lt $lastgcsecs ]; then
196 # We've run gc since our parent ran gc so we can skip
197 skipgc=1
199 else
200 # We don't have any alternates (we're not a forK) so we can skip
201 skipgc=1
205 # be compatibile with gc.pid file from newer Git releases
207 hn="$(hostname)"
208 lockf=gc.pid
209 active=
210 if [ "$(createlock "$lockf")" ]; then
211 # If $lockf is:
212 # 1) less than 12 hours old
213 # 2) contains two fields (pid hostname) NO trailing NL
214 # 3) the hostname is different OR the pid is still alive
215 # then we exit as another active process is holding the lock
216 if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
217 apid=
218 ahost=
219 read -r apid ahost ajunk < "$lockf" || :
220 if [ "$apid" ] && [ "$ahost" ]; then
221 if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
222 active=1
226 else
227 echo >&2 "[$proj] unable to create gc.pid.lock file"
228 exit 1
230 if [ -n "$active" ]; then
231 rm -f "$lockf.lock"
232 echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
233 exit 1
235 printf "%s %s" "$$" "$hn" > "$lockf.lock"
236 chmod 0664 "$lockf.lock"
237 mv -f "$lockf.lock" "$lockf"
239 # At this point, if .allowgc exists, it's now crud to be removed
240 rm -f .allowgc
242 # Remove any existing FETCH_HEAD
243 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
244 # receiving pushes (those never create a FETCH_HEAD).
245 # And if we're fetching because we're a mirror, we know we're not fetching right
246 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
247 # Therefore any existing FETCH_HEAD is junk. And it may be many megabytes if
248 # there were a lot of refs.
249 rm -f FETCH_HEAD
251 # Remove any stale pack remnants that are more than an hour old.
252 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
253 # .pack AND the corresponding .pack DOES NOT exist. A bunch of stale
254 # pack-<sha1>.idx files without their corresponding .pack files are worthless
255 # and just waste space. Normally there shouldn't be any remnants but actually
256 # this can happen when things are interrupted at just the wrong time.
257 # Note that the objects/pack directory is created by git init and should
258 # always exist.
259 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" -print | \
260 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u | \
261 while read packsha; do
262 [ ! -e "objects/pack/pack-$packsha.pack" ] || continue
263 rm -f "objects/pack/pack-$packsha".?*
264 done
266 # Remove any stale pack .keep files that are more than 12 hours old.
267 # We don't do anything to create any permanent pack .keep files, so they must
268 # be remnants from some failed push or something. Removing the .keep will
269 # allow the pack to be properly repacked.
270 find objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20.keep" -print0 | xargs -0 rm -f
272 # Remove any stale tmp_pack_* or tmp_idx_* or tmp_bitmap_* or packtmp-* files
273 # that are more than 12 hours old.
274 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_pack_?*" -print0 | xargs -0 rm -f
275 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_idx_?*" -print0 | xargs -0 rm -f
276 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_bitmap_?*" -print0 | xargs -0 rm -f
277 find objects/pack -maxdepth 1 -type f -mmin +720 -name "packtmp-?*" -print0 | xargs -0 rm -f
279 # Remove any stale shallow_* files that are more than 12 hours old.
280 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
281 # requests a shallow clone.
282 find . -maxdepth 1 -type f -mmin +720 -name "shallow_?*" -print0 | xargs -0 rm -f
284 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
285 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
286 find objects -type f -mmin +720 -name "*.temp" -print0 | xargs -0 rm -f
288 # Remove any stale *.lock files in the htmlcache area that might have been left
289 # behind after an abnormal exit during an attempt to update a cached file and
290 # are more than 1 hour old.
291 ! [ -d htmlcache ] || find htmlcache -type f -mmin +60 -name "*.lock" -print0 | xargs -0 rm -f
293 # Remove any stale git-svn temp files that are more than 12 hours old.
294 # The git-svn process creates temp files with random 10 character names
295 # in the root of $GIT_DIR. Unfortunately they do not have a recognizable
296 # prefix, so we just have to kill any files with a 10-character name. We
297 # do this only for git-svn mirrors. All characters are chosen from
298 # [A-Za-z0-9_] so we can at least check that and fortunately the only
299 # collision is 'FETCH_HEAD' but that shouldn't matter.
300 # There may also be temp files with a Git_ prefix as well.
301 # We also take this opportunity to run 'git svn gc'
302 if is_svn_mirror; then
303 _randchar='[A-Za-z0-9_]'
304 _randchar2="$_randchar$_randchar"
305 _randchar4="$_randchar2$_randchar2"
306 _randchar10="$_randchar4$_randchar4$_randchar2"
307 find . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -print0 | xargs -0 rm -f
308 find . -maxdepth 1 -type f -mmin +720 -name "Git_*" -print0 | xargs -0 rm -f
309 git svn gc || :
312 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
313 if is_gfi_mirror; then
314 find . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -print0 | xargs -0 rm -f
317 # Skip the actual gc if .delaygc is set
318 if [ -e .delaygc ]; then
319 progress "x [$proj] garbage check delayed (except for crud removal)"
320 rm -f "$lockf"
321 exit 0
324 # Do not skip gc if the repo is dirty
325 if [ -n "$skipgc" ] && ! is_dirty; then
326 progress "= [$proj] garbage check nothing but crud removal to do (`date`)"
327 config_set lastgc "$gcstart"
328 rm -f "$lockf"
329 exit 0
332 bumptime=
333 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
334 # set lastparentgc and then update gcstart to be at least 1 second later
335 config_set lastparentgc "$gcstart"
336 bumptime=1
338 if [ -z "$lastreceivesecs" ]; then
339 # set lastreceive and then update gcstart to be at least 1 second later
340 config_set lastreceive "$gcstart"
341 bumptime=1
343 if [ -n "$bumptime" ]; then
344 sleep 1
345 gcstart="$(date "$datefmt")"
348 progress "+ [$proj] garbage check (`date`)"
350 # safe pruning: we put all our objects in all forks, then we can
351 # safely get rid of extra ones; repacks in forks will get rid of
352 # the redundant ones again then; we carefully grab only loose
353 # objects and pack .idx and .pack files
354 forkdir="$proj"
355 if [ -d "../${forkdir##*/}" ]; then
356 # It is enough to copy objects just one level down and get_repo_list
357 # takes a regular expression (which is automatically prefixed with '^')
358 # so we can easily match forks exactly one level down from this project
359 get_repo_list "$forkdir/[^/]*:" |
360 while read fork; do
361 # Ignore forks that do not exist or are symbolic links
362 [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
363 continue
364 # Or do not have a non-zero length alternates file
365 [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
366 continue
367 # Match objects in parent project
368 for d in objects/??; do
369 [ "$d" != "objects/??" ] || continue
370 mkdir -p "$cfg_reporoot/$fork.git/$d"
371 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
372 done
373 # Match packs in parent project
374 mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
375 if [ "$(echo objects/pack/pack-*.idx)" != \
376 "objects/pack/pack-*.idx" ]; then
377 ln -f objects/pack/pack-*.pack "$cfg_reporoot/$fork.git/objects/pack" || :
378 ln -f objects/pack/pack-*.idx "$cfg_reporoot/$fork.git/objects/pack" || :
380 # Update the fork's lastparentgc date (must be current, not $gcstart)
381 GIT_DIR="$cfg_reporoot/$fork.git" git config \
382 gitweb.lastparentgc "$(date "$datefmt")"
383 done
386 quiet=; [ -n "$show_progress" ] || quiet=-q
388 git pack-refs --all
389 repack_gfi_packs
390 touch .gc_in_progress
391 rm -f .gc_failed bundles/*
392 rm -f objects/pack/pack-*.bndl
393 # We use the -A option with git repack so that unreachable objects can live
394 # on for a time as loose objects. This is particularly helpful if we just
395 # happen to be in the process of sending out a ref update for a ref that was
396 # force updated and the old ref value would have otherwise been removed by
397 # repack because it was now unreachable. Admittedly the window for gc to run
398 # and do that before we manage to send out the ref update is not large, but
399 # it would not be difficult to create such a situation. Unfortunately, when
400 # Git unpacks these unreachable objects it will give them the modification
401 # time of the *.pack file they came out of. This could be very, very old.
402 # If that happens, the subsequent git prune --expire some_time_ago will still
403 # remove the object(s) and our pending ref update will still lose out.
404 # To prevent this from happening and to get the behavior we want, we now
405 # touch the modification time of all pack-<sha>.pack files so that any
406 # loosened objects get a current time. Git does not provide any other
407 # mechanism to do this. We do not want to just touch all loose objects
408 # left after the repack because that would cause objects that were loosened
409 # previously to live on which we definitely do not want.
410 touch -c objects/pack/pack-$octet20.pack 2>/dev/null || :
411 # The git repack command may issue a 'disabling bitmap' warning for some
412 # repositories. This is perfectly normal and should be suppressed unless
413 # show_progress is set. Unfortunately that means we have to grep -v the
414 # output. And furthermore, since it's a translated message, we have to
415 # force the language to english to be sure we do it.
416 repackcmd="git repack $packopts -A -d -l $quiet $@"
417 [ -n "$show_progress" ] || \
418 repackcmd="{ LC_ALL=C $repackcmd 2>&1 || touch .gc_failed; } | grep -v 'disabling bitmap' || :"
419 eval "$repackcmd"
420 [ ! -e .gc_failed ] || exit 1
421 allpacks="$(echo objects/pack/pack-$octet20.pack)"
422 curhead="$(cat HEAD)"
423 pkrf=
424 [ ! -e packed-refs ] || pkrf=packed-refs
425 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | awk '{print $1}') | \
426 sed -e 's/ / + /g') ))"
427 # The -A option to `git repack` may have caused some loose objects to pop
428 # out of their packs. We must make these objects group writable so that they
429 # can be freshened by other pushers. Technically we need only do this for
430 # push projects but to enable mirror projects to be more easily converted to
431 # push projects, we go ahead and do it for all projects.
432 { find objects/$octet -type f -name "$octet19" -print0 | xargs -0 chmod ug+w || :; } 2>/dev/null
433 # The git prune command does not take a -q or --quiet but started outputting
434 # 'Checking connectivity' progress messages in v1.7.9. However, we can
435 # suppress those by piping through cat as it only activates the progress
436 # messages when stderr is a tty. We only expire loose objects older than one
437 # day just in case there's some pending action (such as sending out a ref
438 # update) in progress that might want to examine them. This may leave us with
439 # loose objects. That's okay because at the next gc interval, we will always
440 # run gc if we see any loose objects regardless of whether or not we've seen
441 # any updates or we've received new linked objects from our parent. Note that
442 # in order to keep loose objects that just recently became unreferenced but
443 # have a very old modification date around we rely on some help from both the
444 # update.sh and hooks/pre-receive scripts. Furthermore, since Git v2.2.0
445 # (d3038d22 prune: keep objects reachable from recent objects) an unreachable
446 # object that would otherwise be pruned (because it's too old) will be kept
447 # alive by an unreachable object that refers to it that's not old enough to
448 # be pruned yet.
449 prunecmd='git prune --expire 1_day_ago'
450 [ -n "$show_progress" ] || \
451 prunecmd="{ $prunecmd 2>&1 || touch .gc_failed; } | cat"
452 eval "$prunecmd"
453 [ ! -e .gc_failed ] || exit 1
454 git update-server-info
456 # darcs:// mirrors have a xxx.log file that will grow endlessly
457 # if this is a mirror and the file exists, shorten it to 10000 lines
458 # also take this opportunity to optimize the darcs repo
459 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
460 url="$(config_get baseurl || :)"
461 case "$url" in darcs://*)
462 if [ -n "$cfg_mirror_darcs" ]; then
463 url="${url%/}"
464 basedarcs="$(basename "${url#darcs:/}")"
465 if [ -f "$basedarcs.log" ]; then
466 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
467 mv -f "$basedarcs.log.$$" "$basedarcs.log"
469 if [ -d "$basedarcs.darcs" ]; then
471 cd "$basedarcs.darcs"
472 # Note that this does not optimize _darcs/inventories/ :(
473 darcs optimize
477 esac
480 # Create a matching .bndl header file for the all-in-one pack we just created
481 # but only if we're not a fork (otherwise the bundle would not be complete)
482 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
483 if [ ! -s objects/info/alternates ] && [ -n "$var_have_git_172" ]; then
484 # There should only be one pack in $allpacks but if there was a
485 # simultaneous push...
486 # The one we just created will have a .idx and will NOT have a .keep
487 pkfound=
488 pkhead=
489 for pk in $allpacks; do
490 [ -s "$pk" ] || continue
491 pkbase="${pk%.pack}"
492 [ -s "$pkbase.idx" ] || continue
493 [ ! -e "$pkbase.keep" ] || continue
494 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
495 pkfound="$pkbase"
496 break;
498 done
499 if [ -n "$pkfound" -a -n "$pkhead" ]; then
501 echo "# v2 git bundle"
502 sed -ne "/^$octet20 refs\/[^ $tab]*\$/ p" < packed-refs
503 echo "$pkhead HEAD"
504 echo ""
505 } > "$pkbase.bndl"
506 bndletag="$("$cfg_basedir/bin/rangecgi" --etag -m 1 "$pkbase.bndl" "$pkbase.pack" || :)"
507 bndlsha="$(printf '%s' "$bndletag" | git hash-object --stdin || :)"
508 if [ -n "$bndletag" ]; then
509 case "$bndlsha" in $octet20)
510 bndlshatrailer="${bndlsha#????????}"
511 bndlshaprefix="${bndlsha%$bndlshatrailer}"
512 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
513 [ -d bundles ] || mkdir bundles
514 echo "${pkbase#objects/pack/}.bndl" > "bundles/$bndlname"
515 echo "${pkbase#objects/pack/}.pack" >> "bundles/$bndlname"
516 ln -s -f -n "$bndlname" bundles/latest
517 esac
522 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
523 config_set_raw girocco.reposizek "${reposizek:-0}"
525 # We use $gcstart here to avoid a race where a push occurs during the gc itself
526 # and the next future gc could be incorrectly skipped if we used the current
527 # timestamp here instead
528 config_set lastgc "$gcstart"
529 rm -f "$lockf"
531 progress "- [$proj] garbage check (`date`)"