jobd.sh: run jobd.pl more efficiently
[girocco.git] / jobd / gc.sh
blobe4c65e69eed4504a13c3fc363987451dec967aa3
1 #!/bin/sh
3 # NOTE: additional options can be passed to git repack by specifying
4 # them after the project name, for example:
5 # gc.sh my-project -f
7 . @basedir@/shlib.sh
9 set -e
11 if [ $# -lt 1 ]; then
12 echo "Usage: gc.sh projname [extra-repack-args]" >&2
13 exit 1
16 # packing options
17 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
18 quiet=; [ -n "$show_progress" ] || quiet=-q
20 umask 002
21 [ "$cfg_permission_control" != "Hooks" ] || umask 000
22 clean_git_env
24 pidactive() {
25 if _result="$(kill -0 "$1" 2>&1)"; then
26 # process exists and we have permission to signal it
27 return 0
29 case "$_result" in *"not permitted"*)
30 # we do not have permission to signal the process
31 return 0
32 esac
33 # process does not exist
34 return 1
37 createlock() {
38 # A .lock file should only exist for much less than a second.
39 # If we see a stale lock file (> 1h old), remove it and then,
40 # just in case, wait 30 seconds for any process whose .lock
41 # we might have just removed (it's racy) to finish doing what
42 # should take much less than a second to do.
43 _stalelock="$(find "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null)" || :
44 if [ -n "$_stalelock" ]; then
45 rm -f "$_stalelock"
46 sleep 30
48 for _try in p p n; do
49 if (set -C; >"$1.lock") 2>/dev/null; then
50 echo "$1.lock"
51 return 0
53 # delay and try again
54 [ "$_try" != "p" ] || sleep 1
55 done
56 # cannot create lock file
57 return 1
60 # The pre-receive script creates one ref log file per push but we want them to
61 # be coalesced into one ref log file per day. We are guaranteed that any files
62 # we find to coalesce are NOT currently being written to since they are always
63 # written first as temporary files and then moved into place. We attempt to
64 # transfer the most recent modification time to the coalesced log file which
65 # would step on its mod time if it were being written to directly, but if we
66 # find per-process ref log files then it must be a push project and the only
67 # thing that would write directly to the main per-day log file would be a
68 # mirror project so there's actually no conflict.
69 # Also, if the clock is wonky (or was futzed with) we may have both YYYYMMDD
70 # and YYYYMMDD.gz present in which case combine them into YYYYMMDD
71 coalesce_reflogs() {
72 [ -d reflogs ] || return 0
73 rm -f .gc_failed
74 find reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
75 while read -r rname; do
76 if [ -e "$rname.gz" ]; then
77 if [ -s "$rname" ]; then
78 # Presumably the .gz file must have been created before the non-gz
79 # file since it had to be uncompressed at some point therefore
80 # we need to append the non-gz contents to it but keep the non-gz
81 # contents timestamp so we rename to YYYYMMDD_ which will sort first
82 # and be picked up in the next step if we are interrupted in the middle.
83 # If a YYYYMMDD_ file already exists we append to it and transfer the
84 # timestamp. Finally we transfer the YYYYMMDD_ timestamp to the result
85 # and remove the YYYYMMDD_ temporary file leaving the result uncompressed.
86 if [ -e "${rname}_" ]; then
87 cat "$rname" >>"${rname}_"
88 touch -r "$rname" "${rname}_"
89 rm -f "$rname"
90 ! [ -e "$rname" ]
91 else
92 mv "$rname" "${rname}_"
94 gzip -d "$rname.gz" </dev/null
95 [ -e "$rname" ] && ! [ -e "$rname.gz" ]
96 cat "${rname}_" >>"$rname"
97 touch -r "${rname}_" "$rname"
98 rm -f "${rname}_"
99 else
100 # Just remove the empty file to resolve the problem
101 rm -f "$rname"
104 done
105 find reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*" -print | LC_ALL=C sort |
106 while read -r rname; do
107 logname="${rname%%_*}"
108 # If someone's been futzing with the date, the file we want to
109 # append to could already have been compressed, so we just uncompress
110 # it here. The previous block guarantees we do not have both a compressed
111 # and uncompressed version present at the same time.
112 if [ -e "$logname.gz" ]; then
113 gzip -d "$logname.gz" </dev/null
114 [ -e "$logname" ] && ! [ -e "$logname.gz" ]
116 cat "$rname" >>"$logname"
117 touch -r "$rname" "$logname"
118 rm -f "$rname"
119 if [ -e "$rname" ]; then
120 >.gc_failed
121 echo "! [$proj] failed to remove $rname" >&2
122 exit 1 # will only exit subshell created by "|"
124 done
125 ! [ -e .gc_failed ]
128 # Remove any files in reflogs that are older than $cfg_reflogs_lifetime days
129 prune_reflogs() {
130 [ -d reflogs ] || return 0
131 exp="$(( ${cfg_reflogs_lifetime:-1} * 1440 ))"
132 [ $exp -gt 0 ] || exp=1440
133 [ $exp -le 43200 ] || exp=43200
134 find reflogs -maxdepth 1 -type f -mmin "+$exp" -exec rm -f '{}' + || :
137 # Compact any reflogs that are not today's UTC date unless a .gz version exists
138 compact_reflogs() {
139 [ -d reflogs ] || return 0
140 _td="reflogs/$(TZ=UTC date '+%Y%m%d')"
141 find reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
142 while read -r rname; do
143 [ "$rname" != "$_td" ] || continue
144 ! [ -e "$rname.gz" ] || continue
145 gzip -9 "$rname" </dev/null
146 done
149 # return true if there's more than one objects/pack-<sha>.pack file or
150 # ANY sha-1 files in objects
151 is_dirty() {
152 _packs=$(find objects/pack -type f -name "pack-$octet20.pack" -print | head -n 2 | LC_ALL=C wc -l)
153 if [ $_packs != 1 ] && [ $_packs != 0 ]; then
154 return 0
156 _objs=$(find objects/$octet -type f -name "$octet19" -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l)
157 [ $_objs -ne 0 ]
160 # make sure combine-packs uses the correct Git executable
161 run_combine_packs() {
162 PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@"
165 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
166 # input pack names are read from standard input one per line delimited by the first
167 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
168 # all arguments, if any, are passed to pack-objects as additional options
169 # returns non-zero on failure AND creates .gc_failed in that case
170 combine_packs() {
171 rm -f .gc_failed
172 find objects/pack -maxdepth 1 -type f -name '*.zap*' -exec rm -f '{}' + || :
173 run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty || {
174 >.gc_failed
175 return 1
177 return 0
180 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
181 repack_gfi_packs() {
182 [ -n "$gfi_mirror" ] || return 0
183 [ -d objects/pack ] || { rm -f gfi-packs; return 0; }
184 progress "~ [$proj] redeltifying poor quality git fast-import packs"
185 combine_packs --ignore-missing --no-reuse-delta <gfi-packs
186 rm -f gfi-packs
187 return 0
190 # combine small packs into larger pack(s)
191 # we avoid any keep, bndl or bitmap packs
192 # if the optional argument is non-empty even a single small pack will be redeltad
193 combine_small_packs() {
194 _didprogress=
195 _minsmallpacks=2
196 if [ -n "$1" ] && [ -n "$noreusedeltaopt" ]; then
197 _minsmallpacks=1
199 _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
200 _lpo="$_lpo --object-limit $var_redelta_threshold objects/pack"
201 while
202 _cnt="$(list_packs --count $_lpo)" || :
203 test "${_cnt:-0}" -ge $_minsmallpacks
205 [ -n "$_didprogress" ] || {
206 progress "~ [$proj] combining small packs into a single larger pack"
207 _didprogress=1
209 _newp="$(list_packs $_lpo | combine_packs --names $noreusedeltaopt)"
210 _newc="$(( $(echo "$_newp" | LC_ALL=C wc -w) ))"
211 # be paranoid and exit the loop if we haven't reduced the number of packs
212 [ $_newc -lt $_cnt ] || break
213 _minsmallpacks=2
214 done
215 return 0
218 # Unfortunately git-svn lacks the ability to store newly fetched revisions as a pack.
219 # However, the fetch code conveniently sets .svnpack just before it runs git-svn fetch
220 # so that it's easy to find all the objects that have been fetched by git-svn and
221 # combine them into a pack. The --no-reuse-delta option is meaningless here since
222 # everything to be packed is a loose object and therefore not a delta so deltification
223 # will always take place.
224 make_svn_pack() {
225 [ -f .svnpack ] && [ -n "$svn_mirror" ] || return 0
226 rm -f .svnpackgc
227 mv -f .svnpack .svnpackgc
228 progress "~ [$proj] combining loose git-svn objects into a pack"
229 _newp="$(find objects/$octet -maxdepth 1 -type f -newer .svnpackgc -name "$octet19" -print 2>/dev/null |
230 LC_ALL=C awk -F / '{print $2 $3}' |
231 run_combine_packs --objects --names $packopts --incremental --all-progress-implied $quiet --non-empty)" || {
232 mv -f .svnpackgc .svnpack
233 >.gc_failed
234 return 1
236 if [ -n "$_newp" ]; then
237 # remove the now-redundant loose objects -- this is always safe
238 # even during a concurrent push because a reprepare_packed_git
239 # will be triggered if an object that should be there is not
240 # found thereby finding it in the new pack instead
241 git prune-packed $quiet
243 rm -f .svnpackgc
246 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
247 pack_is_complete() {
248 # Must have a matching .idx file and a non-empty packed-refs file
249 [ -s "${1%.pack}.idx" ] || return 1
250 [ -s "$2" ] || return 1
251 _headsha=
252 case "$3" in
253 $octet20)
254 _headsha="$3"
256 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
257 _headmatch="${3#ref:}"
258 _headmatch="${_headmatch# }"
259 _headmatchpat="$(echo "$_headmatch" | LC_ALL=C sed -e 's/\([.$]\)/\\\1/g')"
260 _headsha="$(LC_ALL=C grep -e "^$octet20 $_headmatchpat\$" <"$2" |
261 LC_ALL=C cut -d ' ' -f 1)"
262 case "$_headsha" in $octet20) :;; *)
263 return 1
264 esac
267 # bad HEAD
268 return 1
269 esac
270 rm -rf pack_is_complete_test
271 mkdir pack_is_complete_test
272 mkdir pack_is_complete_test/refs
273 mkdir pack_is_complete_test/objects
274 mkdir pack_is_complete_test/objects/pack
275 echo "$_headsha" >pack_is_complete_test/HEAD
276 ln -s "$1" pack_is_complete_test/objects/pack/
277 ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
278 ln -s "$2" pack_is_complete_test/packed-refs
279 _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null)" || :
280 rm -rf pack_is_complete_test
281 [ -n "$_count" ] || return 1
282 [ "$_count" -gt 0 ] 2>/dev/null || return 1
283 echo "$_headsha"
286 # On return a "$lockf" will have been created that must be removed when gc is done
287 lock_gc() {
288 # be compatibile with gc.pid file from newer Git releases
289 lockf=gc.pid
290 hn="$(hostname)"
291 active=
292 if [ "$(createlock "$lockf")" ]; then
293 # If $lockf is:
294 # 1) less than 12 hours old
295 # 2) contains two fields (pid hostname) NO trailing NL
296 # 3) the hostname is different OR the pid is still alive
297 # then we exit as another active process is holding the lock
298 if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
299 apid=
300 ahost=
301 read -r apid ahost ajunk <"$lockf" || :
302 if [ "$apid" ] && [ "$ahost" ]; then
303 if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
304 active=1
308 else
309 echo >&2 "[$proj] unable to create gc.pid.lock file"
310 exit 1
312 if [ -n "$active" ]; then
313 rm -f "$lockf.lock"
314 echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
315 exit 1
317 printf "%s %s" "$$" "$hn" >"$lockf.lock"
318 chmod 0664 "$lockf.lock"
319 mv -f "$lockf.lock" "$lockf"
322 # Create a repack subdirectory such that running repack in it will pack the
323 # same things that a pack in the normal directory would except that the pack
324 # is guaranteed to be generated in an optimized order by adding a suitable
325 # synthesized ref in the refs/tags namespace (yes, pack-objects.c really does
326 # behave differently depending on the contents of the refs/tags namespace).
327 # Before calling this, pack-refs --all MUST be performed or the wrong pack
328 # will end up being made.
329 # If a ref deletion is pushed after making the repack subdir but before the
330 # the actual repack, the discarded objects will be packed -- no big deal,
331 # they'll get discarded the next time gc runs.
332 # If a fast-forward ref update is pushed after making the repack subdir but
333 # before the actual repack, it will be picked up and the new objects packed
334 # (subject to the normal git repack race about picking such updates up).
335 # If a non-fast-forward ref update is pushed after making the repack subdir but
336 # before the actual repack, it will be picked up like a fast-forward update but
337 # the discarded objects will be included like a ref deletion (until the next
338 # scheduled gc takes place).
339 make_repack_dir() {
340 ! [ -d repack ] || rm -rf repack
341 ! [ -d repack ] || { echo >&2 "[$proj] cannot remove repack subdirectory"; exit 1; }
342 mkdir repack repack/refs
343 [ -d info ] || mkdir info
344 ln -s ../config repack/config
345 ln -s ../info repack/info
346 ln -s ../objects repack/objects
347 ln -s ../../refs repack/refs/refs
348 _lines=$(( $(LC_ALL=C wc -l <packed-refs) ))
349 sed 's, refs/, refs/!/,' <packed-refs >repack/packed-refs
350 headref="$(git rev-parse --verify --quiet HEAD)" || :
351 if [ -n "$headref" ]; then
352 echo "$headref refs/heads/!!" >>repack/packed-refs
353 _lines=$(( $_lines + 1 ))
355 optref="$(git rev-list -n 1 --all 2>/dev/null)" || :
356 if [ -n "$optref" ]; then
357 echo "$optref refs/tags/!" >>repack/packed-refs
358 _lines=$(( $_lines + 1 ))
359 echo "$optref" >repack/HEAD
360 else
361 cat HEAD >repack/HEAD
363 if [ $(LC_ALL=C wc -l <repack/packed-refs) -ne "$_lines" ]; then
364 echo >&2 "[$proj] error: make_repack_dir failed packed-refs line count sanity check"
365 exit 1
369 # Remove any crud that's been left behind by interrupted operations
370 # that did not clean up after themselves
371 remove_crud() {
372 # Remove any existing FETCH_HEAD
373 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
374 # receiving pushes (those never create a FETCH_HEAD).
375 # And if we're fetching because we're a mirror, we know we're not fetching right
376 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
377 # Therefore any existing FETCH_HEAD is junk. And it may be many megabytes if
378 # there were a lot of refs.
379 rm -f FETCH_HEAD
381 # remove any existing pack_is_complete_test or repack subdirectories
382 # If either exists when this function is called it's crud
383 rm -rf pack_is_complete_test repack
385 # Remove any stale pack remnants that are more than an hour old.
386 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
387 # .pack AND the corresponding .pack DOES NOT exist. A bunch of stale
388 # pack-<sha1>.idx files without their corresponding .pack files are worthless
389 # and just waste space. Normally there shouldn't be any remnants but actually
390 # this can happen when things are interrupted at just the wrong time.
391 # Note that the objects/pack directory is created by git init and should
392 # always exist.
393 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" -print |
394 LC_ALL=C sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u |
395 while read packsha; do
396 ! [ -e "objects/pack/pack-$packsha.pack" ] || continue
397 rm -f "objects/pack/pack-$packsha".?*
398 done
400 # Remove any stale tmp reflogs files that are more than one hour old.
401 # Since they are created only while the pre-receive hook is running and
402 # all it does is process a bunch of refs passed to it on standard input
403 # it's inconceivable that it would ever take as much as an hour to run.
404 if [ -d reflogs ]; then
405 find reflogs -maxdepth 1 -type f -mmin +60 -name "tmp_*" -exec rm -f '{}' + || :
408 # Remove any stale object tmp_obj_* files that are more than 3 hours old.
409 # Really these files should only exist very briefly so there shouldn't be any
410 # but things happen that can end up leaving them behind.
411 find objects/$octet -maxdepth 1 -type f -mmin +180 -name "tmp_obj_?*" -exec rm -f '{}' + 2>/dev/null || :
413 # Remove any stale pack .keep files that are more than 12 hours old.
414 # We don't do anything to create any permanent pack .keep files, so they must
415 # be remnants from some failed push or something. Removing the .keep will
416 # allow the pack to be properly repacked.
417 find objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20.keep" -exec rm -f '{}' + || :
419 # Remove any stale tmp_pack_*, tmp_idx_*, tmp_bitmap_*, packtmp-* or .tmp-*-pack* files
420 # that are more than 12 hours old.
421 find objects/pack -maxdepth 1 -type f -mmin +720 \( \
422 -name "tmp_pack_?*" -o -name "tmp_idx_?*" -o -name "tmp_bitmap_?*" -o \
423 -name "packtmp-?*" -o -name ".tmp-?*-pack*" \
424 \) -exec rm -f '{}' + || :
426 # Remove any stale incoming-* object quarantine directories that are
427 # more than 12 hours old. These are new with Git >= 2.11.0.
428 find objects -maxdepth 1 -type d -name 'incoming-?*' -mmin +720 \
429 -exec rm -rf '{}' + || :
431 # Remove any stale shallow_* files that are more than 12 hours old.
432 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
433 # requests a shallow clone.
434 find . -maxdepth 1 -type f -mmin +720 -name "shallow_?*" -exec rm -f '{}' + || :
436 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
437 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
438 find objects -type f -mmin +720 -name "*.temp" -exec rm -f '{}' + || :
440 # Remove any stale *.lock files in the htmlcache area that might have been left
441 # behind after an abnormal exit during an attempt to update a cached file and
442 # are more than 1 hour old.
443 ! [ -d htmlcache ] || find htmlcache -type f -mmin +60 -name "*.lock" -exec rm -f '{}' + || :
445 # Remove any stale git-svn temp files that are more than 12 hours old.
446 # The git-svn process creates temp files with random 10 character names
447 # in the root of $GIT_DIR. Unfortunately they do not have a recognizable
448 # prefix, so we just have to kill any files with a 10-character name. We
449 # do this only for git-svn mirrors. All characters are chosen from
450 # [A-Za-z0-9_] so we can at least check that and fortunately the only
451 # collision is 'FETCH_HEAD' but that shouldn't matter.
452 # There may also be temp files with a Git_ prefix as well.
453 if [ -n "$svn_mirror" ]; then
454 _randchar='[A-Za-z0-9_]'
455 _randchar2="$_randchar$_randchar"
456 _randchar4="$_randchar2$_randchar2"
457 _randchar10="$_randchar4$_randchar4$_randchar2"
458 find . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -exec rm -f '{}' + || :
459 find . -maxdepth 1 -type f -mmin +720 -name "Git_*" -exec rm -f '{}' + || :
462 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
463 if [ -n "$gfi_mirror" ]; then
464 find . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -exec rm -f '{}' + || :
467 # Remove any stale core or *.core or core.* files that are more than 3 days old.
468 find . -maxdepth 1 -type f -mmin +4320 \( -name "core" -o -name "*.core" -o -name "core.*" \) \
469 -exec rm -f '{}' + || :
473 ## Garbage Collection Types
475 ## There are two kinds of possible garbage collection (gc) operations:
477 ## 1. A normal, full gc
478 ## 2. A "mini" gc
480 ## If the full garbage collection interval has expired (or gc has never been
481 ## run), then a normal, full gc will take place. Otherwise, a "mini" gc will
482 ## take place if the file .needsgc exists.
484 ## A "mini" gc is similar to "git gc --auto" in that it may not end up actually
485 ## doing anything unless the right conditions are present so it's not a burden
486 ## to run it often. If the file .needsgc exists, a "mini" gc will occur at
487 ## the next opportunity.
489 ## Note, however, that the .nogc file suppresses ALL gc activity (normal or mini).
492 proj="${1%.git}"
493 shift
494 cd "$cfg_reporoot/$proj.git"
495 [ -d objects/pack ] || { rm -f gfi-packs; mkdir -p objects/pack; }
496 mirror_url="$(get_mirror_url)"
497 svn_mirror=
498 ! is_svn_mirror_url "$mirror_url" || svn_mirror=1
499 gfi_mirror=
500 if [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$mirror_url"; then
501 gfi_mirror=1
504 # If git config --bool --get girocco.redelta is explicitly false then automatic
505 # redelta when there are less than $var_redelta_threshold objects will be suppressed.
506 # On the other hand, if git config --get girocco.redelta is "always" then, on a full
507 # gc only, for the final repack, deltas will always be recomputed.
508 # This can be set on a per-project basis to avoid unusual pathological gc behavior.
509 # Setting this will hurt efficiency of the affected repository.
510 # Note that fast-import packs ALWAYS get new deltas regardless of this setting.
511 noreusedeltaopt="--no-reuse-delta"
512 [ "$(git config --bool --get girocco.redelta 2>/dev/null || :)" != "false" ] || noreusedeltaopt=
513 alwaysredelta=
514 [ "$(git config --get girocco.redelta 2>/dev/null || :)" != "always" ] || alwaysredelta=1
516 trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
517 trap 'exit 130' INT
518 trap 'exit 143' TERM
520 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
521 datefmt='+%a, %d %b %Y %T %z'
523 isminigc=
524 if [ "${force_gc:-0}" = "0" ] && check_interval lastgc $cfg_min_gc_interval; then
525 if [ -e .needsgc ]; then
526 isminigc=1
527 else
528 progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
529 exit 0
532 if [ -e .nogc ]; then
533 progress "x [$proj] garbage check disabled"
534 exit 0
537 if [ -n "$isminigc" ]; then
538 # Perform a "mini" gc
539 # Note that .delaygc is ignored here as that's only intended for full gc
540 lock_gc
541 rm -f .allowgc .needsgc
542 remove_crud
543 coalesce_reflogs
544 prune_reflogs
545 compact_reflogs
546 miniactive=
547 if [ -f .svnpack ] && [ -n "$svn_mirror" ]; then
548 miniactive=1
549 progress "+ [$proj] mini garbage check ($(date))"
550 make_svn_pack
552 if [ -z "$cfg_delay_gfi_redelta" ] && [ -n "$gfi_mirror" ]; then
553 # $Girocco::Config::delay_gfi_redelta is false, force redeltification now
554 if [ -z "$miniactive" ]; then
555 miniactive=1
556 progress "+ [$proj] mini garbage check ($(date))"
558 repack_gfi_packs
560 # If there aren't at least 10 non-keep, non-bitmap, non-bndl packs then
561 # don't actually process them yet
562 lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
563 packcnt="$(list_packs --count $lpo objects/pack)" || :
564 if [ "${packcnt:-0}" -ge 10 ]; then
565 if [ -z "$miniactive" ]; then
566 miniactive=1
567 progress "+ [$proj] mini garbage check ($(date))"
569 if [ -n "$gfi_mirror" ]; then
570 repack_gfi_packs
571 packcnt="$(list_packs --count $lpo objects/pack)" || :
573 # if repack_gfi_packs dropped the pack count to < 10 don't combine
574 if [ "${packcnt:-0}" -ge 10 ]; then
575 combine_small_packs
576 packcnt="$(list_packs --count $lpo objects/pack)" || :
578 # if we still have more than 10 packs trigger a full gc
579 if [ "${packcnt:-0}" -ge 10 ]; then
580 # We shouldn't be in a .delaygc state at this point, but if
581 # we are then nuke it because we really need a full gc now
582 rm -f .delaygc
583 git config --unset gitweb.lastgc
584 rm -f "$lockf"
585 progress "- [$proj] mini garbage check triggering full gc too many packs ($(date))"
586 exit 0
589 rm -f "$lockf"
590 if [ -n "$miniactive" ]; then
591 git update-server-info
592 progress "- [$proj] mini garbage check ($(date))"
593 else
594 progress "= [$proj] mini garbage check nothing but crud removal to do ($(date))"
596 exit 0
599 # Avoid unnecessary garbage collections:
600 # 1. If lastreceive is set and is older than lastgc
601 # -AND-
602 # 2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
604 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
605 # If we are a fork and lastparentgc is NOT set we MUST run gc
607 # If the repo is dirty after removing any crud we MUST run gc
609 gcstart="$(date "$datefmt")"
610 skipgc=
611 isfork=
612 ! [ -s objects/info/alternates ] || isfork=1
613 lastparentgcsecs=
614 [ -z "$isfork" ] || lastparentgcsecs="$(config_get_date_seconds lastparentgc)" || :
615 lastreceivesecs=
616 if lastreceivesecs="$(config_get_date_seconds lastreceive)" &&
617 [ "${force_gc:-0}" = "0" ] &&
618 lastgcsecs="$(config_get_date_seconds lastgc)" &&
619 [ $lastreceivesecs -lt $lastgcsecs ]; then
620 # We've run gc since we last received, so maybe we can skip,
621 # check if not fork or fork and lastparentgc < lastgc
622 if [ -n "$isfork" ]; then
623 if [ -n "$lastparentgcsecs" ] &&
624 [ $lastparentgcsecs -lt $lastgcsecs ]; then
625 # We've run gc since our parent ran gc so we can skip
626 skipgc=1
628 else
629 # We don't have any alternates (we're not a forK) so we can skip
630 skipgc=1
634 # Prevent any other simultaneous gc operations
635 lock_gc
637 # At this point, if .allowgc exists, it's now crud to be removed
638 rm -f .allowgc
640 # Ideally we would do this in post-receive, but that would mean duplicating the
641 # logic so it's available in the chroot jail and that's highly undesirable
642 # Instead, since the first gc will be triggered immediately following the first
643 # push, we do the check here as it's quick and harmless if HEAD is already valid
644 check_and_set_head || :
646 # Always get rid of crud
647 remove_crud
649 # Always perform reflogs maintenance
650 coalesce_reflogs
651 prune_reflogs
652 compact_reflogs
654 # Run 'git svn gc' now for svn mirrors
655 if [ -n "$svn_mirror" ]; then
656 git svn gc || :
659 # Skip the actual gc if .delaygc is set
660 if [ -e .delaygc ]; then
661 progress "x [$proj] garbage check delayed (except for crud removal)"
662 rm -f "$lockf"
663 exit 0
666 # Do not skip gc if the repo is dirty
667 if [ -n "$skipgc" ] && ! is_dirty; then
668 progress "= [$proj] garbage check nothing but crud removal to do ($(date))"
669 config_set lastgc "$gcstart"
670 rm -f "$lockf"
671 exit 0
674 bumptime=
675 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
676 # set lastparentgc and then update gcstart to be at least 1 second later
677 config_set lastparentgc "$gcstart"
678 bumptime=1
680 if [ -z "$lastreceivesecs" ]; then
681 # set lastreceive and then update gcstart to be at least 1 second later
682 config_set lastreceive "$gcstart"
683 bumptime=1
685 if [ -n "$bumptime" ]; then
686 sleep 1
687 gcstart="$(date "$datefmt")"
690 progress "+ [$proj] garbage check ($(date))"
692 newdeltas=
693 [ -z "$alwaysredelta" ] || newdeltas=-f
694 for _arg in "$@"; do
695 case "$_arg" in "-f"|"-F")
696 newdeltas="$_arg"
697 esac
698 done
699 if [ -z "$newdeltas" ] && [ -n "$gfi_mirror" ]; then
700 if [ $(list_packs --exclude-no-idx --count objects/pack) -le \
701 $(list_packs --exclude-no-idx --count --quiet --only gfi-packs) ]; then
702 # Don't bother with repack_gfi_packs since everything's being repacked
703 newdeltas=-f
706 if [ -z "$newdeltas" ] && [ -n "$noreusedeltaopt" ] &&
707 [ $(list_packs --exclude-no-idx --count-objects objects/pack) -le $var_redelta_threshold ]; then
708 # There aren't enough objects to worry about so just redelta to get the best pack
709 newdeltas=-f
711 if [ -z "$newdeltas" ]; then
712 # Since we're not going to recompute deltas overall, we need to do the
713 # "mini" maintenance so that we can get more optimal deltas
714 [ -z "$noreusedeltaopt" ] || make_svn_pack
715 repack_gfi_packs
716 force_single_pack_redelta=
717 [ -n "$gfi_mirror" ] || [ -n "$svn_mirror" ] || force_single_pack_redelta=1
718 [ -z "$noreusedeltaopt" ] || combine_small_packs $force_single_pack_redelta
722 ## Safe Pruning In Forks
724 ## We are about to perform garbage collection. We do NOT use the "git gc"
725 ## command directly as it does not provide enough control over the fine details
726 ## that we require. However, we DO maintain a "gc.pid" file during our garbage
727 ## collection so that a simultaneous "git gc" by an administrator will be
728 ## blocked (and similarly we refuse to start garbage collection if we cannot
729 ## create the "gc.pid" file). When we say "gc" in the below description we are
730 ## referring to our "gc.sh" script, NOT the "git gc" command.
732 ## If the project we are running garbage collection (gc) on has any forks we
733 ## must be careful not to remove any objects that while no longer referenced by
734 ## this project (the parent) are still referenced by one or more forks (the
735 ## children) otherwise the children will become corrupt and we can't abide
736 ## corrupt children.
738 ## One way to accomplish this is to simply hard-link all currently existing
739 ## loose objects and packs in the parent into all the children that refer to the
740 ## parent (via a line in their objects/info/alternates file) before beginning
741 ## the gc operation and then relying on a subsequent gc in the child to clean up
742 ## any excess objects/packs. We used to use this strategy but it's very
743 ## inefficient because:
745 ## 1. The disk space used by the old pack(s)/object(s) will not be reclaimed
746 ## until all children (and their children, if any) run gc by which time
747 ## it's quite possible the topmost parent will have run gc again and
748 ## hard-linked yet another old pack down to its children (not to mention
749 ## loose objects).
751 ## 2. As we are now using the "-A" option with "git repack", any new objects
752 ## in the parent that are not referenced by children will continually get
753 ## exploded out of the hard-linked pack in the children whenever the
754 ## children run gc.
756 ## 3. To avoid suboptimal and/or unnecessarily many packs being hard-linked
757 ## into child forks, we must run the "mini" gc maintenance before we
758 ## perform the hard-linking into the children which provides yet another
759 ## source of inefficiency.
761 ## Since we are using the "-A" option to "git repack" (that was not always the
762 ## case) to guarantee we can access old ref values for long enough to send out
763 ## a meaningful mail.sh notification, we now have another, more efficient,
764 ## option available to prevent corruption of child forks that continue to refer
765 ## to objects that are no longer reachable from any ref in the parent.
767 ## The only things that need be copied (or hard-linked) into the child fork(s)
768 ## are those objects that have become unreachable from any ref in the parent.
769 ## They are the only things that could ever be removed by "git prune" and
770 ## therefore the only things we need to prevent the loss of in order to avoid
771 ## corruption of the child fork(s).
773 ## Therefore we now use the following strategy instead to avoid excessive disk
774 ## use and lots of unnecessary loose objects in child forks:
776 ## 1. Run "git repack -A -d -l" in the parent BEFORE doing anything about
777 ## child forks.
779 ## 2. Hard-link all remaining existing loose objects in the parent into the
780 ## immediate child forks.
782 ## 3. Now run "git prune" in the parent.
784 ## With this new strategy we avoid the need to run any "mini" gc maintenance
785 ## before copying (or hard-linking) anything down to the child forks.
786 ## Furthermore, only when the parent performs a non-fast-forward update will
787 ## anything ever be transferred to the children leaving them unperturbed in the
788 ## vast majority of cases. Finally, even if the parent references objects the
789 ## children do not, those objects will no longer continually end up in the
790 ## children as unreachable loose objects after the children run gc.
793 git pack-refs --all
794 make_repack_dir
795 touch .gc_in_progress
796 rm -f .gc_failed bundles/*
797 rm -f objects/pack/pack-*.bndl
798 # We use the -A option with git repack so that unreachable objects can live
799 # on for a time as loose objects. This is particularly helpful if we just
800 # happen to be in the process of sending out a ref update for a ref that was
801 # force updated and the old ref value would have otherwise been removed by
802 # repack because it was now unreachable. Admittedly the window for gc to run
803 # and do that before we manage to send out the ref update is not large, but
804 # it would not be difficult to create such a situation. Unfortunately, when
805 # Git unpacks these unreachable objects it will give them the modification
806 # time of the *.pack file they came out of. This could be very, very old.
807 # If that happens, the subsequent git prune --expire some_time_ago will still
808 # remove the object(s) and our pending ref update will still lose out.
809 # To prevent this from happening and to get the behavior we want, we now
810 # touch the modification time of all pack-<sha>.pack files so that any
811 # loosened objects get a current time. Git does not provide any other
812 # mechanism to do this. We do not want to just touch all loose objects
813 # left after the repack because that would cause objects that were loosened
814 # previously to live on which we definitely do not want.
815 list_packs --exclude-no-idx objects/pack | xargs touch -c 2>/dev/null || :
816 # We wish to keep deltas from our last full pack so if we're not redeltaing
817 # then make sure the .pack associated with the .bitmap has a newer mod time
818 # (If there is no .bitmap then touch the pack with the most objects instead.)
819 if [ -z "$newdeltas" ]; then
820 bmpack="$(list_packs --exclude-no-bitmap --exclude-no-idx --max-matches 1 objects/pack)"
821 [ -n "$bmpack" ] || bmpack="$(list_packs --exclude-no-idx --max-matches 1 --object-limit -1 --include-boundary objects/pack)"
822 if [ -n "$bmpack" ] && [ -f "$bmpack" ] && [ -s "$bmpack" ]; then
823 sleep 1
824 touch -c "$bmpack" 2>/dev/null || :
827 # The git repack command only supports bitmaps if all objects are being packed.
828 # While it is theoretically possible that a project with a non-empty alternates
829 # file ends up packing all objects (because it does not actually use any of the
830 # objects found in the alternates), it's very unlikely. And, in the unlikely
831 # event that did occur, clients would see a message about only using one bitmap
832 # because Git can only use one bitmap at a time and at least one of the
833 # alternates is bound to have a bitmap. Therefore if we see a non-empty
834 # alternates file, we disable writing bitmaps which avoids the warning and any
835 # possibility of a client warning as well.
836 nobm=
837 [ -z "$var_have_git_172" ] || ! [ -s objects/info/alternates ] ||
838 nobm='-c repack.writebitmaps=false -c pack.writebitmaps=false'
839 progress "~ [$proj] running full gc repack${nobm:+ (bitmaps disabled)}"
840 cd repack
841 # We run git repack from the repack subdirectory so we can force optimized packs
842 # to be generated even for repositories that do not have any tagged commits
843 git $nobm repack $packopts -A -d -l $quiet $newdeltas $@
844 cd ..
845 rm -rf repack
846 ! [ -e .gc_failed ] || exit 1
847 # These, if they exist, are now meaningless and need to be removed
848 rm -f gfi-packs .needsgc .svnpack .svnpackgc
849 allpacks="$(echo objects/pack/pack-$octet20.pack)"
850 curhead="$(cat HEAD)"
851 pkrf=
852 ! [ -e packed-refs ] || pkrf=packed-refs
853 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | LC_ALL=C awk '{print $1}') |
854 LC_ALL=C sed -e 's/ / + /g') ))"
855 git update-server-info
856 # The -A option to `git repack` may have caused some loose objects to pop
857 # out of their packs. We must make these objects group writable so that they
858 # can be freshened by other pushers. Technically we need only do this for
859 # push projects but to enable mirror projects to be more easily converted to
860 # push projects, we go ahead and do it for all projects.
861 { find objects/$octet -type f -name "$octet19" -exec chmod ug+w '{}' + || :; } 2>/dev/null
863 if has_forks "$proj"; then
864 progress "~ [$proj] hard-linking loose objects into immediate child forks"
865 # We have to update the lastparentgc time in the child forks even if they do not get any
866 # new "loose objects" because they need to run gc just in case the parent now has some
867 # objects that used to only be in the child so they can be removed from the child.
868 # For example, a "patch" might be developed first in a fork and then later accepted into
869 # the parent in which case the objects making up the patch in the child fork are now
870 # redundant (since they're now in the parent as well) and need to be removed from the
871 # child fork which can only happen if the child fork runs gc.
872 forkdir="$proj"
873 # It is enough to copy objects just one level down and get_repo_list
874 # takes a regular expression (which is automatically prefixed with '^')
875 # so we can easily match forks exactly one level down from this project
876 get_repo_list "$forkdir/[^/]*:" |
877 while read fork; do
878 # Ignore forks that do not exist or are symbolic links
879 ! [ -L "$cfg_reporoot/$fork.git" ] && [ -d "$cfg_reporoot/$fork.git" ] ||
880 continue
881 # Or do not have a non-zero length alternates file
882 [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] ||
883 continue
884 # Match objects in parent project
885 for d in objects/$octet; do
886 [ "$d" != "objects/$octet" ] || continue
887 mkdir -p "$cfg_reporoot/$fork.git/$d"
888 find "$d" -maxdepth 1 -type f -name "$octet19" -exec \
889 "$var_sh_bin" -c 'ln -f "$@" '"'$cfg_reporoot/$fork.git/$d/'" sh '{}' + || :
890 done
891 # Update the fork's lastparentgc date (must be current, not $gcstart)
892 git --git-dir="$cfg_reporoot/$fork.git" config \
893 gitweb.lastparentgc "$(date "$datefmt")"
894 done
897 # The git prune command does not take a -q or --quiet but started outputting
898 # 'Checking connectivity' progress messages in v1.7.9. However, we can
899 # suppress those by piping through cat as it only activates the progress
900 # messages when stderr is a tty. We only expire loose objects older than one
901 # day just in case there's some pending action (such as sending out a ref
902 # update) in progress that might want to examine them. This may leave us with
903 # loose objects. That's okay because at the next gc interval, we will always
904 # run gc if we see any loose objects regardless of whether or not we've seen
905 # any updates or we've received new linked objects from our parent. Note that
906 # in order to keep loose objects that just recently became unreferenced but
907 # have a very old modification date around we rely on some help from both the
908 # update.sh and hooks/pre-receive scripts. Furthermore, since Git v2.2.0
909 # (d3038d22 prune: keep objects reachable from recent objects) an unreachable
910 # object that would otherwise be pruned (because it's too old) will be kept
911 # alive by an unreachable object that refers to it that's not old enough to
912 # be pruned yet.
913 prunecmd='git prune --expire 1_day_ago'
914 [ -n "$show_progress" ] ||
915 prunecmd="{ $prunecmd 2>&1 || touch .gc_failed; } | cat"
916 progress "~ [$proj] pruning expired unreachable loose objects"
917 eval "$prunecmd"
918 ! [ -e .gc_failed ] || exit 1
920 # darcs:// mirrors have a xxx.log file that will grow endlessly
921 # if this is a mirror and the file exists, shorten it to 10000 lines
922 # also take this opportunity to optimize the darcs repo
923 if ! [ -e .nofetch ] && [ -n "$cfg_mirror" ]; then
924 url="$(config_get baseurl)" || :
925 case "$url" in darcs://*)
926 if [ -n "$cfg_mirror_darcs" ]; then
927 url="${url%/}"
928 basedarcs="$(basename "${url#darcs:/}")"
929 if [ -f "$basedarcs.log" ]; then
930 tail -n 10000 "$basedarcs.log" >"$basedarcs.log.$$"
931 mv -f "$basedarcs.log.$$" "$basedarcs.log"
933 if [ -d "$basedarcs.darcs" ]; then
935 cd "$basedarcs.darcs"
936 # without show_progress suppress non-error output
937 [ -n "$show_progress" ] || exec >/dev/null
938 # Note that this does not optimize _darcs/inventories/ :(
939 darcs optimize || :
943 esac
946 # Create a matching .bndl header file for the all-in-one pack we just created
947 # but only if we're not a fork (otherwise the bundle would not be complete)
948 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
949 if ! [ -s objects/info/alternates ] && [ -n "$var_have_git_172" ]; then
950 # There should only be one pack in $allpacks but if there was a
951 # simultaneous push...
952 # The one we just created will have a .idx and will NOT have a .keep
953 progress "~ [$proj] creating downloadble bundle header"
954 pkfound=
955 pkhead=
956 for pk in $allpacks; do
957 [ -s "$pk" ] || continue
958 pkbase="${pk%.pack}"
959 [ -s "$pkbase.idx" ] || continue
960 ! [ -e "$pkbase.keep" ] || continue
961 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
962 pkfound="$pkbase"
963 break;
965 done
966 if [ -n "$pkfound" ] && [ -n "$pkhead" ]; then
968 symref=
969 case "$curhead" in "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
970 symref="${curhead#ref:}"
971 symref="${symref# }"
972 esac
973 bndlurl=
974 [ -z "$cfg_httpbundleurl" ] || bndlurl=" url=$cfg_httpbundleurl/$proj.git/clone.bundle"
975 echo "# v2 git bundle"
976 LC_ALL=C sed -ne "/^$octet20 refs\/[^ $tab]*\$/ p" <packed-refs
977 if [ -n "$symref" ]; then
978 printf "$pkhead HEAD\0symref=HEAD:%s%s\n" "$symref" "$bndlurl"
979 else
980 if [ -n "$bndlurl" ]; then
981 printf "$pkhead HEAD\0%s\n" "${bndlurl:# }"
982 else
983 echo "$pkhead HEAD"
986 echo ""
987 } >"$pkbase.bndl"
988 bndletag="$("$cfg_basedir/bin/rangecgi" --etag -m 1 "$pkbase.bndl" "$pkbase.pack")" || :
989 bndlsha="$(printf '%s' "$bndletag" | git hash-object --stdin)" || :
990 if [ -n "$bndletag" ]; then
991 case "$bndlsha" in $octet20)
992 bndlshatrailer="${bndlsha#????????}"
993 bndlshaprefix="${bndlsha%$bndlshatrailer}"
994 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
995 [ -d bundles ] || mkdir bundles
996 echo "${pkbase#objects/pack/}.bndl" >"bundles/$bndlname"
997 echo "${pkbase#objects/pack/}.pack" >>"bundles/$bndlname"
998 ln -s -f -n "$bndlname" bundles/latest
999 esac
1004 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
1005 config_set_raw girocco.reposizek "${reposizek:-0}"
1007 # We use $gcstart here to avoid a race where a push occurs during the gc itself
1008 # and the next future gc could be incorrectly skipped if we used the current
1009 # timestamp here instead
1010 config_set lastgc "$gcstart"
1011 rm -f "$lockf"
1013 progress "- [$proj] garbage check ($(date))"