3 # This is a shell library for common gc related functions
4 # used by various Girocco scripts.
6 # shlib.sh always sets this, it's an error to source
7 # this script without having already sourced shlib.sh
8 [ -n "$var_git_exec_path" ] ||
exit 2
10 # default packing options
11 packopts
="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
12 quiet
="-q"; [ "${show_progress:-0}" = "0" ] || quiet
=
15 if _result
="$(kill -0 "$1" 2>&1)"; then
16 # process exists and we have permission to signal it
19 case "$_result" in *"not permitted"*)
20 # we do not have permission to signal the process
23 # process does not exist
28 # A .lock file should only exist for much less than a second.
29 # If we see a stale lock file (> 1h old), remove it and then,
30 # just in case, wait 30 seconds for any process whose .lock
31 # we might have just removed (it's racy) to finish doing what
32 # should take much less than a second to do.
33 _stalelock
="$(find -L "$1.lock
" -maxdepth 1 -mmin +60 -print 2>/dev/null)" ||
:
34 if [ -n "$_stalelock" ]; then
39 if (set -C; >"$1.lock") 2>/dev
/null
; then
44 [ "$_try" != "p" ] ||
sleep 1
46 # cannot create lock file
50 # Create a gc.pid lockfile
51 # $1 => name of variable to store result in
53 # variable named by $1 will contain the name of the newly create lockfile (i.e. "gc.pid")
55 # variable named by $1 will contain the failure reason
57 # be compatibile with gc.pid file from newer Git releases
61 if [ "$(createlock "$_lockf")" ]; then
63 # 1) less than 12 hours old
64 # 2) contains two fields (pid hostname) NO trailing NL
65 # 3) the hostname is different OR the pid is still alive
66 # then we exit as another active process is holding the lock
67 if [ "$(find -L "$_lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
70 read -r _apid _ahost _ajunk
<"$_lockf" ||
:
71 if [ "$_apid" ] && [ "$_ahost" ]; then
72 if [ "$_ahost" != "$_hn" ] || pidactive
"$_apid"; then
78 eval "$1="'"unable to create $_lockf.lock file"'
81 if [ -n "$_active" ]; then
83 eval "$1="'"gc already running on machine '\''$_ahost'\'' pid '\''$_apid'\''"'
86 printf "%s %s" "$$" "$_hn" >"$_lockf.lock"
87 chmod 0664 "$_lockf.lock"
88 mv -f "$_lockf.lock" "$_lockf"
93 # make sure combine-packs uses the correct Git executable
95 PATH
="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@
/jobd
/combine-packs.sh
"$@"
98 # combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
99 # input pack names are read from standard input one per line delimited by the first
100 # ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
101 # all arguments, if any, are passed to pack-objects as additional options
102 # first removes any pre-existing "*.zap*" sentinels that may be leftover from any
103 # previously aborted "--replace" operations
104 # returns non-zero on failure
105 combine_packs_std
() {
106 find -L objects
/pack
-maxdepth 1 -type f
-name '*.zap*' -exec rm -f '{}' + ||
:
107 run_combine_packs
--replace "$@" $packopts --all-progress-implied $quiet --non-empty
110 # duplicate the first file to the name given by the second file making sure that
111 # the second file appears atomically all-at-once after the copy has been completed
112 # and does not appear at all if the copy fails (in which case this function fails)
113 # if the second file already exists this function fails with status 1
114 # if the file names are the same this function returns immediately with success
116 [ "$1" != "$2" ] ||
return 0
117 ! [ -e "$2" ] ||
return 1
119 *?
/?
*) _tmpdir
="${2%/*}";;
122 _tmpfile
="$(mktemp "${_tmpdir:-.}/packtmp-XXXXXX
")" ||
return 1
123 cp -fp "$1" "$_tmpfile" ||
return 1
124 mv -f "$_tmpfile" "$2"
127 # rename_pack oldnamepath newnamepath
128 # note that .keep and .bndl files are left untouched and not moved at all!
130 [ $# -eq 2 ] && [ "$1" != "$2" ] ||
{
131 echo >&2 "[$proj] incorrect use of rename_pack function"
134 # Git assumes that if the destination of the rename already exists
135 # that it is, in fact, a copy of the same bytes so silently succeeds
136 # without doing anything. We duplicate that logic here.
137 # Git checks for the .idx file first before even trying to use a pack
138 # so it should be the last moved and the first removed.
139 for ext
in pack bitmap idx
; do
140 [ -f "$1.$ext" ] ||
continue
141 ln "$1.$ext" "$2.$ext" >/dev
/null
2>&1 ||
142 dupe_file
"$1.$ext" "$2.$ext" >/dev
/null
2>&1 ||
143 [ -f "$2.$ext" ] ||
{
144 echo >&2 "[$proj] unable to move $1.$ext to $2.$ext"
148 for ext
in idx pack bitmap
; do
154 # current directory must already be set to the $GIT_DIR
155 # see if there are "lotsa" loose objects
156 # "lotsa" is defined as the 17, 68, 71 and 86 object directories existing
157 # and there being at least 5 total objects between them which corresponds
158 # to an approximate average of 320 loose objects before this function starts
159 # returning true and triggering a "mini" gc to pack up loose objects
160 lotsa_loose_objects
() {
161 [ -d objects
/17 ] && [ -d objects
/68 ] && [ -d objects
/71 ] && [ -d objects
/86 ] ||
return 1
162 _objs
=$
(( $
(find -L objects
/17 objects
/68 objects
/71 objects
/86 -maxdepth 1 -name "$octet19*" -type f
-print 2>/dev
/null | LC_ALL
=C
wc -l) ))
163 [ ${_objs:-0} -ge 5 ]
166 # same as lotsa_loose_objects but first runs `git prune-packed` if it can get a gc lock
167 lotsa_loose_pruned_objects
() {
168 lotsa_loose_objects ||
return $?
169 v_lock_gc _gclock ||
return 0
170 git prune-packed
--quiet
175 # a "single object" pack is either a pack containing just one object
176 # or a pack containing zero objects (which is a degenerate case that
177 # normally can never happen). And it must have a name matching
178 # the pack-<sha1>*.pack pattern where either the "infix" suffix is "_l"
179 # or the name does not contain any "_" characters at all (and, obviously,
180 # must have a matching .idx file). Any .keep, .bundle, or .bitmap
181 # associated packs are automatically excluded from the count.
182 # "lotsa" here is defined as 20 or more.
183 lotsa_single_object_packs
() {
184 __lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
185 _lpo01
="$__lpo --exclude-limit 2"
186 # "$0=substr($0,19,length-23)" strips the leading "objects/pack/pack-" and trailing ".pack"
188 list_packs --quiet $_lpo01 objects/pack 2>/dev/null |
189 LC_ALL=C awk 'BEGIN {c=0} {$0=substr($0,19,length-23)} !/_/ || /^[0-9a-f]*_l$/ {c+=1} END {print c}'
191 [ ${_sopacks:-0} -ge 20 ]
194 # returns true if either lotsa_loose_pruned_objects or lotsa_single_object_packs is true
195 lotsa_loose_objects_or_sopacks
() {
196 lotsa_single_object_packs || lotsa_loose_pruned_objects
199 # pack any existing, non-packed loose objects into a new _l.pack file then run prune-packed
200 # note that prune-packed is NOT run beforehand -- the caller must do that if needed
201 # loose objects need not be part of complete commits/trees as --weak-naming is used
202 # if there end up being too many loose packs, attempt to combine the packs too
203 pack_incremental_loose_objects
() {
204 _lpacks
="$(run_combine_packs </dev/null --names --loose --weak-naming --incremental --non-empty --all-progress-implied ${quiet:---progress} $packopts)"
205 if [ -n "$_lpacks" ]; then
206 # We need to identify these packs later so we don't combine_packs them
207 for _objpack
in $_lpacks; do
208 rename_pack
"objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" ||
:
210 git prune-packed
$quiet
213 __lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
214 _lpo01
="$__lpo --exclude-limit 2"
215 _lpol
="$__lpo --exclude-no-sfx _l"
216 list_packs
--quiet $_lpo01 objects
/pack
2>/dev
/null |
217 while read -r _apack
&& _apack
="${_apack%.pack}" && [ -n "$_apack" ]; do
218 case "$_apack" in *_
*);;*)
219 rename_pack
"$_apack" "${_apack}_l" ||
:
222 { _packs
="$(list_packs --quiet --count $_lpol objects/pack || :)" ||
:; } 2>/dev
/null
223 [ "${_packs:-0}" -lt 20 ] ||
{
224 combine_small_incremental_loose_packs
226 { _packs
="$(list_packs --quiet --count $_lpol objects/pack || :)" ||
:; } 2>/dev
/null
227 [ "${_packs:-0}" -lt 20 ] || combine_large_incremental_loose_packs
231 # same as pack_incremental_loose_objects except
232 # returns true if locked and packed and unlocked or
233 # false if could not lock (with err in $lockerr)
234 pack_incremental_loose_objects_if_lockable
() {
235 if v_lock_gc _gclock
; then
236 pack_incremental_loose_objects ||
:
245 # combine small _l packs into larger pack(s) using --weak-naming
246 # we avoid any non _l, keep, bndl or bitmap packs
247 combine_small_incremental_loose_packs
() {
248 _lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
249 _lpo
="$_lpo --exclude-no-sfx _l"
250 _lpo
="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
252 _cnt
="$(list_packs --count $_lpo)" ||
:
253 test "${_cnt:-0}" -ge 2
255 _newp
="$(list_packs $_lpo | combine_packs_std --names --weak-naming --no-reuse-delta)"
256 # We need to identify these packs later so we don't combine_packs them
257 for _objpack
in $_newp; do
258 rename_pack
"objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" ||
:
261 # be paranoid and exit the loop if we haven't reduced the number of packs
262 [ $_newc -lt $_cnt ] ||
break
267 # combine large[ish] _l packs into larger pack(s) using --weak-naming
268 # we avoid any non _l, keep, bndl or bitmap packs
269 combine_large_incremental_loose_packs
() {
270 _lpo
="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
271 _lpo
="$_lpo --exclude-no-sfx _l"
272 _lpo
="$_lpo --quiet --exclude-limit -$(( ( $var_redelta_threshold / 2 ) + 1 )) objects/pack"
274 _cnt
="$(list_packs --count $_lpo)" ||
:
275 test "${_cnt:-0}" -ge 2
277 _newp
="$(list_packs $_lpo | combine_packs_std --names --weak-naming)"
278 # We need to identify these packs later so we don't combine_packs them
279 for _objpack
in $_newp; do
280 rename_pack
"objects/pack/pack-$_objpack" "objects/pack/pack-${_objpack}_l" ||
:
283 # be paranoid and exit the loop if we haven't reduced the number of packs
284 [ $_newc -lt $_cnt ] ||
break