jobd/gc.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4
   5 set -e
   6 trap 'if [ $? != 0 ]; then echo "gc failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
   7
   8 # packing options
   9 packopts='--window=50 --window-memory=1g --depth=50'
  10
  11 umask 002
  12 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  13
  14 pidactive() {
  15         if _result="$(kill -0 "$1" 2>&1)"; then
  16                 # process exists and we have permission to signal it
  17                 return 0
  18         fi
  19         case "$_result" in *"not permitted"*)
  20                 # we do not have permission to signal the process
  21                 return 0
  22         esac
  23         # process does not exist
  24         return 1
  25 }
  26
  27 createlock() {
  28         # A .lock file should only exist for much less than a second.
  29         # If we see a stale lock file (> 1h old), remove it and then,
  30         # just in case, wait 30 seconds for any process whose .lock
  31         # we might have just removed (it's racy) to finish doing what
  32         # should take much less than a second to do.
  33         _stalelock="$(find "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
  34         if [ -n "$_stalelock" ]; then
  35                 rm -f "$_stalelock"
  36                 sleep 30
  37         fi
  38         for _try in p p n; do
  39                 if (set -C; > "$1.lock") 2>/dev/null; then
  40                         echo "$1.lock"
  41                         return 0
  42                 fi
  43                 # delay and try again
  44                 [ "$_try" != "p" ] || sleep 1
  45         done
  46         # cannot create lock file
  47         return 1
  48 }
  49
  50 # return true if there's more than one objects/pack-<sha>.pack file or
  51 # ANY sha-1 files in objects
  52 is_dirty() {
  53         _packs=$(find objects/pack -type f -name "pack-$octet20.pack" -print | wc -l)
  54         if [ "$_packs" != 1 ] && [ "$_packs" != 0 ]; then
  55                 return 0
  56         fi
  57         _objs=$(find objects/$octet -type f -name "$octet19" -print 2>/dev/null | wc -l)
  58         [ "$_objs" != 0 ]
  59 }
  60
  61 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
  62 repack_gfi_packs() {
  63         is_gfi_mirror || return 0
  64         [ -s gfi-packs ] || return 0
  65         while IFS=': ' read -r _pack _junk; do
  66                 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
  67                         git show-index < "${_pack%.pack}.idx" | cut -d ' ' -f 2
  68                 fi
  69         done < gfi-packs | \
  70         git pack-objects $packopts --no-reuse-delta --delta-base-offset \
  71                 --non-empty --threads=1 $quiet objects/pack/packtmp | \
  72         while read -r _newpack; do
  73                 rm -f objects/pack/pack-$_newpack.*
  74                 ln -f objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
  75                 ln -f objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
  76                 rm -f objects/pack/packtmp-$_newpack.*
  77         done
  78         rm -f gfi-packs
  79 }
  80
  81 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
  82 pack_is_complete() {
  83         # Must have a matching .idx file and a non-empty packed-refs file
  84         [ -s "${1%.pack}.idx" ] || return 1
  85         [ -s "$2" ] || return 1
  86         _headsha=
  87         case "$3" in
  88                 $octet20)
  89                         _headsha="$3"
  90                         ;;
  91                 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
  92                         _headmatch="${3#ref:}"
  93                         _headmatch="${_headmatch# }"
  94                         _headmatchpat="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
  95                         _headsha="$(grep -e "^$octet20 $_headmatchpat\$" < "$2" | \
  96                                     cut -d ' ' -f 1)"
  97                         case "$_headsha" in $octet20) :;; *)
  98                                 return 1
  99                         esac
 100                         ;;
 101                 *)
 102                         # bad HEAD
 103                         return 1
 104         esac
 105         rm -rf pack_is_complete_test
 106         mkdir pack_is_complete_test
 107         mkdir pack_is_complete_test/refs
 108         mkdir pack_is_complete_test/objects
 109         mkdir pack_is_complete_test/objects/pack
 110         echo "$_headsha" > pack_is_complete_test/HEAD
 111         ln -s "$1" pack_is_complete_test/objects/pack/
 112         ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
 113         ln -s "$2" pack_is_complete_test/packed-refs
 114         _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
 115         rm -rf pack_is_complete_test
 116         [ -n "$_count" ] || return 1
 117         [ "$_count" -gt 0 ] 2>/dev/null || return 1
 118         echo "$_headsha"
 119 }
 120
 121 proj="$1"
 122 cd "$cfg_reporoot/$proj.git"
 123
 124 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
 125 datefmt='+%a, %d %b %Y %T %z'
 126
 127 if check_interval lastgc $cfg_min_gc_interval; then
 128         progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
 129         exit 0
 130 fi
 131 if [ -e .nogc ]; then
 132         progress "x [$proj] garbage check disabled"
 133         exit 0
 134 fi
 135
 136 # Avoid unnecessary garbage collections:
 137 #   1. If lastreceive is set and is older than lastgc
 138 #   -AND-
 139 #   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
 140
 141 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
 142 # If we are a fork and lastparentgc is NOT set we MUST run gc
 143
 144 # If the repo is dirty after removing any crud we MUST run gc
 145
 146 gcstart="$(date "$datefmt")"
 147 skipgc=
 148 isfork=
 149 [ -s objects/info/alternates ] && isfork=1
 150 lastparentgcsecs=
 151 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
 152 lastreceivesecs=
 153 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
 154    lastgcsecs="$(config_get_date_seconds lastgc)" && \
 155    [ $lastreceivesecs -lt $lastgcsecs ]; then
 156         # We've run gc since we last received, so maybe we can skip,
 157         # check if not fork or fork and lastparentgc < lastgc
 158         if [ -n "$isfork" ]; then
 159                 if [ -n "$lastparentgcsecs" ] && \
 160                    [ $lastparentgcsecs -lt $lastgcsecs ]; then
 161                         # We've run gc since our parent ran gc so we can skip
 162                         skipgc=1
 163                 fi
 164         else
 165                 # We don't have any alternates (we're not a forK) so we can skip
 166                 skipgc=1
 167         fi
 168 fi
 169
 170 # be compatibile with gc.pid file from newer Git releases
 171
 172 hn="$(hostname)"
 173 lockf=gc.pid
 174 active=
 175 if [ "$(createlock "$lockf")" ]; then
 176         # If $lockf is:
 177         #   1) less than 12 hours old
 178         #   2) contains two fields (pid hostname) NO trailing NL
 179         #   3) the hostname is different OR the pid is still alive
 180         # then we exit as another active process is holding the lock
 181         if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
 182                 apid=
 183                 ahost=
 184                 read -r apid ahost ajunk < "$lockf" || :
 185                 if [ "$apid" ] && [ "$ahost" ]; then
 186                         if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
 187                                 active=1
 188                         fi
 189                 fi
 190         fi
 191 else
 192         echo >&2 "[$proj] unable to create gc.pid.lock file"
 193         exit 1
 194 fi
 195 if [ -n "$active" ]; then
 196         rm -f "$lockf.lock"
 197         echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
 198         exit 1
 199 fi
 200 printf "%s %s" "$$" "$hn" > "$lockf.lock"
 201 chmod 0664 "$lockf.lock"
 202 mv -f "$lockf.lock" "$lockf"
 203
 204 # Remove any stale pack remnants that are more than an hour old.
 205 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
 206 # .pack AND the corresponding .pack DOES NOT exist.  A bunch of stale
 207 # pack-<sha1>.idx files without their corresponding .pack files are worthless
 208 # and just waste space.  Normally there shouldn't be any remnants but actually
 209 # this can happen when things are interrupted at just the wrong time.
 210 # Note that the objects/pack directory is created by git init and should
 211 # always exist.
 212 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" -print | \
 213 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u | \
 214 while read packsha; do
 215         [ ! -e "objects/pack/pack-$packsha.pack" ] || continue
 216         rm -f "objects/pack/pack-$packsha".?*
 217 done
 218
 219 # Remove any stale tmp_pack_* files that are more than 12 hours old.
 220 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_pack_?*" -print | \
 221 while read packtmp; do
 222         rm -f "$packtmp"
 223 done
 224 find objects/pack -maxdepth 1 -type f -mmin +720 -name "packtmp-?*" -print | \
 225 while read packtmp; do
 226         rm -f "$packtmp"
 227 done
 228
 229 # Remove any stale git-svn temp files that are more than 12 hours old.
 230 # The git-svn process creates temp files with random 10 character names
 231 # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 232 # prefix, so we just have to kill any files with a 10-character name.  We
 233 # do this only for git-svn mirrors.  All characters are chosen from
 234 # [A-Za-z0-9_] so we can at least check that and fortunately the only
 235 # collision is 'FETCH_HEAD' but that shouldn't matter.
 236 # There may also be temp files with a Git_ prefix as well.
 237 if is_svn_mirror; then
 238         _randchar='[A-Za-z0-9_]'
 239         _randchar2="$_randchar$_randchar"
 240         _randchar4="$_randchar2$_randchar2"
 241         _randchar10="$_randchar4$_randchar4$_randchar2"
 242         find . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -print | \
 243         while read tmpcrud; do
 244                 rm -f "$tmpcrud"
 245         done
 246         find . -maxdepth 1 -type f -mmin +720 -name "Git_*" -print | \
 247         while read tmpcrud; do
 248                 rm -f "$tmpcrud"
 249         done
 250 fi
 251
 252 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
 253 if is_gfi_mirror; then
 254         find . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -print | \
 255         while read fastcrash; do
 256                 rm -f "$fastcrash"
 257         done
 258 fi
 259
 260 # Do not skip gc if the repo is dirty
 261 if [ -n "$skipgc" ] && ! is_dirty; then
 262         progress "= [$proj] garbage check nothing but crud removal to do (`date`)"
 263         config_set lastgc "$gcstart"
 264         rm -f "$lockf"
 265         exit 0
 266 fi
 267
 268 bumptime=
 269 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
 270         # set lastparentgc and then update gcstart to be at least 1 second later
 271         config_set lastparentgc "$gcstart"
 272         bumptime=1
 273 fi
 274 if [ -z "$lastreceivesecs" ]; then
 275         # set lastreceive and then update gcstart to be at least 1 second later
 276         config_set lastreceive "$gcstart"
 277         bumptime=1
 278 fi
 279 if [ -n "$bumptime" ]; then
 280         sleep 1
 281         gcstart="$(date "$datefmt")"
 282 fi
 283
 284 progress "+ [$proj] garbage check (`date`)"
 285
 286 # safe pruning: we put all our objects in all forks, then we can
 287 # safely get rid of extra ones; repacks in forks will get rid of
 288 # the redundant ones again then
 289 forkdir="$1"
 290 if [ -d "../${forkdir##*/}" ]; then
 291         # It is enough to copy objects just one level down and get_repo_list
 292         # takes a regular expression (which is automatically prefixed with '^')
 293         # so we can easily match forks exactly one level down from this project
 294         get_repo_list "$forkdir/[^/]*:" |
 295                 while read fork; do
 296                         # Ignore forks that do not exist or are symbolic links
 297                         [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
 298                                 continue
 299                         # Or do not have a non-zero length alternates file
 300                         [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
 301                                 continue
 302                         # Match objects in parent project
 303                         for d in objects/?? objects/pack; do
 304                                 [ "$d" != "objects/??" ] || continue
 305                                 mkdir -p "$cfg_reporoot/$fork.git/$d"
 306                                 [ "$d" != "objects/pack" ] || \
 307                                         [ "$(echo objects/pack/*)" != \
 308                                                         "objects/pack/*" ] || \
 309                                         continue
 310                                 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
 311                         done
 312                         # Update the fork's lastparentgc date (must be current, not $gcstart)
 313                         GIT_DIR="$cfg_reporoot/$fork.git" git config \
 314                                 gitweb.lastparentgc "$(date "$datefmt")"
 315                 done
 316 fi
 317
 318 quiet=; [ -n "$show_progress" ] || quiet=-q
 319
 320 git pack-refs --all
 321 repack_gfi_packs
 322 rm -f bundles/*
 323 rm -f objects/pack/pack-*.bndl
 324 git repack $packopts -a -d -l $quiet
 325 allpacks="$(echo objects/pack/pack-$octet20.pack)"
 326 curhead="$(cat HEAD)"
 327 eval "reposizek=$(( $(echo 0 $(du -k $allpacks 2>/dev/null | awk '{print $1}') | \
 328         sed -e 's/ / + /g') ))"
 329 git prune
 330 git update-server-info
 331
 332 # darcs:// mirrors have a xxx.log file that will grow endlessly
 333 # if this is a mirror and the file exists, shorten it to 10000 lines
 334 # also take this opportunity to optimize the darcs repo
 335 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
 336         url="$(config_get baseurl || :)"
 337         case "$url" in darcs://*)
 338                 if [ -n "$cfg_mirror_darcs" ]; then
 339                         url="${url%/}"
 340                         basedarcs="$(basename "${url#darcs:/}")"
 341                         if [ -f "$basedarcs.log" ]; then
 342                                 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
 343                                 mv -f "$basedarcs.log.$$" "$basedarcs.log"
 344                         fi
 345                         if [ -d "$basedarcs.darcs" ]; then
 346                                 (
 347                                         cd "$basedarcs.darcs"
 348                                         # Note that this does not optimize _darcs/inventories/ :(
 349                                         darcs optimize
 350                                 )
 351                         fi
 352                 fi
 353         esac
 354 fi
 355
 356 # Create a matching .bndl header file for the all-in-one pack we just created
 357 # but only if we're not a fork (otherwise the bundle would not be complete)
 358 if [ ! -s objects/info/alternates ]; then
 359         # There should only be one pack in $allpacks but if there was a
 360         # simultaneous push...
 361         # The one we just created will have a .idx and will NOT have a .keep
 362         pkfound=
 363         pkhead=
 364         for pk in $allpacks; do
 365                 [ -s "$pk" ] || continue
 366                 pkbase="${pk%.pack}"
 367                 [ -s "$pkbase.idx" ] || continue
 368                 [ ! -e "$pkbase.keep" ] || continue
 369                 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
 370                         pkfound="$pkbase"
 371                         break;
 372                 fi
 373         done
 374         if [ -n "$pkfound" -a -n "$pkhead" ]; then
 375                 {
 376                         echo "# v2 git bundle"
 377                         sed -ne "/^$octet20 refs\/[^    ]*\$/ p" < packed-refs
 378                         echo "$pkhead HEAD"
 379                         echo ""
 380                 } > "$pkbase.bndl"
 381                 bndlsha="${pkbase#objects/pack/pack-}"
 382                 bndlshatrailer="${bndlsha#????????}"
 383                 bndlshaprefix="${bndlsha%$bndlshatrailer}"
 384                 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
 385                 [ -d bundles ] || mkdir bundles
 386                 echo "$bndlsha" > "bundles/$bndlname"
 387         fi
 388 fi
 389
 390 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
 391 config_set_raw girocco.reposizek "${reposizek:-0}"
 392
 393 # We use $gcstart here to avoid a race where a push occurs during the gc itself
 394 # and the next future gc could be incorrectly skipped if we used the current
 395 # timestamp here instead
 396 config_set lastgc "$gcstart"
 397 rm -f "$lockf"
 398
 399 progress "- [$proj] garbage check (`date`)"