jobd/gc.sh

   1 #!/bin/sh
   2
   3 # NOTE: additional options can be passed to git repack by specifying
   4 #       them after the project name, for example:
   5 #         gc.sh my-project -f
   6
   7 . @basedir@/shlib.sh
   8
   9 set -e
  10
  11 # packing options
  12 packopts='--window=50 --window-memory=1g --depth=50'
  13
  14 umask 002
  15 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  16
  17 pidactive() {
  18         if _result="$(kill -0 "$1" 2>&1)"; then
  19                 # process exists and we have permission to signal it
  20                 return 0
  21         fi
  22         case "$_result" in *"not permitted"*)
  23                 # we do not have permission to signal the process
  24                 return 0
  25         esac
  26         # process does not exist
  27         return 1
  28 }
  29
  30 createlock() {
  31         # A .lock file should only exist for much less than a second.
  32         # If we see a stale lock file (> 1h old), remove it and then,
  33         # just in case, wait 30 seconds for any process whose .lock
  34         # we might have just removed (it's racy) to finish doing what
  35         # should take much less than a second to do.
  36         _stalelock="$(find "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
  37         if [ -n "$_stalelock" ]; then
  38                 rm -f "$_stalelock"
  39                 sleep 30
  40         fi
  41         for _try in p p n; do
  42                 if (set -C; > "$1.lock") 2>/dev/null; then
  43                         echo "$1.lock"
  44                         return 0
  45                 fi
  46                 # delay and try again
  47                 [ "$_try" != "p" ] || sleep 1
  48         done
  49         # cannot create lock file
  50         return 1
  51 }
  52
  53 # return true if there's more than one objects/pack-<sha>.pack file or
  54 # ANY sha-1 files in objects
  55 is_dirty() {
  56         _packs=$(find objects/pack -type f -name "pack-$octet20.pack" -print | wc -l)
  57         if [ $_packs != 1 ] && [ $_packs != 0 ]; then
  58                 return 0
  59         fi
  60         _objs=$(find objects/$octet -type f -name "$octet19" -print 2>/dev/null | wc -l)
  61         [ $_objs -ne 0 ]
  62 }
  63
  64 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
  65 repack_gfi_packs() {
  66         is_gfi_mirror || return 0
  67         [ -s gfi-packs ] || return 0
  68         while IFS=': ' read -r _pack _junk; do
  69                 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
  70                         git show-index < "${_pack%.pack}.idx" | cut -d ' ' -f 2
  71                 fi
  72         done < gfi-packs | \
  73         git pack-objects $packopts --no-reuse-delta --delta-base-offset \
  74                 --non-empty --threads=1 $quiet objects/pack/packtmp | \
  75         while read -r _newpack; do
  76                 rm -f objects/pack/pack-$_newpack.*
  77                 ln -f objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
  78                 ln -f objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
  79                 rm -f objects/pack/packtmp-$_newpack.*
  80         done
  81         rm -f gfi-packs
  82 }
  83
  84 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
  85 pack_is_complete() {
  86         # Must have a matching .idx file and a non-empty packed-refs file
  87         [ -s "${1%.pack}.idx" ] || return 1
  88         [ -s "$2" ] || return 1
  89         _headsha=
  90         case "$3" in
  91                 $octet20)
  92                         _headsha="$3"
  93                         ;;
  94                 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
  95                         _headmatch="${3#ref:}"
  96                         _headmatch="${_headmatch# }"
  97                         _headmatchpat="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
  98                         _headsha="$(grep -e "^$octet20 $_headmatchpat\$" < "$2" | \
  99                                     cut -d ' ' -f 1)"
 100                         case "$_headsha" in $octet20) :;; *)
 101                                 return 1
 102                         esac
 103                         ;;
 104                 *)
 105                         # bad HEAD
 106                         return 1
 107         esac
 108         rm -rf pack_is_complete_test
 109         mkdir pack_is_complete_test
 110         mkdir pack_is_complete_test/refs
 111         mkdir pack_is_complete_test/objects
 112         mkdir pack_is_complete_test/objects/pack
 113         echo "$_headsha" > pack_is_complete_test/HEAD
 114         ln -s "$1" pack_is_complete_test/objects/pack/
 115         ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
 116         ln -s "$2" pack_is_complete_test/packed-refs
 117         _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
 118         rm -rf pack_is_complete_test
 119         [ -n "$_count" ] || return 1
 120         [ "$_count" -gt 0 ] 2>/dev/null || return 1
 121         echo "$_headsha"
 122 }
 123
 124 proj="${1%.git}"
 125 shift
 126 cd "$cfg_reporoot/$proj.git"
 127
 128 trap 'if [ $? != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
 129 trap 'exit 130' INT
 130 trap 'exit 143' TERM
 131
 132 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
 133 datefmt='+%a, %d %b %Y %T %z'
 134
 135 if check_interval lastgc $cfg_min_gc_interval; then
 136         progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
 137         exit 0
 138 fi
 139 if [ -e .nogc ]; then
 140         progress "x [$proj] garbage check disabled"
 141         exit 0
 142 fi
 143
 144 # Avoid unnecessary garbage collections:
 145 #   1. If lastreceive is set and is older than lastgc
 146 #   -AND-
 147 #   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
 148
 149 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
 150 # If we are a fork and lastparentgc is NOT set we MUST run gc
 151
 152 # If the repo is dirty after removing any crud we MUST run gc
 153
 154 gcstart="$(date "$datefmt")"
 155 skipgc=
 156 isfork=
 157 [ -s objects/info/alternates ] && isfork=1
 158 lastparentgcsecs=
 159 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
 160 lastreceivesecs=
 161 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
 162    lastgcsecs="$(config_get_date_seconds lastgc)" && \
 163    [ $lastreceivesecs -lt $lastgcsecs ]; then
 164         # We've run gc since we last received, so maybe we can skip,
 165         # check if not fork or fork and lastparentgc < lastgc
 166         if [ -n "$isfork" ]; then
 167                 if [ -n "$lastparentgcsecs" ] && \
 168                    [ $lastparentgcsecs -lt $lastgcsecs ]; then
 169                         # We've run gc since our parent ran gc so we can skip
 170                         skipgc=1
 171                 fi
 172         else
 173                 # We don't have any alternates (we're not a forK) so we can skip
 174                 skipgc=1
 175         fi
 176 fi
 177
 178 # be compatibile with gc.pid file from newer Git releases
 179
 180 hn="$(hostname)"
 181 lockf=gc.pid
 182 active=
 183 if [ "$(createlock "$lockf")" ]; then
 184         # If $lockf is:
 185         #   1) less than 12 hours old
 186         #   2) contains two fields (pid hostname) NO trailing NL
 187         #   3) the hostname is different OR the pid is still alive
 188         # then we exit as another active process is holding the lock
 189         if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
 190                 apid=
 191                 ahost=
 192                 read -r apid ahost ajunk < "$lockf" || :
 193                 if [ "$apid" ] && [ "$ahost" ]; then
 194                         if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
 195                                 active=1
 196                         fi
 197                 fi
 198         fi
 199 else
 200         echo >&2 "[$proj] unable to create gc.pid.lock file"
 201         exit 1
 202 fi
 203 if [ -n "$active" ]; then
 204         rm -f "$lockf.lock"
 205         echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
 206         exit 1
 207 fi
 208 printf "%s %s" "$$" "$hn" > "$lockf.lock"
 209 chmod 0664 "$lockf.lock"
 210 mv -f "$lockf.lock" "$lockf"
 211
 212 # Remove any stale pack remnants that are more than an hour old.
 213 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
 214 # .pack AND the corresponding .pack DOES NOT exist.  A bunch of stale
 215 # pack-<sha1>.idx files without their corresponding .pack files are worthless
 216 # and just waste space.  Normally there shouldn't be any remnants but actually
 217 # this can happen when things are interrupted at just the wrong time.
 218 # Note that the objects/pack directory is created by git init and should
 219 # always exist.
 220 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" -print | \
 221 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u | \
 222 while read packsha; do
 223         [ ! -e "objects/pack/pack-$packsha.pack" ] || continue
 224         rm -f "objects/pack/pack-$packsha".?*
 225 done
 226
 227 # Remove any stale pack .keep files that are more than 12 hours old.
 228 # We don't do anything to create any permanent pack .keep files, so they must
 229 # be remnants from some failed push or something.  Removing the .keep will
 230 # allow the pack to be properly repacked.
 231 find objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20.keep" -print | \
 232 while read packkeep; do
 233         rm -f "$packkeep"
 234 done
 235
 236 # Remove any stale tmp_pack_* or tmp_idx_* or tmp_bitmap_* or packtmp-* files
 237 # that are more than 12 hours old.
 238 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_pack_?*" -print | \
 239 while read packtmp; do
 240         rm -f "$packtmp"
 241 done
 242 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_idx_?*" -print | \
 243 while read packtmp; do
 244         rm -f "$packtmp"
 245 done
 246 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_bitmap_?*" -print | \
 247 while read packtmp; do
 248         rm -f "$packtmp"
 249 done
 250 find objects/pack -maxdepth 1 -type f -mmin +720 -name "packtmp-?*" -print | \
 251 while read packtmp; do
 252         rm -f "$packtmp"
 253 done
 254
 255 # Remove any stale shallow_* files that are more than 12 hours old.
 256 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
 257 # requests a shallow clone.
 258 find . -maxdepth 1 -type f -mmin +720 -name "shallow_?*" -print | \
 259 while read turd; do
 260         rm -f "$turd"
 261 done
 262
 263 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
 264 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
 265 find objects -type f -mmin +720 -name "*.temp" -print | \
 266 while read objtmp; do
 267         rm -f "$objtmp"
 268 done
 269
 270 # Remove any stale git-svn temp files that are more than 12 hours old.
 271 # The git-svn process creates temp files with random 10 character names
 272 # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 273 # prefix, so we just have to kill any files with a 10-character name.  We
 274 # do this only for git-svn mirrors.  All characters are chosen from
 275 # [A-Za-z0-9_] so we can at least check that and fortunately the only
 276 # collision is 'FETCH_HEAD' but that shouldn't matter.
 277 # There may also be temp files with a Git_ prefix as well.
 278 if is_svn_mirror; then
 279         _randchar='[A-Za-z0-9_]'
 280         _randchar2="$_randchar$_randchar"
 281         _randchar4="$_randchar2$_randchar2"
 282         _randchar10="$_randchar4$_randchar4$_randchar2"
 283         find . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -print | \
 284         while read tmpcrud; do
 285                 rm -f "$tmpcrud"
 286         done
 287         find . -maxdepth 1 -type f -mmin +720 -name "Git_*" -print | \
 288         while read tmpcrud; do
 289                 rm -f "$tmpcrud"
 290         done
 291 fi
 292
 293 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
 294 if is_gfi_mirror; then
 295         find . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -print | \
 296         while read fastcrash; do
 297                 rm -f "$fastcrash"
 298         done
 299 fi
 300
 301 # Do not skip gc if the repo is dirty
 302 if [ -n "$skipgc" ] && ! is_dirty; then
 303         progress "= [$proj] garbage check nothing but crud removal to do (`date`)"
 304         config_set lastgc "$gcstart"
 305         rm -f "$lockf"
 306         exit 0
 307 fi
 308
 309 bumptime=
 310 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
 311         # set lastparentgc and then update gcstart to be at least 1 second later
 312         config_set lastparentgc "$gcstart"
 313         bumptime=1
 314 fi
 315 if [ -z "$lastreceivesecs" ]; then
 316         # set lastreceive and then update gcstart to be at least 1 second later
 317         config_set lastreceive "$gcstart"
 318         bumptime=1
 319 fi
 320 if [ -n "$bumptime" ]; then
 321         sleep 1
 322         gcstart="$(date "$datefmt")"
 323 fi
 324
 325 progress "+ [$proj] garbage check (`date`)"
 326
 327 # safe pruning: we put all our objects in all forks, then we can
 328 # safely get rid of extra ones; repacks in forks will get rid of
 329 # the redundant ones again then
 330 forkdir="$proj"
 331 if [ -d "../${forkdir##*/}" ]; then
 332         # It is enough to copy objects just one level down and get_repo_list
 333         # takes a regular expression (which is automatically prefixed with '^')
 334         # so we can easily match forks exactly one level down from this project
 335         get_repo_list "$forkdir/[^/]*:" |
 336                 while read fork; do
 337                         # Ignore forks that do not exist or are symbolic links
 338                         [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
 339                                 continue
 340                         # Or do not have a non-zero length alternates file
 341                         [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
 342                                 continue
 343                         # Match objects in parent project
 344                         for d in objects/?? objects/pack; do
 345                                 [ "$d" != "objects/??" ] || continue
 346                                 mkdir -p "$cfg_reporoot/$fork.git/$d"
 347                                 [ "$d" != "objects/pack" ] || \
 348                                         [ "$(echo objects/pack/*)" != \
 349                                                         "objects/pack/*" ] || \
 350                                         continue
 351                                 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
 352                         done
 353                         # Update the fork's lastparentgc date (must be current, not $gcstart)
 354                         GIT_DIR="$cfg_reporoot/$fork.git" git config \
 355                                 gitweb.lastparentgc "$(date "$datefmt")"
 356                 done
 357 fi
 358
 359 quiet=; [ -n "$show_progress" ] || quiet=-q
 360
 361 git pack-refs --all
 362 repack_gfi_packs
 363 rm -f bundles/*
 364 rm -f objects/pack/pack-*.bndl
 365 git repack $packopts -a -d -l $quiet "$@"
 366 allpacks="$(echo objects/pack/pack-$octet20.pack)"
 367 curhead="$(cat HEAD)"
 368 pkrf=
 369 [ ! -e packed-refs ] || pkrf=packed-refs
 370 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | awk '{print $1}') | \
 371         sed -e 's/ / + /g') ))"
 372 git prune
 373 git update-server-info
 374
 375 # darcs:// mirrors have a xxx.log file that will grow endlessly
 376 # if this is a mirror and the file exists, shorten it to 10000 lines
 377 # also take this opportunity to optimize the darcs repo
 378 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
 379         url="$(config_get baseurl || :)"
 380         case "$url" in darcs://*)
 381                 if [ -n "$cfg_mirror_darcs" ]; then
 382                         url="${url%/}"
 383                         basedarcs="$(basename "${url#darcs:/}")"
 384                         if [ -f "$basedarcs.log" ]; then
 385                                 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
 386                                 mv -f "$basedarcs.log.$$" "$basedarcs.log"
 387                         fi
 388                         if [ -d "$basedarcs.darcs" ]; then
 389                                 (
 390                                         cd "$basedarcs.darcs"
 391                                         # Note that this does not optimize _darcs/inventories/ :(
 392                                         darcs optimize
 393                                 )
 394                         fi
 395                 fi
 396         esac
 397 fi
 398
 399 # Create a matching .bndl header file for the all-in-one pack we just created
 400 # but only if we're not a fork (otherwise the bundle would not be complete)
 401 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
 402 if [ ! -s objects/info/alternates ] && [ -n "$var_have_git_172" ]; then
 403         # There should only be one pack in $allpacks but if there was a
 404         # simultaneous push...
 405         # The one we just created will have a .idx and will NOT have a .keep
 406         pkfound=
 407         pkhead=
 408         for pk in $allpacks; do
 409                 [ -s "$pk" ] || continue
 410                 pkbase="${pk%.pack}"
 411                 [ -s "$pkbase.idx" ] || continue
 412                 [ ! -e "$pkbase.keep" ] || continue
 413                 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
 414                         pkfound="$pkbase"
 415                         break;
 416                 fi
 417         done
 418         if [ -n "$pkfound" -a -n "$pkhead" ]; then
 419                 {
 420                         echo "# v2 git bundle"
 421                         sed -ne "/^$octet20 refs\/[^    ]*\$/ p" < packed-refs
 422                         echo "$pkhead HEAD"
 423                         echo ""
 424                 } > "$pkbase.bndl"
 425                 bndlsha="${pkbase#objects/pack/pack-}"
 426                 bndlshatrailer="${bndlsha#????????}"
 427                 bndlshaprefix="${bndlsha%$bndlshatrailer}"
 428                 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
 429                 [ -d bundles ] || mkdir bundles
 430                 echo "$bndlsha" > "bundles/$bndlname"
 431         fi
 432 fi
 433
 434 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
 435 config_set_raw girocco.reposizek "${reposizek:-0}"
 436
 437 # We use $gcstart here to avoid a race where a push occurs during the gc itself
 438 # and the next future gc could be incorrectly skipped if we used the current
 439 # timestamp here instead
 440 config_set lastgc "$gcstart"
 441 rm -f "$lockf"
 442
 443 progress "- [$proj] garbage check (`date`)"