jobd/gc.sh

   1 #!/bin/sh
   2
   3 # NOTE: additional options can be passed to git repack by specifying
   4 #       them after the project name, for example:
   5 #         gc.sh my-project -f
   6
   7 . @basedir@/shlib.sh
   8
   9 set -e
  10
  11 if [ $# -lt 1 ]; then
  12         echo "Usage: gc.sh projname [extra-repack-args]" >&2
  13         exit 1
  14 fi
  15
  16 # packing options
  17 packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
  18
  19 umask 002
  20 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  21
  22 pidactive() {
  23         if _result="$(kill -0 "$1" 2>&1)"; then
  24                 # process exists and we have permission to signal it
  25                 return 0
  26         fi
  27         case "$_result" in *"not permitted"*)
  28                 # we do not have permission to signal the process
  29                 return 0
  30         esac
  31         # process does not exist
  32         return 1
  33 }
  34
  35 createlock() {
  36         # A .lock file should only exist for much less than a second.
  37         # If we see a stale lock file (> 1h old), remove it and then,
  38         # just in case, wait 30 seconds for any process whose .lock
  39         # we might have just removed (it's racy) to finish doing what
  40         # should take much less than a second to do.
  41         _stalelock="$(find "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null || :)"
  42         if [ -n "$_stalelock" ]; then
  43                 rm -f "$_stalelock"
  44                 sleep 30
  45         fi
  46         for _try in p p n; do
  47                 if (set -C; > "$1.lock") 2>/dev/null; then
  48                         echo "$1.lock"
  49                         return 0
  50                 fi
  51                 # delay and try again
  52                 [ "$_try" != "p" ] || sleep 1
  53         done
  54         # cannot create lock file
  55         return 1
  56 }
  57
  58 # return true if there's more than one objects/pack-<sha>.pack file or
  59 # ANY sha-1 files in objects
  60 is_dirty() {
  61         _packs=$(find objects/pack -type f -name "pack-$octet20.pack" -print | wc -l)
  62         if [ $_packs != 1 ] && [ $_packs != 0 ]; then
  63                 return 0
  64         fi
  65         _objs=$(find objects/$octet -type f -name "$octet19" -print 2>/dev/null | wc -l)
  66         [ $_objs -ne 0 ]
  67 }
  68
  69 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
  70 repack_gfi_packs() {
  71         is_gfi_mirror || return 0
  72         [ -s gfi-packs ] || return 0
  73         while IFS=': ' read -r _pack _junk; do
  74                 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
  75                         git show-index < "${_pack%.pack}.idx" | cut -d ' ' -f 2
  76                 fi
  77         done < gfi-packs | \
  78         git pack-objects $packopts --no-reuse-delta --delta-base-offset \
  79                 --non-empty $quiet objects/pack/packtmp | \
  80         while read -r _newpack; do
  81                 rm -f objects/pack/pack-$_newpack.*
  82                 ln -f objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
  83                 ln -f objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
  84                 rm -f objects/pack/packtmp-$_newpack.*
  85         done
  86         rm -f gfi-packs
  87 }
  88
  89 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
  90 pack_is_complete() {
  91         # Must have a matching .idx file and a non-empty packed-refs file
  92         [ -s "${1%.pack}.idx" ] || return 1
  93         [ -s "$2" ] || return 1
  94         _headsha=
  95         case "$3" in
  96                 $octet20)
  97                         _headsha="$3"
  98                         ;;
  99                 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
 100                         _headmatch="${3#ref:}"
 101                         _headmatch="${_headmatch# }"
 102                         _headmatchpat="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
 103                         _headsha="$(grep -e "^$octet20 $_headmatchpat\$" < "$2" | \
 104                                     cut -d ' ' -f 1)"
 105                         case "$_headsha" in $octet20) :;; *)
 106                                 return 1
 107                         esac
 108                         ;;
 109                 *)
 110                         # bad HEAD
 111                         return 1
 112         esac
 113         rm -rf pack_is_complete_test
 114         mkdir pack_is_complete_test
 115         mkdir pack_is_complete_test/refs
 116         mkdir pack_is_complete_test/objects
 117         mkdir pack_is_complete_test/objects/pack
 118         echo "$_headsha" > pack_is_complete_test/HEAD
 119         ln -s "$1" pack_is_complete_test/objects/pack/
 120         ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
 121         ln -s "$2" pack_is_complete_test/packed-refs
 122         _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
 123         rm -rf pack_is_complete_test
 124         [ -n "$_count" ] || return 1
 125         [ "$_count" -gt 0 ] 2>/dev/null || return 1
 126         echo "$_headsha"
 127 }
 128
 129 proj="${1%.git}"
 130 shift
 131 cd "$cfg_reporoot/$proj.git"
 132
 133 trap 'if [ $? != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
 134 trap 'exit 130' INT
 135 trap 'exit 143' TERM
 136
 137 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
 138 datefmt='+%a, %d %b %Y %T %z'
 139
 140 if check_interval lastgc $cfg_min_gc_interval; then
 141         progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
 142         exit 0
 143 fi
 144 if [ -e .nogc ]; then
 145         progress "x [$proj] garbage check disabled"
 146         exit 0
 147 fi
 148
 149 # Avoid unnecessary garbage collections:
 150 #   1. If lastreceive is set and is older than lastgc
 151 #   -AND-
 152 #   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
 153
 154 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
 155 # If we are a fork and lastparentgc is NOT set we MUST run gc
 156
 157 # If the repo is dirty after removing any crud we MUST run gc
 158
 159 gcstart="$(date "$datefmt")"
 160 skipgc=
 161 isfork=
 162 [ -s objects/info/alternates ] && isfork=1
 163 lastparentgcsecs=
 164 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
 165 lastreceivesecs=
 166 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
 167    lastgcsecs="$(config_get_date_seconds lastgc)" && \
 168    [ $lastreceivesecs -lt $lastgcsecs ]; then
 169         # We've run gc since we last received, so maybe we can skip,
 170         # check if not fork or fork and lastparentgc < lastgc
 171         if [ -n "$isfork" ]; then
 172                 if [ -n "$lastparentgcsecs" ] && \
 173                    [ $lastparentgcsecs -lt $lastgcsecs ]; then
 174                         # We've run gc since our parent ran gc so we can skip
 175                         skipgc=1
 176                 fi
 177         else
 178                 # We don't have any alternates (we're not a forK) so we can skip
 179                 skipgc=1
 180         fi
 181 fi
 182
 183 # be compatibile with gc.pid file from newer Git releases
 184
 185 hn="$(hostname)"
 186 lockf=gc.pid
 187 active=
 188 if [ "$(createlock "$lockf")" ]; then
 189         # If $lockf is:
 190         #   1) less than 12 hours old
 191         #   2) contains two fields (pid hostname) NO trailing NL
 192         #   3) the hostname is different OR the pid is still alive
 193         # then we exit as another active process is holding the lock
 194         if [ "$(find "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
 195                 apid=
 196                 ahost=
 197                 read -r apid ahost ajunk < "$lockf" || :
 198                 if [ "$apid" ] && [ "$ahost" ]; then
 199                         if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
 200                                 active=1
 201                         fi
 202                 fi
 203         fi
 204 else
 205         echo >&2 "[$proj] unable to create gc.pid.lock file"
 206         exit 1
 207 fi
 208 if [ -n "$active" ]; then
 209         rm -f "$lockf.lock"
 210         echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
 211         exit 1
 212 fi
 213 printf "%s %s" "$$" "$hn" > "$lockf.lock"
 214 chmod 0664 "$lockf.lock"
 215 mv -f "$lockf.lock" "$lockf"
 216
 217 # At this point, if .allowgc exists, it's now crud to be removed
 218 rm -f .allowgc
 219
 220 # Remove any existing FETCH_HEAD
 221 # There can only be a FETCH_HEAD if we've been fetching, not if we've been
 222 # receiving pushes (those never create a FETCH_HEAD).
 223 # And if we're fetching because we're a mirror, we know we're not fetching right
 224 # now since jobd.pl never runs a project's fetch simultaneously with its gc.
 225 # Therefore any existing FETCH_HEAD is junk.  And it may be many megabytes if
 226 # there were a lot of refs.
 227 rm -f FETCH_HEAD
 228
 229 # Remove any stale pack remnants that are more than an hour old.
 230 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
 231 # .pack AND the corresponding .pack DOES NOT exist.  A bunch of stale
 232 # pack-<sha1>.idx files without their corresponding .pack files are worthless
 233 # and just waste space.  Normally there shouldn't be any remnants but actually
 234 # this can happen when things are interrupted at just the wrong time.
 235 # Note that the objects/pack directory is created by git init and should
 236 # always exist.
 237 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" -print | \
 238 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u | \
 239 while read packsha; do
 240         [ ! -e "objects/pack/pack-$packsha.pack" ] || continue
 241         rm -f "objects/pack/pack-$packsha".?*
 242 done
 243
 244 # Remove any stale pack .keep files that are more than 12 hours old.
 245 # We don't do anything to create any permanent pack .keep files, so they must
 246 # be remnants from some failed push or something.  Removing the .keep will
 247 # allow the pack to be properly repacked.
 248 find objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20.keep" -print0 | xargs -0 rm -f
 249
 250 # Remove any stale tmp_pack_* or tmp_idx_* or tmp_bitmap_* or packtmp-* files
 251 # that are more than 12 hours old.
 252 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_pack_?*" -print0 | xargs -0 rm -f
 253 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_idx_?*" -print0 | xargs -0 rm -f
 254 find objects/pack -maxdepth 1 -type f -mmin +720 -name "tmp_bitmap_?*" -print0 | xargs -0 rm -f
 255 find objects/pack -maxdepth 1 -type f -mmin +720 -name "packtmp-?*" -print0 | xargs -0 rm -f
 256
 257 # Remove any stale shallow_* files that are more than 12 hours old.
 258 # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
 259 # requests a shallow clone.
 260 find . -maxdepth 1 -type f -mmin +720 -name "shallow_?*" -print0 | xargs -0 rm -f
 261
 262 # Remove any stale *.temp files in the objects area that are more than 12 hours old.
 263 # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
 264 find objects -type f -mmin +720 -name "*.temp" -print0 | xargs -0 rm -f
 265
 266 # Remove any stale *.lock files in the htmlcache area that might have been left
 267 # behind after an abnormal exit during an attempt to update a cached file and
 268 # are more than 1 hour old.
 269 ! [ -d htmlcache ] || find htmlcache -type f -mmin +60 -name "*.lock" -print0 | xargs -0 rm -f
 270
 271 # Remove any stale git-svn temp files that are more than 12 hours old.
 272 # The git-svn process creates temp files with random 10 character names
 273 # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 274 # prefix, so we just have to kill any files with a 10-character name.  We
 275 # do this only for git-svn mirrors.  All characters are chosen from
 276 # [A-Za-z0-9_] so we can at least check that and fortunately the only
 277 # collision is 'FETCH_HEAD' but that shouldn't matter.
 278 # There may also be temp files with a Git_ prefix as well.
 279 # We also take this opportunity to run 'git svn gc'
 280 if is_svn_mirror; then
 281         _randchar='[A-Za-z0-9_]'
 282         _randchar2="$_randchar$_randchar"
 283         _randchar4="$_randchar2$_randchar2"
 284         _randchar10="$_randchar4$_randchar4$_randchar2"
 285         find . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -print0 | xargs -0 rm -f
 286         find . -maxdepth 1 -type f -mmin +720 -name "Git_*" -print0 | xargs -0 rm -f
 287         git svn gc || :
 288 fi
 289
 290 # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
 291 if is_gfi_mirror; then
 292         find . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -print0 | xargs -0 rm -f
 293 fi
 294
 295 # Skip the actual gc if .delaygc is set
 296 if [ -e .delaygc ]; then
 297         progress "x [$proj] garbage check delayed (except for crud removal)"
 298         rm -f "$lockf"
 299         exit 0
 300 fi
 301
 302 # Do not skip gc if the repo is dirty
 303 if [ -n "$skipgc" ] && ! is_dirty; then
 304         progress "= [$proj] garbage check nothing but crud removal to do (`date`)"
 305         config_set lastgc "$gcstart"
 306         rm -f "$lockf"
 307         exit 0
 308 fi
 309
 310 bumptime=
 311 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
 312         # set lastparentgc and then update gcstart to be at least 1 second later
 313         config_set lastparentgc "$gcstart"
 314         bumptime=1
 315 fi
 316 if [ -z "$lastreceivesecs" ]; then
 317         # set lastreceive and then update gcstart to be at least 1 second later
 318         config_set lastreceive "$gcstart"
 319         bumptime=1
 320 fi
 321 if [ -n "$bumptime" ]; then
 322         sleep 1
 323         gcstart="$(date "$datefmt")"
 324 fi
 325
 326 progress "+ [$proj] garbage check (`date`)"
 327
 328 # safe pruning: we put all our objects in all forks, then we can
 329 # safely get rid of extra ones; repacks in forks will get rid of
 330 # the redundant ones again then; we carefully grab only loose
 331 # objects and pack .idx and .pack files
 332 forkdir="$proj"
 333 if [ -d "../${forkdir##*/}" ]; then
 334         # It is enough to copy objects just one level down and get_repo_list
 335         # takes a regular expression (which is automatically prefixed with '^')
 336         # so we can easily match forks exactly one level down from this project
 337         get_repo_list "$forkdir/[^/]*:" |
 338                 while read fork; do
 339                         # Ignore forks that do not exist or are symbolic links
 340                         [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
 341                                 continue
 342                         # Or do not have a non-zero length alternates file
 343                         [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
 344                                 continue
 345                         # Match objects in parent project
 346                         for d in objects/??; do
 347                                 [ "$d" != "objects/??" ] || continue
 348                                 mkdir -p "$cfg_reporoot/$fork.git/$d"
 349                                 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
 350                         done
 351                         # Match packs in parent project
 352                         mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
 353                         if [ "$(echo objects/pack/pack-*.idx)" != \
 354                                     "objects/pack/pack-*.idx" ]; then
 355                                 ln -f objects/pack/pack-*.pack "$cfg_reporoot/$fork.git/objects/pack" || :
 356                                 ln -f objects/pack/pack-*.idx "$cfg_reporoot/$fork.git/objects/pack" || :
 357                         fi
 358                         # Update the fork's lastparentgc date (must be current, not $gcstart)
 359                         GIT_DIR="$cfg_reporoot/$fork.git" git config \
 360                                 gitweb.lastparentgc "$(date "$datefmt")"
 361                 done
 362 fi
 363
 364 quiet=; [ -n "$show_progress" ] || quiet=-q
 365
 366 git pack-refs --all
 367 repack_gfi_packs
 368 rm -f .gcfailed bundles/*
 369 rm -f objects/pack/pack-*.bndl
 370 # The git repack command may issue a 'disabling bitmap' warning for some
 371 # repositories.  This is perfectly normal and should be suppressed unless
 372 # show_progress is set.  Unfortunately that means we have to grep -v the
 373 # output.  And furthermore, since it's a translated message, we have to
 374 # force the language to english to be sure we do it.
 375 repackcmd="git repack $packopts -a -d -l $quiet $@"
 376 [ -n "$show_progress" ] || \
 377 repackcmd="{ LC_ALL=C $repackcmd 2>&1 || touch .gcfailed; } | grep -v 'disabling bitmap' || :"
 378 eval "$repackcmd"
 379 [ ! -e .gcfailed ] || exit 1
 380 allpacks="$(echo objects/pack/pack-$octet20.pack)"
 381 curhead="$(cat HEAD)"
 382 pkrf=
 383 [ ! -e packed-refs ] || pkrf=packed-refs
 384 eval "reposizek=$(( $(echo 0 $(du -k $pkrf $allpacks 2>/dev/null | awk '{print $1}') | \
 385         sed -e 's/ / + /g') ))"
 386 # The git prune command does not take a -q or --quiet but started outputting
 387 # 'Checking connectivity' progress messages in v1.7.9.  However, we can
 388 # suppress those by piping through cat as it only activates the progress
 389 # messages when stderr is a tty.
 390 prunecmd='git prune'
 391 [ -n "$show_progress" ] || \
 392 prunecmd="{ $prunecmd 2>&1 || touch .gcfailed; } | cat"
 393 eval "$prunecmd"
 394 [ ! -e .gcfailed ] || exit 1
 395 git update-server-info
 396
 397 # darcs:// mirrors have a xxx.log file that will grow endlessly
 398 # if this is a mirror and the file exists, shorten it to 10000 lines
 399 # also take this opportunity to optimize the darcs repo
 400 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
 401         url="$(config_get baseurl || :)"
 402         case "$url" in darcs://*)
 403                 if [ -n "$cfg_mirror_darcs" ]; then
 404                         url="${url%/}"
 405                         basedarcs="$(basename "${url#darcs:/}")"
 406                         if [ -f "$basedarcs.log" ]; then
 407                                 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
 408                                 mv -f "$basedarcs.log.$$" "$basedarcs.log"
 409                         fi
 410                         if [ -d "$basedarcs.darcs" ]; then
 411                                 (
 412                                         cd "$basedarcs.darcs"
 413                                         # Note that this does not optimize _darcs/inventories/ :(
 414                                         darcs optimize
 415                                 )
 416                         fi
 417                 fi
 418         esac
 419 fi
 420
 421 # Create a matching .bndl header file for the all-in-one pack we just created
 422 # but only if we're not a fork (otherwise the bundle would not be complete)
 423 # and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
 424 if [ ! -s objects/info/alternates ] && [ -n "$var_have_git_172" ]; then
 425         # There should only be one pack in $allpacks but if there was a
 426         # simultaneous push...
 427         # The one we just created will have a .idx and will NOT have a .keep
 428         pkfound=
 429         pkhead=
 430         for pk in $allpacks; do
 431                 [ -s "$pk" ] || continue
 432                 pkbase="${pk%.pack}"
 433                 [ -s "$pkbase.idx" ] || continue
 434                 [ ! -e "$pkbase.keep" ] || continue
 435                 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
 436                         pkfound="$pkbase"
 437                         break;
 438                 fi
 439         done
 440         if [ -n "$pkfound" -a -n "$pkhead" ]; then
 441                 {
 442                         echo "# v2 git bundle"
 443                         sed -ne "/^$octet20 refs\/[^ $tab]*\$/ p" < packed-refs
 444                         echo "$pkhead HEAD"
 445                         echo ""
 446                 } > "$pkbase.bndl"
 447                 bndletag="$("$cfg_basedir/bin/rangecgi" --etag "$pkbase.bndl" "$pkbase.pack" || :)"
 448                 bndlsha="$(printf '%s' "$bndletag" | git hash-object --stdin || :)"
 449                 if [ -n "$bndletag" ]; then
 450                         case "$bndlsha" in $octet20)
 451                                 bndlshatrailer="${bndlsha#????????}"
 452                                 bndlshaprefix="${bndlsha%$bndlshatrailer}"
 453                                 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
 454                                 [ -d bundles ] || mkdir bundles
 455                                 echo "${pkbase#objects/pack/}.bndl" > "bundles/$bndlname"
 456                                 echo "${pkbase#objects/pack/}.pack" >> "bundles/$bndlname"
 457                                 ln -s -f -n "$bndlname" bundles/latest
 458                         esac
 459                 fi
 460         fi
 461 fi
 462
 463 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
 464 config_set_raw girocco.reposizek "${reposizek:-0}"
 465
 466 # We use $gcstart here to avoid a race where a push occurs during the gc itself
 467 # and the next future gc could be incorrectly skipped if we used the current
 468 # timestamp here instead
 469 config_set lastgc "$gcstart"
 470 rm -f "$lockf"
 471
 472 progress "- [$proj] garbage check (`date`)"