jobd/gc.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4
   5 set -e
   6 trap 'if [ $? != 0 ]; then echo "gc failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
   7
   8 # packing options
   9 packopts='--window=50 --window-memory=1g --depth=50'
  10
  11 umask 002
  12 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  13
  14 # SHA-1 pattern
  15 octet='[0-9a-f][0-9a-f]'
  16 octet4="$octet$octet$octet$octet"
  17 octet20="$octet4$octet4$octet4$octet4$octet4"
  18
  19 pidactive() {
  20         if _result="$(kill -0 "$1" 2>&1)"; then
  21                 # process exists and we have permission to signal it
  22                 return 0
  23         fi
  24         case "$_result" in *"not permitted"*)
  25                 # we do not have permission to signal the process
  26                 return 0
  27         esac
  28         # process does not exist
  29         return 1
  30 }
  31
  32 createlock() {
  33         for _try in p n; do
  34                 if (set -C; > "$1.lock") 2>/dev/null; then
  35                         echo "$1.lock"
  36                         return 0
  37                 fi
  38                 # delay and try again
  39                 [ "$_try" != "p" ] || sleep 1
  40         done
  41         # cannot create temp file
  42         return 1
  43 }
  44
  45 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
  46 repack_gfi_packs() {
  47         is_gfi_mirror || return 0
  48         [ -s gfi-packs ] || return 0
  49         while IFS=': ' read -r _pack _junk; do
  50                 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
  51                         git show-index < "${_pack%.pack}.idx" | cut -d ' ' -f 2
  52                 fi
  53         done < gfi-packs | \
  54         git pack-objects $packopts --no-reuse-delta --delta-base-offset \
  55                 --non-empty --threads=1 $quiet objects/pack/packtmp | \
  56         while read -r _newpack; do
  57                 rm -f objects/pack/pack-$_newpack.*
  58                 ln -f objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
  59                 ln -f objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
  60                 rm -f objects/pack/packtmp-$_newpack.*
  61         done
  62         rm -f gfi-packs
  63 }
  64
  65 # HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
  66 pack_is_complete() {
  67         # Must have a matching .idx file and a non-empty packed-refs file
  68         [ -s "${1%.pack}.idx" ] || return 1
  69         [ -s "$2" ] || return 1
  70         _headsha=
  71         case "$3" in
  72                 $octet20)
  73                         _headsha="$3"
  74                         ;;
  75                 "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
  76                         _headmatch="${3#ref:}"
  77                         _headmatch="${_headmatch# }"
  78                         _headmatchpat="$(echo "$_headmatch" | sed -e 's/\([.$]\)/\\\1/g')"
  79                         _headsha="$(grep -e "^$octet20 $_headmatchpat\$" < "$2" | \
  80                                     cut -d ' ' -f 1)"
  81                         case "$_headsha" in $octet20) :;; *)
  82                                 return 1
  83                         esac
  84                         ;;
  85                 *)
  86                         # bad HEAD
  87                         return 1
  88         esac
  89         rm -rf pack_is_complete_test
  90         mkdir pack_is_complete_test
  91         mkdir pack_is_complete_test/refs
  92         mkdir pack_is_complete_test/objects
  93         mkdir pack_is_complete_test/objects/pack
  94         echo "$_headsha" > pack_is_complete_test/HEAD
  95         ln -s "$1" pack_is_complete_test/objects/pack/
  96         ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
  97         ln -s "$2" pack_is_complete_test/packed-refs
  98         _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null || :)"
  99         rm -rf pack_is_complete_test
 100         [ -n "$_count" ] || return 1
 101         [ "$_count" -gt 0 ] 2>/dev/null || return 1
 102         echo "$_headsha"
 103 }
 104
 105 proj="$1"
 106 cd "$cfg_reporoot/$proj.git"
 107
 108 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
 109 datefmt='+%a, %d %b %Y %T %z'
 110
 111 if check_interval lastgc $cfg_min_gc_interval; then
 112         progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
 113         exit 0
 114 fi
 115 if [ -e .nogc ]; then
 116         progress "x [$proj] garbage check disabled"
 117         exit 0
 118 fi
 119
 120 # Avoid unnecessary garbage collections:
 121 #   1. If lastreceive is set and is older than lastgc
 122 #   -AND-
 123 #   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
 124
 125 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
 126 # If we are a fork and lastparentgc is NOT set we MUST run gc
 127
 128 gcstart="$(date "$datefmt")"
 129 skipgc=
 130 isfork=
 131 [ -s objects/info/alternates ] && isfork=1
 132 lastparentgcsecs=
 133 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
 134 lastreceivesecs=
 135 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
 136    lastgcsecs="$(config_get_date_seconds lastgc)" && \
 137    [ $lastreceivesecs -lt $lastgcsecs ]; then
 138         # We've run gc since we last received, so maybe we can skip,
 139         # check if not fork or fork and lastparentgc < lastgc
 140         if [ -n "$isfork" ]; then
 141                 if [ -n "$lastparentgcsecs" ] && \
 142                    [ $lastparentgcsecs -lt $lastgcsecs ]; then
 143                         # We've run gc since our parent ran gc so we can skip
 144                         skipgc=1
 145                 fi
 146         else
 147                 # We don't have any alternates (we're not a forK) so we can skip
 148                 skipgc=1
 149         fi
 150 fi
 151
 152 # be compatibile with gc.pid file from newer Git releases
 153
 154 hn="$(hostname)"
 155 lockf=gc.pid
 156 active=
 157 if [ "$(createlock "$lockf")" ]; then
 158         # If $lockf is:
 159         #   1) less than 12 hours old
 160         #   2) contains two fields (pid hostname) NO trailing NL
 161         #   3) the hostname is different OR the pid is still alive
 162         # then we exit as another active process is holding the lock
 163         if [ "$(find "$lockf" -mmin -720 -print 2>/dev/null)" ]; then
 164                 apid=
 165                 ahost=
 166                 read -r apid ahost ajunk < "$lockf" || :
 167                 if [ "$apid" ] && [ "$ahost" ]; then
 168                         if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
 169                                 active=1
 170                         fi
 171                 fi
 172         fi
 173 else
 174         echo >&2 "[$proj] unable to create gc.pid.lock file"
 175         exit 1
 176 fi
 177 if [ -n "$active" ]; then
 178         rm "$lockf.lock"
 179         echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
 180         exit 1
 181 fi
 182 printf "%s %s" "$$" "$hn" > "$lockf.lock"
 183 chmod 0664 "$lockf.lock"
 184 mv -f "$lockf.lock" "$lockf"
 185
 186 if [ -n "$skipgc" ]; then
 187         progress "= [$proj] garbage check nothing to do (`date`)"
 188         config_set lastgc "$gcstart"
 189         rm "$lockf"
 190         exit 0
 191 fi
 192
 193 bumptime=
 194 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
 195         # set lastparentgc and then update gcstart to be at least 1 second later
 196         config_set lastparentgc "$gcstart"
 197         bumptime=1
 198 fi
 199 if [ -z "$lastreceivesecs" ]; then
 200         # set lastreceive and then update gcstart to be at least 1 second later
 201         config_set lastreceive "$gcstart"
 202         bumptime=1
 203 fi
 204 if [ -n "$bumptime" ]; then
 205         sleep 1
 206         gcstart="$(date "$datefmt")"
 207 fi
 208
 209 progress "+ [$proj] garbage check (`date`)"
 210
 211 # Remove any stale pack remnants that are more than an hour old.
 212 # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
 213 # .pack AND the corresponding .pack DOES NOT exist.  A bunch of stale
 214 # pack-<sha1>.idx files without their corresponding .pack files are worthless
 215 # and just waste space.  Normally there shouldn't be any remnants but actually
 216 # this can happen when things are interrupted at just the wrong time.
 217 # Note that the objects/pack directory is created by git init and should
 218 # always exist.
 219 find objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20.?*" | \
 220 sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | sort -u | \
 221 while read packsha; do
 222         [ ! -e "objects/pack/pack-$packsha.pack" ] || continue
 223         rm -f "objects/pack/pack-$packsha".?*
 224 done
 225
 226 # safe pruning: we put all our objects in all forks, then we can
 227 # safely get rid of extra ones; repacks in forks will get rid of
 228 # the redundant ones again then
 229 forkdir="$1"
 230 if [ -d "../${forkdir##*/}" ]; then
 231         # It is enough to copy objects just one level down and get_repo_list
 232         # takes a regular expression (which is automatically prefixed with '^')
 233         # so we can easily match forks exactly one level down from this project
 234         get_repo_list "$forkdir/[^/]*:" |
 235                 while read fork; do
 236                         # Ignore forks that do not exist or are symbolic links
 237                         [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
 238                                 continue
 239                         # Or do not have a non-zero length alternates file
 240                         [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
 241                                 continue
 242                         # Match objects in parent project
 243                         for d in objects/?? objects/pack; do
 244                                 [ "$d" != "objects/??" ] || continue
 245                                 mkdir -p "$cfg_reporoot/$fork.git/$d"
 246                                 [ "$d" != "objects/pack" ] || \
 247                                         [ "$(echo objects/pack/*)" != \
 248                                                         "objects/pack/*" ] || \
 249                                         continue
 250                                 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
 251                         done
 252                         # Update the fork's lastparentgc date (must be current, not $gcstart)
 253                         GIT_DIR="$cfg_reporoot/$fork.git" git config \
 254                                 gitweb.lastparentgc "$(date "$datefmt")"
 255                 done
 256 fi
 257
 258 quiet=; [ -n "$show_progress" ] || quiet=-q
 259
 260 git pack-refs --all
 261 repack_gfi_packs
 262 rm -f bundles/*
 263 rm -f objects/pack/pack-*.bndl
 264 git repack $packopts -a -d -l $quiet
 265 allpacks="$(echo objects/pack/pack-$octet20.pack)"
 266 curhead="$(cat HEAD)"
 267 eval "reposizek=$(( $(echo 0 $(du -k $allpacks 2>/dev/null | awk '{print $1}') | \
 268         sed -e 's/ / + /g') ))"
 269 git prune
 270 git update-server-info
 271
 272 # darcs:// mirrors have a xxx.log file that will grow endlessly
 273 # if this is a mirror and the file exists, shorten it to 10000 lines
 274 # also take this opportunity to optimize the darcs repo
 275 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
 276         url="$(config_get baseurl || :)"
 277         case "$url" in darcs://*)
 278                 if [ -n "$cfg_mirror_darcs" ]; then
 279                         url="${url%/}"
 280                         basedarcs="$(basename "${url#darcs:/}")"
 281                         if [ -f "$basedarcs.log" ]; then
 282                                 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
 283                                 mv -f "$basedarcs.log.$$" "$basedarcs.log"
 284                         fi
 285                         if [ -d "$basedarcs.darcs" ]; then
 286                                 (
 287                                         cd "$basedarcs.darcs"
 288                                         # Note that this does not optimize _darcs/inventories/ :(
 289                                         darcs optimize
 290                                 )
 291                         fi
 292                 fi
 293         esac
 294 fi
 295
 296 # Create a matching .bndl header file for the all-in-one pack we just created
 297 # but only if we're not a fork (otherwise the bundle would not be complete)
 298 if [ ! -s objects/info/alternates ]; then
 299         # There should only be one pack in $allpacks but if there was a
 300         # simultaneous push...
 301         # The one we just created will have a .idx and will NOT have a .keep
 302         pkfound=
 303         pkhead=
 304         for pk in $allpacks; do
 305                 [ -s "$pk" ] || continue
 306                 pkbase="${pk%.pack}"
 307                 [ -s "$pkbase.idx" ] || continue
 308                 [ ! -e "$pkbase.keep" ] || continue
 309                 if pkhead="$(pack_is_complete "$PWD/$pk" "$PWD/packed-refs" "$curhead")"; then
 310                         pkfound="$pkbase"
 311                         break;
 312                 fi
 313         done
 314         if [ -n "$pkfound" -a -n "$pkhead" ]; then
 315                 {
 316                         echo "# v2 git bundle"
 317                         sed -ne "/^$octet20 refs\/[^    ]*\$/ p" < packed-refs
 318                         echo "$pkhead HEAD"
 319                         echo ""
 320                 } > "$pkbase.bndl"
 321                 bndlsha="${pkbase#objects/pack/pack-}"
 322                 bndlshatrailer="${bndlsha#????????}"
 323                 bndlshaprefix="${bndlsha%$bndlshatrailer}"
 324                 bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
 325                 [ -d bundles ] || mkdir bundles
 326                 echo "$bndlsha" > "bundles/$bndlname"
 327         fi
 328 fi
 329
 330 # Record the size of this repo as the sum of its *.pack sizes as 1024-byte blocks
 331 config_set_raw girocco.reposizek "${reposizek:-0}"
 332
 333 # We use $gcstart here to avoid a race where a push occurs during the gc itself
 334 # and the next future gc could be incorrectly skipped if we used the current
 335 # timestamp here instead
 336 config_set lastgc "$gcstart"
 337 rm "$lockf"
 338
 339 progress "- [$proj] garbage check (`date`)"