jobd/gc.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4
   5 set -e
   6 trap 'if [ $? != 0 ]; then echo "gc failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
   7
   8 # packing options
   9 packopts='--window=50 --window-memory=1g --depth=50'
  10
  11 umask 002
  12 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  13
  14 pidactive() {
  15         if _result="$(kill -0 "$1" 2>&1)"; then
  16                 # process exists and we have permission to signal it
  17                 return 0
  18         fi
  19         case "$_result" in *"not permitted"*)
  20                 # we do not have permission to signal the process
  21                 return 0
  22         esac
  23         # process does not exist
  24         return 1
  25 }
  26
  27 createlock() {
  28         for _try in p n; do
  29                 if (set -C; > "$1.lock") 2>/dev/null; then
  30                         echo "$1.lock"
  31                         return 0
  32                 fi
  33                 # delay and try again
  34                 [ "$_try" != "p" ] || sleep 1
  35         done
  36         # cannot create temp file
  37         return 1
  38 }
  39
  40 # if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
  41 repack_gfi_packs() {
  42         is_gfi_mirror || return 0
  43         [ -s gfi-packs ] || return 0
  44         while IFS=': ' read -r _pack _junk; do
  45                 if [ -s "$_pack" -a -s "${_pack%.pack}.idx" ]; then
  46                         git show-index < "${_pack%.pack}.idx" | cut -d ' ' -f 2
  47                 fi
  48         done < gfi-packs | \
  49         git pack-objects $packopts --no-reuse-delta --delta-base-offset \
  50                 --non-empty --threads=1 $quiet objects/pack/packtmp | \
  51         while read -r _newpack; do
  52                 rm -f objects/pack/pack-$_newpack.*
  53                 ln objects/pack/packtmp-$_newpack.pack objects/pack/pack-$_newpack.pack
  54                 ln objects/pack/packtmp-$_newpack.idx objects/pack/pack-$_newpack.idx
  55                 rm -f objects/pack/packtmp-$_newpack.*
  56         done
  57         rm -f gfi-packs
  58 }
  59
  60 proj="$1"
  61 cd "$cfg_reporoot/$proj.git"
  62
  63 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
  64 datefmt='+%a, %d %b %Y %T %z'
  65
  66 if check_interval lastgc $cfg_min_gc_interval; then
  67         progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
  68         exit 0
  69 fi
  70 if [ -e .nogc ]; then
  71         progress "x [$proj] garbage check disabled"
  72         exit 0
  73 fi
  74
  75 # Avoid unnecessary garbage collections:
  76 #   1. If lastreceive is set and is older than lastgc
  77 #   -AND-
  78 #   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc
  79
  80 # If lastgc is NOT set or lastreceive is NOT set we MUST run gc
  81 # If we are a fork and lastparentgc is NOT set we MUST run gc
  82
  83 gcstart="$(date "$datefmt")"
  84 skipgc=
  85 isfork=
  86 [ -s objects/info/alternates ] && isfork=1
  87 lastparentgcsecs=
  88 [ -n "$isfork" ] && lastparentgcsecs="$(config_get_date_seconds lastparentgc || :)"
  89 lastreceivesecs=
  90 if lastreceivesecs="$(config_get_date_seconds lastreceive)" && \
  91    lastgcsecs="$(config_get_date_seconds lastgc)" && \
  92    [ $lastreceivesecs -lt $lastgcsecs ]; then
  93         # We've run gc since we last received, so maybe we can skip,
  94         # check if not fork or fork and lastparentgc < lastgc
  95         if [ -n "$isfork" ]; then
  96                 if [ -n "$lastparentgcsecs" ] && \
  97                    [ $lastparentgcsecs -lt $lastgcsecs ]; then
  98                         # We've run gc since our parent ran gc so we can skip
  99                         skipgc=1
 100                 fi
 101         else
 102                 # We don't have any alternates (we're not a forK) so we can skip
 103                 skipgc=1
 104         fi
 105 fi
 106
 107 # be compatibile with gc.pid file from newer Git releases
 108
 109 hn="$(hostname)"
 110 lockf=gc.pid
 111 active=
 112 if [ "$(createlock "$lockf")" ]; then
 113         # If $lockf is:
 114         #   1) less than 12 hours old
 115         #   2) contains two fields (pid hostname) NO trailing NL
 116         #   3) the hostname is different OR the pid is still alive
 117         # then we exit as another active process is holding the lock
 118         if [ "$(find "$lockf" -mmin -720 -print 2>/dev/null)" ]; then
 119                 apid=
 120                 ahost=
 121                 read -r apid ahost ajunk < "$lockf" || :
 122                 if [ "$apid" ] && [ "$ahost" ]; then
 123                         if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
 124                                 active=1
 125                         fi
 126                 fi
 127         fi
 128 else
 129         echo >&2 "[$proj] unable to create gc.pid.lock file"
 130         exit 1
 131 fi
 132 if [ -n "$active" ]; then
 133         rm "$lockf.lock"
 134         echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
 135         exit 1
 136 fi
 137 printf "%s %s" "$$" "$hn" > "$lockf.lock"
 138 chmod 0664 "$lockf.lock"
 139 mv -f "$lockf.lock" "$lockf"
 140
 141 if [ -n "$skipgc" ]; then
 142         progress "= [$proj] garbage check nothing to do (`date`)"
 143         config_set lastgc "$gcstart"
 144         rm "$lockf"
 145         exit 0
 146 fi
 147
 148 bumptime=
 149 if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
 150         # set lastparentgc and then update gcstart to be at least 1 second later
 151         config_set lastparentgc "$gcstart"
 152         bumptime=1
 153 fi
 154 if [ -z "$lastreceivesecs" ]; then
 155         # set lastreceive and then update gcstart to be at least 1 second later
 156         config_set lastreceive "$gcstart"
 157         bumptime=1
 158 fi
 159 if [ -n "$bumptime" ]; then
 160         sleep 1
 161         gcstart="$(date "$datefmt")"
 162 fi
 163
 164 progress "+ [$proj] garbage check (`date`)"
 165
 166 # safe pruning: we put all our objects in all forks, then we can
 167 # safely get rid of extra ones; repacks in forks will get rid of
 168 # the redundant ones again then
 169 forkdir="$1"
 170 if [ -d "../${forkdir##*/}" ]; then
 171         # It is enough to copy objects just one level down and get_repo_list
 172         # takes a regular expression (which is automatically prefixed with '^')
 173         # so we can easily match forks exactly one level down from this project
 174         get_repo_list "$forkdir/[^/]*:" |
 175                 while read fork; do
 176                         # Ignore forks that do not exist or are symbolic links
 177                         [ ! -L "$cfg_reporoot/$fork.git" -a -d "$cfg_reporoot/$fork.git" ] || \
 178                                 continue
 179                         # Or do not have a non-zero length alternates file
 180                         [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] || \
 181                                 continue
 182                         # Match objects in parent project
 183                         for d in objects/?? objects/pack; do
 184                                 [ "$d" != "objects/??" ] || continue
 185                                 mkdir -p "$cfg_reporoot/$fork.git/$d"
 186                                 [ "$d" != "objects/pack" ] || \
 187                                         [ "$(echo objects/pack/*)" != \
 188                                                         "objects/pack/*" ] || \
 189                                         continue
 190                                 ln -f "$d"/* "$cfg_reporoot/$fork.git/$d" || :
 191                         done
 192                         # Update the fork's lastparentgc date (must be current, not $gcstart)
 193                         GIT_DIR="$cfg_reporoot/$fork.git" git config \
 194                                 gitweb.lastparentgc "$(date "$datefmt")"
 195                 done
 196 fi
 197
 198 quiet=; [ -n "$show_progress" ] || quiet=-q
 199 git pack-refs --all
 200 repack_gfi_packs
 201 git repack $packopts -a -d -l $quiet
 202 git prune
 203 git update-server-info
 204 # darcs:// mirrors have a xxx.log file that will grow endlessly
 205 # if this is a mirror and the file exists, shorten it to 10000 lines
 206 # also take this opportunity to optimize the darcs repo
 207 if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then
 208         url="$(config_get baseurl || :)"
 209         case "$url" in darcs://*)
 210                 if [ -n "$cfg_mirror_darcs" ]; then
 211                         url="${url%/}"
 212                         basedarcs="$(basename "${url#darcs:/}")"
 213                         if [ -f "$basedarcs.log" ]; then
 214                                 tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$"
 215                                 mv -f "$basedarcs.log.$$" "$basedarcs.log"
 216                         fi
 217                         if [ -d "$basedarcs.darcs" ]; then
 218                                 (
 219                                         cd "$basedarcs.darcs"
 220                                         # Note that this does not optimize _darcs/inventories/ :(
 221                                         darcs optimize
 222                                 )
 223                         fi
 224                 fi
 225         esac
 226 fi
 227 # We use $gcstart here to avoid a race where a push occurs during the gc itself
 228 # and the next future gc could be incorrectly skipped if we used the current
 229 # timestamp here instead
 230 config_set lastgc "$gcstart"
 231 rm "$lockf"
 232
 233 progress "- [$proj] garbage check (`date`)"