hooks: get out of the mob for free

[girocco.git] / jobd / gc.sh

#!/bin/sh

# NOTE: additional options can be passed to git repack by specifying
#       them after the project name, for example:
#         gc.sh my-project -f

. @basedir@/shlib.sh

set -e

if [ $# -lt 1 ]; then
        echo "Usage: gc.sh projname [extra-repack-args]" >&2
        exit 1
fi

# packing options
packopts="--depth=50 --window=50 --window-memory=${var_window_memory:-1g}"
quiet=; [ -n "$show_progress" ] || quiet=-q

umask 002
[ "$cfg_permission_control" != "Hooks" ] || umask 000
clean_git_env

pidactive() {
        if _result="$(kill -0 "$1" 2>&1)"; then
                # process exists and we have permission to signal it
                return 0
        fi
        case "$_result" in *"not permitted"*)
                # we do not have permission to signal the process
                return 0
        esac
        # process does not exist
        return 1
}

createlock() {
        # A .lock file should only exist for much less than a second.
        # If we see a stale lock file (> 1h old), remove it and then,
        # just in case, wait 30 seconds for any process whose .lock
        # we might have just removed (it's racy) to finish doing what
        # should take much less than a second to do.
        _stalelock="$(find -L "$1.lock" -maxdepth 1 -mmin +60 -print 2>/dev/null)" || :
        if [ -n "$_stalelock" ]; then
                rm -f "$_stalelock"
                sleep 30
        fi
        for _try in p p n; do
                if (set -C; >"$1.lock") 2>/dev/null; then
                        echo "$1.lock"
                        return 0
                fi
                # delay and try again
                [ "$_try" != "p" ] || sleep 1
        done
        # cannot create lock file
        return 1
}

# The pre-receive script creates one ref log file per push but we want them to
# be coalesced into one ref log file per day.  We are guaranteed that any files
# we find to coalesce are NOT currently being written to since they are always
# written first as temporary files and then moved into place.  We attempt to
# transfer the most recent modification time to the coalesced log file which
# would step on its mod time if it were being written to directly, but if we
# find per-process ref log files then it must be a push project and the only
# thing that would write directly to the main per-day log file would be a
# mirror project so there's actually no conflict.
# Also, if the clock is wonky (or was futzed with) we may have both YYYYMMDD
# and YYYYMMDD.gz present in which case combine them into YYYYMMDD
coalesce_reflogs() {
        [ -d reflogs ] || return 0
        rm -f .gc_failed
        find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
        while read -r rname; do
                if [ -e "$rname.gz" ]; then
                        if [ -s "$rname" ]; then
                                # Presumably the .gz file must have been created before the non-gz
                                # file since it had to be uncompressed at some point therefore
                                # we need to append the non-gz contents to it but keep the non-gz
                                # contents timestamp so we rename to YYYYMMDD_ which will sort first
                                # and be picked up in the next step if we are interrupted in the middle.
                                # If a YYYYMMDD_ file already exists we append to it and transfer the
                                # timestamp.  Finally we transfer the YYYYMMDD_ timestamp to the result
                                # and remove the YYYYMMDD_ temporary file leaving the result uncompressed.
                                if [ -e "${rname}_" ]; then
                                        cat "$rname" >>"${rname}_"
                                        touch -r "$rname" "${rname}_"
                                        rm -f "$rname"
                                        ! [ -e "$rname" ]
                                else
                                        mv "$rname" "${rname}_"
                                fi
                                gzip -d "$rname.gz" </dev/null
                                [ -e "$rname" ] && ! [ -e "$rname.gz" ]
                                cat "${rname}_" >>"$rname"
                                touch -r "${rname}_" "$rname"
                                rm -f "${rname}_"
                        else
                                # Just remove the empty file to resolve the problem
                                rm -f "$rname"
                        fi
                fi
        done
        find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*" -print | LC_ALL=C sort |
        while read -r rname; do
                logname="${rname%%_*}"
                # If someone's been futzing with the date, the file we want to
                # append to could already have been compressed, so we just uncompress
                # it here.  The previous block guarantees we do not have both a compressed
                # and uncompressed version present at the same time.
                if [ -e "$logname.gz" ]; then
                        gzip -d "$logname.gz" </dev/null
                        [ -e "$logname" ] && ! [ -e "$logname.gz" ]
                fi
                cat "$rname" >>"$logname"
                touch -r "$rname" "$logname"
                rm -f "$rname"
                if [ -e "$rname" ]; then
                        >.gc_failed
                        echo "! [$proj] failed to remove $rname" >&2
                        exit 1 # will only exit subshell created by "|"
                fi
        done
        ! [ -e .gc_failed ]
}

# Remove any files in reflogs that are older than $cfg_reflogs_lifetime days
prune_reflogs() {
        [ -d reflogs ] || return 0
        exp="$(( ${cfg_reflogs_lifetime:-1} * 1440 ))"
        [ $exp -gt 0 ] || exp=1440
        [ $exp -le 43200 ] || exp=43200
        find -L reflogs -maxdepth 1 -type f -mmin "+$exp" -exec rm -f '{}' + || :
}

# Compact any reflogs that are not today's UTC date unless a .gz version exists
compact_reflogs() {
        [ -d reflogs ] || return 0
        _td="reflogs/$(TZ=UTC date '+%Y%m%d')"
        find -L reflogs -maxdepth 1 -type f -name "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" -print |
        while read -r rname; do
                [ "$rname" != "$_td" ] || continue
                ! [ -e "$rname.gz" ] || continue
                gzip -9 "$rname" </dev/null
        done
}

# return true if there's more than one objects/pack-<sha>.pack file or
# ANY sha-1 files in objects
is_dirty() {
        _packs=$(find -L objects/pack -type f -name "pack-$octet20*.pack" -print | head -n 2 | LC_ALL=C wc -l)
        if [ $_packs != 1 ] && [ $_packs != 0 ]; then
                return 0
        fi
        _objs=$(find -L objects/$octet -type f -name "$octet19*" -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l)
        [ $_objs -ne 0 ]
}

# make sure combine-packs uses the correct Git executable
run_combine_packs() {
        PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/combine-packs.sh "$@"
}

# combine the input pack(s) into a new pack (or possibly packs if packSizeLimit set)
# input pack names are read from standard input one per line delimited by the first
# ':', ' ' or '\n' character on the line (which allows gfi-packs to be read directly)
# all arguments, if any, are passed to pack-objects as additional options
# returns non-zero on failure AND creates .gc_failed in that case
combine_packs() {
        rm -f .gc_failed
        find -L objects/pack -maxdepth 1 -type f -name '*.zap*' -exec rm -f '{}' + || :
        run_combine_packs --replace "$@" $packopts --all-progress-implied $quiet --non-empty || {
                >.gc_failed
                return 1
        }
        return 0
}

# if the current directory is_gfi_mirror then repack all packs listed in gfi-packs
repack_gfi_packs() {
        [ -n "$gfi_mirror" ] || return 0
        [ -d objects/pack ] || { rm -f gfi-packs; return 0; }
        progress "~ [$proj] redeltifying poor quality git fast-import packs"
        combine_packs --ignore-missing --no-reuse-delta <gfi-packs
        rm -f gfi-packs
        return 0
}

# combine small packs into larger pack(s)
# we avoid any keep, bndl or bitmap packs
# if the optional argument is non-empty even a single small pack will be redeltad
combine_small_packs() {
        _didprogress=
        _minsmallpacks=2
        if [ -n "$1" ] && [ -n "$noreusedeltaopt" ]; then
                _minsmallpacks=1
        fi
        _lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl"
        _lpo="$_lpo --exclude-sfx _u --exclude-sfx _o"
        _lpo="$_lpo --quiet --object-limit $var_redelta_threshold objects/pack"
        while
                _cnt="$(list_packs --count $_lpo)" || :
                test "${_cnt:-0}" -ge $_minsmallpacks
        do
                [ -n "$_didprogress" ] || {
                        progress "~ [$proj] combining small packs into a single larger pack"
                        _didprogress=1
                }
                _newp="$(list_packs $_lpo | combine_packs --names $noreusedeltaopt)"
                _newc="$(( $(echo "$_newp" | LC_ALL=C wc -w) ))"
                # be paranoid and exit the loop if we haven't reduced the number of packs
                [ $_newc -lt $_cnt ] || break
                _minsmallpacks=2
        done
        return 0
}

# Unfortunately git-svn lacks the ability to store newly fetched revisions as a pack.
# However, the fetch code conveniently sets .svnpack just before it runs git-svn fetch
# so that it's easy to find all the objects that have been fetched by git-svn and
# combine them into a pack.  The --no-reuse-delta option is meaningless here since
# everything to be packed is a loose object and therefore not a delta so deltification
# will always take place.
make_svn_pack() {
        [ -f .svnpack ] && [ -n "$svn_mirror" ] || return 0
        rm -f .svnpackgc
        mv -f .svnpack .svnpackgc
        progress "~ [$proj] combining loose git-svn objects into a pack"
        _newp="$(find -L objects/$octet -maxdepth 1 -type f -newer .svnpackgc -name "$octet19*" -print 2>/dev/null |
        LC_ALL=C awk -F / '{print $2 $3}' |
        run_combine_packs --objects --names $packopts --incremental --all-progress-implied $quiet --non-empty)" || {
                mv -f .svnpackgc .svnpack
                >.gc_failed
                return 1
        }
        if [ -n "$_newp" ]; then
                # remove the now-redundant loose objects -- this is always safe
                # even during a concurrent push because a reprepare_packed_git
                # will be triggered if an object that should be there is not
                # found thereby finding it in the new pack instead
                git prune-packed $quiet
        fi
        rm -f .svnpackgc
}

# HEADSHA="$(pack_is_complete /full/path/to/some.pack /full/path/to/packed-refs "$(cat HEAD)")"
pack_is_complete() {
        # Must have a matching .idx file and a non-empty packed-refs file
        [ -s "${1%.pack}.idx" ] || return 1
        [ -s "$2" ] || return 1
        _headsha=
        case "$3" in
                $octet20*)
                        _headsha="$3"
                        ;;
                "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
                        _headmatch="${3#ref:}"
                        _headmatch="${_headmatch# }"
                        _headmatchpat="$(echo "$_headmatch" | LC_ALL=C sed -e 's/\([.$]\)/\\\1/g')"
                        _headsha="$(LC_ALL=C grep -e "^$octet20$hexdig* $_headmatchpat\$" <"$2" |
                                    LC_ALL=C cut -d ' ' -f 1)"
                        case "$_headsha" in $octet20*) :;; *)
                                return 1
                        esac
                        ;;
                *)
                        # bad HEAD
                        return 1
        esac
        rm -rf pack_is_complete_test
        mkdir pack_is_complete_test
        mkdir pack_is_complete_test/refs
        mkdir pack_is_complete_test/objects
        mkdir pack_is_complete_test/objects/pack
        echo "$_headsha" >pack_is_complete_test/HEAD
        ln -s "$1" pack_is_complete_test/objects/pack/
        ln -s "${1%.pack}.idx" pack_is_complete_test/objects/pack/
        ln -s "$2" pack_is_complete_test/packed-refs
        _count="$(git --git-dir=pack_is_complete_test rev-list --count --all 2>/dev/null)" || :
        rm -rf pack_is_complete_test
        [ -n "$_count" ] || return 1
        [ "$_count" -gt 0 ] 2>/dev/null || return 1
        echo "$_headsha"
}

# On return a "$lockf" will have been created that must be removed when gc is done
lock_gc() {
        # be compatibile with gc.pid file from newer Git releases
        lockf=gc.pid
        hn="$(hostname)"
        active=
        if [ "$(createlock "$lockf")" ]; then
                # If $lockf is:
                #   1) less than 12 hours old
                #   2) contains two fields (pid hostname) NO trailing NL
                #   3) the hostname is different OR the pid is still alive
                # then we exit as another active process is holding the lock
                if [ "$(find -L "$lockf" -maxdepth 1 -mmin -720 -print 2>/dev/null)" ]; then
                        apid=
                        ahost=
                        read -r apid ahost ajunk <"$lockf" || :
                        if [ "$apid" ] && [ "$ahost" ]; then
                                if [ "$ahost" != "$hn" ] || pidactive "$apid"; then
                                        active=1
                                fi
                        fi
                fi
        else
                echo >&2 "[$proj] unable to create gc.pid.lock file"
                exit 1
        fi
        if [ -n "$active" ]; then
                rm -f "$lockf.lock"
                echo >&2 "[$proj] gc already running on machine '$ahost' pid '$apid'"
                exit 1
        fi
        printf "%s %s" "$$" "$hn" >"$lockf.lock"
        chmod 0664 "$lockf.lock"
        mv -f "$lockf.lock" "$lockf"
}

# Create a repack subdirectory such that running repack in it will pack the
# same things that a pack in the normal directory would except that the pack
# is guaranteed to be generated in an optimized order by adding a suitable
# synthesized ref in the refs/tags namespace (yes, pack-objects.c really does
# behave differently depending on the contents of the refs/tags namespace).
# Before calling this, pack-refs --all MUST be performed or the wrong pack
# will end up being made.
#
# If a ref deletion is pushed after making the repack subdir but before the
# the actual repack, the discarded objects will be packed -- no big deal,
# they'll get discarded the next time gc runs.
#
# If a fast-forward ref update is pushed after making the repack subdir but
# before the actual repack, it will be picked up and the new objects packed
# (subject to the normal git repack race about picking such updates up).
#
# If a non-fast-forward ref update is pushed after making the repack subdir but
# before the actual repack, it will be picked up like a fast-forward update but
# the discarded objects will be included like a ref deletion (until the next
# scheduled gc takes place).
#
# We retain a copy of the original packed-refs file as repack/packed-refs.orig
# If ref deletions come in while we're repacking, the original packed-refs
# file will be modified, but we'll still pack the deleted ref(s).
# If the packed-refs.orig file is used to create the bundle header we avoid
# a situation where the bundle contains a ref state that never actually
# existed in reality (for example a new branch is pushed and then an old
# branch deleted afterwards -- the deletion would show up in the bundle
# because it will cause the original packed-refs file to be re-written, but
# the new branch creation will not unless we do another pack-refs which might
# lead to having in incomplete bundle).  Therefore we want to keep a copy of
# the original packed-refs file around.  We do the same thing for HEAD.
#
# It's possible that the "objects" subdirectory is a symbolic link.
# Git does support this.  However, during the repacking process, new packs
# will be created in repack/alt/pack and then moved into objects/pack.
# In order for this to work seemlessly, they must both be on the same
# filesystem.  But when objects (or even objects/pack) is a symbolic link they
# might not be.  For this reason a "repack" subdirectory is created under
# objects/pack and the repack/alt/pack directory symbolicly linked to it.
#
# Git allows not just HEAD to be a symbolic-ref, but any ref anywhere in the
# refs namespace.  We are concerned about ref name collisions and getting the
# right tag set to get an optimal pack.  We can safely duplicate the ref space
# under refs/heads, refs/notes and refs/remotes without any risk of unwanted
# collisions and this will likely make over 99%+ of all symbolic refs found
# in the wild work properly.  Girocco itself never creates any symbolic refs
# inside the refs namespace; this is a nod to simultaneously using a Girocco
# repository for other purposes.
make_repack_dir() {
        ! [ -d repack ] || rm -rf repack
        ! [ -d repack ] || { echo >&2 "[$proj] cannot remove repack subdirectory"; exit 1; }
        [ -d objects/pack ] || mkdir -p objects/pack
        ! [ -d objects/pack/repack ] || rm -rf objects/pack/repack
        ! [ -d objects/pack/repack ] || { echo >&2 "[$proj] cannot remove objects/pack/repack subdirectory"; exit 1; }
        mkdir repack repack/refs repack/alt objects/pack/repack
        [ -d info ] || mkdir info
        ln -s ../config  repack/config
        ln -s ../info    repack/info
        ln -s ../objects repack/objects
        ln -s "$PWD/objects/pack/repack" repack/alt/pack
        ln -s ../../refs repack/refs/refs
        _lines=$(( $(LC_ALL=C wc -l <packed-refs) ))
        cat HEAD >repack/HEAD.orig
        cat packed-refs >repack/packed-refs.orig
        if [ $(LC_ALL=C wc -l <repack/packed-refs.orig) -ne "$_lines" ]; then
                echo >&2 "[$proj] error: make_repack_dir failed original packed-refs line count sanity check"
                exit 1
        fi
        # Note: Git v1.5.0 introduced the "# pack-refs with:" header line for the packed-refs file
        sed '/^# pack-refs/d; s, refs/, refs/!/,' <repack/packed-refs.orig >repack/packed-refs
        headref="$(git rev-parse --verify --quiet HEAD)" || :
        if [ -n "$headref" ]; then
                echo "$headref refs/!=/HEAD" >>repack/packed-refs
                echo "$headref refs/heads/!" >>repack/packed-refs
                _lines=$(( $_lines + 2 ))
        fi
        if [ $(( $(LC_ALL=C wc -l <repack/packed-refs) + 1 )) -ne "$_lines" ]; then
                echo >&2 "[$proj] error: make_repack_dir failed packed-refs initial line count sanity check"
                exit 1
        fi
        sed -n '\, refs/heads/,p; \, refs/notes/,p; \, refs/remotes/,p' <repack/packed-refs.orig >>repack/packed-refs
        _newlines="$(( $(LC_ALL=C wc -l <repack/packed-refs) ))"
        if [ $(( $_newlines + 1 )) -lt "$_lines" ]; then
                echo >&2 "[$proj] error: make_repack_dir failed packed-refs extra line count sanity check"
                exit 1
        fi
        _lines="$_newlines"
        optref="$(git rev-list -n 1 --all 2>/dev/null)" || :
        if [ -n "$optref" ]; then
                echo "$optref refs/tags/!" >>repack/packed-refs
                _lines=$(( $_lines + 1 ))
                echo "$optref" >repack/HEAD
        else
                cat HEAD >repack/HEAD
        fi
        if [ $(LC_ALL=C wc -l <repack/packed-refs) -ne "$_lines" ]; then
                echo >&2 "[$proj] error: make_repack_dir failed packed-refs line count sanity check"
                exit 1
        fi
}

# Remove any crud that's been left behind by interrupted operations
# that did not clean up after themselves
remove_crud() {
        # Remove any existing FETCH_HEAD
        # There can only be a FETCH_HEAD if we've been fetching, not if we've been
        # receiving pushes (those never create a FETCH_HEAD).
        # And if we're fetching because we're a mirror, we know we're not fetching right
        # now since jobd.pl never runs a project's fetch simultaneously with its gc.
        # Therefore any existing FETCH_HEAD is junk.  And it may be many megabytes if
        # there were a lot of refs.
        rm -f FETCH_HEAD

        # remove any existing pack_is_complete_test or repack subdirectories
        # If either exists when this function is called it's crud
        rm -rf pack_is_complete_test repack objects/pack/repack

        # Remove any stale pack remnants that are more than an hour old.
        # Stale pack fragments are defined as any pack-<sha1>.ext where .ext is NOT
        # .pack AND the corresponding .pack DOES NOT exist.  A bunch of stale
        # pack-<sha1>.idx files without their corresponding .pack files are worthless
        # and just waste space.  Normally there shouldn't be any remnants but actually
        # this can happen when things are interrupted at just the wrong time.
        # Note that the objects/pack directory is created by git init and should
        # always exist.
        find -L objects/pack -maxdepth 1 -type f -mmin +60 -name "pack-$octet20*.?*" -print |
        LC_ALL=C sed -e 's/^objects\/pack\/pack-//; s/\..*$//' | LC_ALL=C sort -u |
        while read packsha; do
                ! [ -e "objects/pack/pack-$packsha.pack" ] || continue
                rm -f "objects/pack/pack-$packsha".?*
        done

        # Remove any stale tmp reflogs files that are more than one hour old.
        # Since they are created only while the pre-receive hook is running and
        # all it does is process a bunch of refs passed to it on standard input
        # it's inconceivable that it would ever take as much as an hour to run.
        if [ -d reflogs ]; then
                find -L reflogs -maxdepth 1 -type f -mmin +60 -name "tmp_*" -exec rm -f '{}' + || :
        fi

        # Remove any stale object tmp_obj_* files that are more than 3 hours old.
        # Really these files should only exist very briefly so there shouldn't be any
        # but things happen that can end up leaving them behind.
        find -L objects/$octet -maxdepth 1 -type f -mmin +180 -name "tmp_obj_?*" -exec rm -f '{}' + 2>/dev/null || :

        # Remove any stale pack .keep files that are more than 12 hours old.
        # We don't do anything to create any permanent pack .keep files, so they must
        # be remnants from some failed push or something.  Removing the .keep will
        # allow the pack to be properly repacked.
        find -L objects/pack -maxdepth 1 -type f -mmin +720 -name "pack-$octet20*.keep" -exec rm -f '{}' + || :

        # Remove any stale tmp_pack_*, tmp_idx_*, tmp_bitmap_*, packtmp-* or .tmp-*-pack* files
        # that are more than 12 hours old.
        find -L objects/pack -maxdepth 1 -type f -mmin +720 \( \
                -name "tmp_pack_?*" -o -name "tmp_idx_?*" -o -name "tmp_bitmap_?*" -o \
                -name "packtmp-?*" -o -name ".tmp-?*-pack*" \
                \) -exec rm -f '{}' + || :

        # Remove any stale incoming-* object quarantine directories that are
        # more than 12 hours old.  These are new with Git >= 2.11.0.
        find -L objects -maxdepth 1 -type d -name 'incoming-?*' -mmin +720 \
                -exec rm -rf '{}' + || :

        # Remove any stale shallow_* files that are more than 12 hours old.
        # These can be left behind by Git >= 1.8.4.2 and < 2.0.0 when a client
        # requests a shallow clone.
        find -L . -maxdepth 1 -type f -mmin +720 -name "shallow_?*" -exec rm -f '{}' + || :

        # Remove any stale *.temp files in the objects area that are more than 12 hours old.
        # This can be stale sha1.temp, or stale *.pack.temp so we kill all stale *.temp.
        find -L objects -type f -mmin +720 -name "*.temp" -exec rm -f '{}' + || :

        # Remove any stale *.lock files in the htmlcache area that might have been left
        # behind after an abnormal exit during an attempt to update a cached file and
        # are more than 1 hour old.
        ! [ -d htmlcache ] || find -L htmlcache -type f -mmin +60 -name "*.lock" -exec rm -f '{}' + || :

        # Remove any stale git-svn temp files that are more than 12 hours old.
        # The git-svn process creates temp files with random 10 character names
        # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
        # prefix, so we just have to kill any files with a 10-character name.  We
        # do this only for git-svn mirrors.  All characters are chosen from
        # [A-Za-z0-9_] so we can at least check that and fortunately the only
        # collision is 'FETCH_HEAD' but that shouldn't matter.
        # There may also be temp files with a Git_ prefix as well.
        if [ -n "$svn_mirror" ]; then
                _randchar='[A-Za-z0-9_]'
                _randchar2="$_randchar$_randchar"
                _randchar4="$_randchar2$_randchar2"
                _randchar10="$_randchar4$_randchar4$_randchar2"
                find -L . -maxdepth 1 -type f -mmin +720 -name "$_randchar10" -exec rm -f '{}' + || :
                find -L . -maxdepth 1 -type f -mmin +720 -name "Git_*" -exec rm -f '{}' + || :
        fi

        # Remove any stale fast_import_crash_<pid> files that are more than 3 days old.
        if [ -n "$gfi_mirror" ]; then
                find -L . -maxdepth 1 -type f -mmin +4320 -name "fast_import_crash_?*" -exec rm -f '{}' + || :
        fi

        # Remove any stale core or *.core or core.* files that are more than 3 days old.
        find -L . -maxdepth 1 -type f -mmin +4320 \( -name "core" -o -name "*.core" -o -name "core.*" \) \
                -exec rm -f '{}' + || :
}

#
## Garbage Collection Types
##
## There are two kinds of possible garbage collection (gc) operations:
##
## 1.  A normal, full gc
## 2.  A "mini" gc
##
## If the full garbage collection interval has expired (or gc has never been
## run), then a normal, full gc will take place.  Otherwise, a "mini" gc will
## take place if the file .needsgc exists.
##
## A "mini" gc is similar to "git gc --auto" in that it may not end up actually
## doing anything unless the right conditions are present so it's not a burden
## to run it often.  If the file .needsgc exists, a "mini" gc will occur at
## the next opportunity.
##
## Note, however, that the .nogc file suppresses ALL gc activity (normal or mini).
#

proj="${1%.git}"
shift
cd "$cfg_reporoot/$proj.git"
[ -d objects/pack ] || { rm -f gfi-packs; mkdir -p objects/pack; }
mirror_url="$(get_mirror_url)" || :
svn_mirror=
! is_svn_mirror_url "$mirror_url" || svn_mirror=1
gfi_mirror=
if [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$mirror_url"; then
        gfi_mirror=1
fi

# If git config --bool --get girocco.redelta is explicitly false then automatic
# redelta when there are less than $var_redelta_threshold objects will be suppressed.
# On the other hand, if git config --get girocco.redelta is "always" then, on a full
# gc only, for the final repack, deltas will always be recomputed.
# This can be set on a per-project basis to avoid unusual pathological gc behavior.
# Setting this will hurt efficiency of the affected repository.
# Note that fast-import packs ALWAYS get new deltas regardless of this setting.
noreusedeltaopt="--no-reuse-delta"
[ "$(git config --bool --get girocco.redelta 2>/dev/null || :)" != "false" ] || noreusedeltaopt=
alwaysredelta=
[ "$(git config --get girocco.redelta 2>/dev/null || :)" != "always" ] || alwaysredelta=1

# Extract any -f or -F or --no-reuse-object or --no-reuse-delta options
# to be compatible with the old and new gc.sh versions and avoid ugly argument
# duplication in process lists at the same time
# Any options found will override the "girocco.redelta" setting
recompress=
idx=$#
while [ $idx -gt 0 ]; do
        idx=$(( $idx - 1 ))
        opt="$1"
        shift
        case "$opt" in
        -f|--no-reuse-delta)
                alwaysredelta=1
                continue
                ;;
        -F|--no-reuse-object)
                alwaysredelta=1
                recompress=1
                continue
                ;;
        -?*)
                ;;
        *)
                printf >&2 '%s\n' "bad non-option argument: $opt"
                echo >&2 "(Did you perhaps intend to use a --xxx=yyy form?)"
                exit 1
        esac
        [ -z "$opt" ] || set -- "$@" "$opt"
done
if [ -n "$alwaysredelta" ]; then
        noreusedeltaopt="--no-reuse-delta"
        [ -z "$recompress" ] || noreusedeltaopt="--no-reuse-object"
fi

trap 'e=$?; rm -f .gc_in_progress; if [ $e != 0 ]; then echo "gc failed dir: $PWD" >&2; fi' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

# date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
datefmt='+%a, %d %b %Y %T %z'

isminigc=
if [ "${force_gc:-0}" = "0" ] && check_interval lastgc $cfg_min_gc_interval; then
        if [ -e .needsgc ]; then
                isminigc=1
        else
                progress "= [$proj] garbage check skip (last at $(config_get lastgc))"
                exit 0
        fi
fi
if [ -e .nogc ]; then
        progress "x [$proj] garbage check disabled"
        exit 0
fi
if [ -z "$isminigc" ] && [ -e .delaygc ] && [ -e .needsgc ]; then
        # Eligible for a full gc but .delaygc is set so it would be skipped
        # However .needsgc is also set so transform it into a mini instead
        isminigc=1
        progress "~ [$proj] garbage check delayed but checking mini because .needsgc"
fi

if [ -n "$isminigc" ]; then
        # Perform a "mini" gc
        # Note that .delaygc is ignored here as that's only intended for full gc
        lock_gc
        rm -f .allowgc .needsgc
        rm -f objects/pack/pack-*_r.keep
        remove_crud
        coalesce_reflogs
        prune_reflogs
        compact_reflogs
        miniactive=
        if [ -f .svnpack ] && [ -n "$svn_mirror" ]; then
                miniactive=1
                progress "+ [$proj] mini garbage check ($(date))"
                make_svn_pack
        fi
        if [ -z "$cfg_delay_gfi_redelta" ] && [ -n "$gfi_mirror" ]; then
                # $Girocco::Config::delay_gfi_redelta is false, force redeltification now
                if [ -z "$miniactive" ]; then
                        miniactive=1
                        progress "+ [$proj] mini garbage check ($(date))"
                fi
                repack_gfi_packs
        fi
        # If there aren't at least 10 non-keep, non-bitmap, non-bndl packs then
        # don't actually process them yet
        lpo="--exclude-no-idx --exclude-keep --exclude-bitmap --exclude-bndl --quiet"
        packcnt="$(list_packs --count $lpo objects/pack)" || :
        if [ "${packcnt:-0}" -ge 10 ]; then
                if [ -z "$miniactive" ]; then
                        miniactive=1
                        progress "+ [$proj] mini garbage check ($(date))"
                fi
                if [ -n "$gfi_mirror" ]; then
                        repack_gfi_packs
                        packcnt="$(list_packs --count $lpo objects/pack)" || :
                fi
                # if repack_gfi_packs dropped the pack count to < 10 don't combine
                if [ "${packcnt:-0}" -ge 10 ]; then
                        combine_small_packs
                        packcnt="$(list_packs --count $lpo objects/pack)" || :
                fi
                # if we still have more than 10 packs trigger a full gc
                if [ "${packcnt:-0}" -ge 10 ]; then
                        # We shouldn't be in a .delaygc state at this point, but if
                        # we are then nuke it because we really need a full gc now
                        rm -f .delaygc
                        git config --unset gitweb.lastgc
                        rm -f "$lockf"
                        git update-server-info # just in case
                        progress "- [$proj] mini garbage check triggering full gc too many packs ($(date))"
                        exit 0
                fi
        fi
        rm -f "$lockf"
        if [ -n "$miniactive" ]; then
                git update-server-info
                progress "- [$proj] mini garbage check ($(date))"
        else
                progress "= [$proj] mini garbage check nothing but crud removal to do ($(date))"
        fi
        exit 0
fi

# Avoid unnecessary garbage collections:
#   1. If lastreceive is set and is older than lastgc
#   -AND-
#   2. We are not a fork (! -s alternates) -OR- lastparentgc is older than lastgc

# If lastgc is NOT set or lastreceive is NOT set we MUST run gc
# If we are a fork and lastparentgc is NOT set we MUST run gc

# If the repo is dirty after removing any crud we MUST run gc

gcstart="$(date "$datefmt")"
skipgc=
isfork=
! [ -s objects/info/alternates ] || isfork=1
lastparentgcsecs=
[ -z "$isfork" ] || lastparentgcsecs="$(config_get_date_seconds lastparentgc)" || :
lastreceivesecs=
if lastreceivesecs="$(config_get_date_seconds lastreceive)" &&
   [ "${force_gc:-0}" = "0" ] &&
   lastgcsecs="$(config_get_date_seconds lastgc)" &&
   [ $lastreceivesecs -lt $lastgcsecs ]; then
        # We've run gc since we last received, so maybe we can skip,
        # check if not fork or fork and lastparentgc < lastgc
        if [ -n "$isfork" ]; then
                if [ -n "$lastparentgcsecs" ] &&
                   [ $lastparentgcsecs -lt $lastgcsecs ]; then
                        # We've run gc since our parent ran gc so we can skip
                        skipgc=1
                fi
        else
                # We don't have any alternates (we're not a forK) so we can skip
                skipgc=1
        fi
fi

# Prevent any other simultaneous gc operations
lock_gc

# At this point, if .allowgc or .gc_failed exists, it's now crud to be removed
rm -f .allowgc .gc_failed

# Ideally we would do this in post-receive, but that would mean duplicating the
# logic so it's available in the chroot jail and that's highly undesirable
# Instead, since the first gc will be triggered immediately following the first
# push, we do the check here as it's quick and harmless if HEAD is already valid
check_and_set_head || :

# Always get rid of crud
remove_crud

# Always perform reflogs maintenance
coalesce_reflogs
prune_reflogs
compact_reflogs

# Run 'git svn gc' now for svn mirrors
if [ -n "$svn_mirror" ]; then
        git svn gc || :
fi

# Skip the actual gc if .delaygc is set
if [ -e .delaygc ]; then
        progress "x [$proj] garbage check delayed (except for crud removal)"
        rm -f "$lockf"
        exit 0
fi

# Do not skip gc if the repo is dirty
if [ -n "$skipgc" ] && ! is_dirty; then
        progress "= [$proj] garbage check nothing but crud removal to do ($(date))"
        config_set lastgc "$gcstart"
        rm -f "$lockf"
        exit 0
fi

bumptime=
if [ -n "$isfork" ] && [ -z "$lastparentgcsecs" ]; then
        # set lastparentgc and then update gcstart to be at least 1 second later
        config_set lastparentgc "$gcstart"
        bumptime=1
fi
if [ -z "$lastreceivesecs" ]; then
        # set lastreceive and then update gcstart to be at least 1 second later
        config_set lastreceive "$gcstart"
        bumptime=1
fi
if [ -n "$bumptime" ]; then
        sleep 1
        gcstart="$(date "$datefmt")"
fi

progress "+ [$proj] garbage check ($(date))"

newdeltas=
[ -z "$alwaysredelta" ] || newdeltas="$noreusedeltaopt"
if [ -z "$newdeltas" ] && [ -n "$gfi_mirror" ]; then
        if [ $(list_packs --exclude-no-idx --count objects/pack) -le \
             $(list_packs --exclude-no-idx --count --quiet --only gfi-packs) ]; then
                # Don't bother with repack_gfi_packs since everything's being repacked
                newdeltas="--no-reuse-delta"
        fi
fi
if [ -z "$newdeltas" ] && [ -n "$noreusedeltaopt" ] &&
   [ $(list_packs --exclude-no-idx --count-objects objects/pack) -le $var_redelta_threshold ]; then
        # There aren't enough objects to worry about so just redelta to get the best pack
        newdeltas="--no-reuse-delta"
fi
if [ -z "$newdeltas" ]; then
        # Since we're not going to recompute deltas overall, we need to do the
        # "mini" maintenance so that we can get more optimal deltas
        [ -z "$noreusedeltaopt" ] || make_svn_pack
        repack_gfi_packs
        force_single_pack_redelta=
        [ -n "$gfi_mirror" ] || [ -n "$svn_mirror" ] || force_single_pack_redelta=1
        [ -z "$noreusedeltaopt" ] || combine_small_packs $force_single_pack_redelta
fi

#
## Safe Pruning In Forks
##
## We are about to perform garbage collection.  We do NOT use the "git gc" or
## the "git repack" commands directly as they do not provide enough control over
## the fine details.  However, we DO maintain a "gc.pid" file during our garbage
## collection so that a simultaneous "git gc" by an administrator will be
## blocked (and similarly we refuse to start garbage collection if we cannot
## create the "gc.pid" file).
##
## When we say "gc" in the below description we are referring to our "gc.sh"
## script, NOT the "git gc" command.
##
## If the project we are running garbage collection (gc) on has any forks we
## must be careful not to remove any objects that while no longer referenced by
## this project (the parent) are still referenced by one or more forks (the
## children) otherwise the children will become corrupt and we can't abide
## corrupt children.
##
## One way to accomplish this is to simply hard-link all currently existing
## loose objects and packs in the parent into all the children that refer to the
## parent (via a line in their objects/info/alternates file) before beginning
## the gc operation and then relying on a subsequent gc in the child to clean up
## any excess objects/packs.  We used to use this strategy but it's very
## inefficient because:
##
##   1. The disk space used by the old pack(s)/object(s) will not be reclaimed
##      until all children (and their children, if any) run gc by which time
##      it's quite possible the topmost parent will have run gc again and
##      hard-linked yet another old pack down to its children (not to mention
##      loose objects).
##
##   2. When using the "-A" option with "git repack", any new objects in the
##      parent that are not referenced by children will continually get
##      exploded out of the hard-linked pack in the children whenever the
##      children run gc.
##
##   3. To avoid suboptimal and/or unnecessarily many packs being hard-linked
##      into child forks, we must run the "mini" gc maintenance before we
##      perform the hard-linking into the children which provides yet another
##      source of inefficiency.
##
## While we were still using the "-A" option to "git repack" (that was not
## always the case) to guarantee we can access old ref values for long enough
## to send out a meaningful mail.sh notification, another, more efficient,
## option became available to prevent corruption of child forks that continue
## to refer to objects that are no longer reachable from any ref in the parent.
##
## The only things that need be copied (or hard-linked) into the child fork(s)
## are those objects that have become unreachable from any ref in the parent.
##
## When we were using the "git repack -A -d" + "git prune --expire=1.day.ago"
## technique, the only objects that could ever be removed were loose objects
## that "git prune" determined were expired.  In that case, loose objects were
## all that need be hard-linked down to child forks in order to avoid
## corruption of any child fork(s).
##
## The "git repack -A -d" + "git prune --expire=1.day.ago" + hard-linking loose
## objects to child forks technique remains fundamentally sound from the
## perspective of supporting simultaneous gc and push and keeping newly
## unreachable objects around long enough to be sure we can send out meaningful
## ref change notifications and never corrupting any child forks and never
## persisting the lifetime of large old packs containing mostly duplicate or
## unreachable objects as gc percolates through a project's entire fork tree.
##
## However, that technique suffers from one potential prodigious pitfall.
##
## Unreachable objects come flying out of their packs to splatter all over the
## objects subdirectories possibly creating a huge, inefficient mess.
##
## Often this is not an issue.  Even with a lot of rebasing going on, usually
## the only objects that will splatter are some commits, trees and the odd blob
## here and there.  Not enough to be overly concerned about.
##
## However, for the reppository that frequently experiences a lot of non-fast-
## forward updates and/or outright ref deletion, the number of objects suddenly
## popping out of their packs at "git repack -A -d" time can be overwhelming.
##
## To avoid this issue we now use a four phase pack creation strategy.
## This will result in creation of up to four packs (instead of at most one).
##
##   I. A complete pack (with bitmaps if appropriate) gets created including
##      only "reachable" objects from all refs/... refs plus HEAD.  This will
##      also serve as the virtual bundle for the repository.
##
##  II. A pack of recently-became-unreachable objects and friends is created.
##      (The "friends" are ref logs, linked working tree HEADs and indicies.)
##      Because both the pre-receive and update.sh script record all ref
##      changes we can easily choose the cut off point for "recently".
##      It is only the fact we maintain those logs in the reflogs subdirectory
##      that allows this step to be possible.
##
## III. If the repository has any forks with a non-zero length alternates file,
##      yet another pack of "--keep-unreachable" objects is generated that will
##      not actually be kept in the parent, but hard-linked into all the forks.
##
##  IV. Finally, after running "git prune-packed", any remaining loose objects
##      are migrated into a pack of their own.
##
## We then remove any non-.keep packs that existed before we started the
## process being careful to keep any same-pack pushes for the "Push Pack Redux"
## race condition (see README-GC).
##
## By using "git pack-objects" directly we are able to accomplish this with
## very little additional effort.
##
## The packs produced by (III) are treated almost like ".keep" packs by child
## forks in that the objects in them are never repacked into any other
## "--keep-unreachable" packs (but they can migrate into phase I or II packs)
## and those phase III packs are then hard-linked into any grandchild forks.
##
## This avoids the space explosion that could occur if each fork level ended
## up duplicating the "--keep-unreachable" pack space by repacking those
## objects (essentially breaking the hard-link to the single copy of those
## objects).
##
## While it is true that each level of forks could potentially add yet another
## phase III pack to be hard-linked down to its children, such packs will only
## include unreachable objects not already in any phase III packs that were
## received from the parent.
##
## The space for the phase III packs will not be reclaimed until the gc
## finishes percolating through the entire "fork tree" of a project.
##
## This is not much different than the "git repack -A -d" situation where
## all the loose objects are hard-linked down into child forks.  In that
## case forks that actually need any of those objects could gradually reduce
## the number of objects hard-linked into deeper fork levels.
##
## The difference with a phase III "--keep-unreachable" pack is that there
## cannot be any gradual reduction like that since it would require repacking
## the pack and breaking the hard-link thereby increasing storage space.  The
## storage will instead always be reclaimed all at once when all of the
## projects in the "fork tree" complete their gc.
##
## However, the belief is that the huge space win by having all the
## unreachable objects packed up together far eclipses (when many objects are
## involved, the single-pack version can end up using 1/20th or less of the
## disk space compared to having them all as loose objects) any brief minor
## space savings that might occur under the "git repack -A -d" loose object
## system prior to the gc collection completing for all the projects in the
## "fork tree".
#

#
## utility functions
#

# rename_pack oldnamepath newnamepath
# note that .keep files are left untouched and not moved at all!
rename_pack() {
        [ $# -eq 2 ] && [ "$1" != "$2" ] || {
                echo >&2 "[$proj] incorrect use of rename_pack function"
                exit 1
        }
        # Git assumes that if the destination of the rename already exists
        # that it is, in fact, a copy of the same bytes so silently succeeds
        # without doing anything.  We duplicate that logic here.
        # Git checks for the .idx file first before even trying to use a pack
        # so it should be the last moved and the first removed.
        for ext in pack bitmap idx; do
                [ -f "$1.$ext" ] || continue
                ln "$1.$ext" "$2.$ext" >/dev/null 2>&1 ||
                [ -f "$2.$ext" ] || {
                        echo >&2 "[$proj] unable to move $1.$ext to $2.$ext"
                        exit 1
                }
        done
        for ext in idx pack bitmap; do
                rm -f "$1.$ext"
        done
        return 0
}

make_packs_ugw() {
        find -L "$1" -maxdepth 1 -type f ! -perm -ug+w \
                -name "pack-$octet20*.pack" -exec chmod ug+w '{}' + || :
} 2>/dev/null

vcnt() {
        eval "$1="'$(( $# - 1 ))'
}

get_index_tree() {
        if [ -s "$1" ]; then
                GIT_INDEX_FILE="$1"
                export GIT_INDEX_FILE
                git write-tree 2>/dev/null || :
                unset GIT_INDEX_FILE
        fi
}

get_detached_head() {
        if [ -s "$1" ] && read -r _head <"$1" 2>/dev/null; then
                case "$_head" in $octet20*)
                        echo "$_head"
                esac
        fi
}

# compute_extra_reachables
# create lines suitable for a packed-refs file mentioning all the
# other refs we might like to keep.
# the current directory MUST be set to the repository's --git-dir
# the following are included:
#   * refs mentioned in reflogs/... files
#   * tree(s) created from index file(s)
#   * detached linked working tree heads
# Resulting objects are tested for existence and uniqified then output
# one per line under a refs/z* namespace
compute_extra_reachables() {
        {
                digits8='[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]'
                find -L reflogs -mindepth 1 -maxdepth 1 -type f -name "$digits8*" -exec gzip -c -d -f '{}' + |
                awk '{print $2; print $3}'
                ! [ -f index ] || get_index_tree index
                if [ -d worktrees ]; then
                        find -L worktrees -mindepth 2 -maxdepth 2 -name HEAD -type f -print |
                        while read -r lwth; do
                                get_detached_head "$lwth"
                                get_index_tree "${lwth%HEAD}index"
                        done
                fi
        } | LC_ALL=C sort -u |
        git cat-file ${var_have_git_260:+--buffer} --batch-check"${var_have_git_185:+=%(objectname)}" |
        awk '!/missing/ {num++; print $1 " " "refs/" substr("zzzzzzzzzzzz", 1, length(num)) "/" num}'
}

#
## main gc logic
#

# Everything else is more efficient if we do this first
# The "--prune" option is the default since v1.5.0 but it serves as "documentation" here
git pack-refs --all --prune
[ -e packed-refs ] || >>packed-refs # should never happen...

# If we have a logs directory or a worktrees directory expire the ref logs now
# Note that Git itself does not use either --rewrite or --updateref, so neither do we
! [ -d logs ] && ! [ -d worktrees ] || eval git reflog expire --all "${quiet:+>/dev/null 2>&1}" || :

make_repack_dir
! [ -e .gc_failed ] || exit 1
rm -f .gc_in_progress # make sure
touch .gc_in_progress # it's truly fresh
rm -f bundles/* objects/pack/pack-*.bndl
# This is perhaps a bit aggressive in that if we're suffering from "Push Pack Redux"
# and somehow we get run again immediately after the run where "Push Pack Redux" happened
# and we have garbage collection forced, there's just the barest, almost negligible,
# possibility that the "Push Pack Redux" ref updates _still_ have not happened and we
# should not be removing _r .keep files.  None of the normal Girocco processing can
# cause this.  The second run of this script would have to use the force gc option
# for it to even be possible in the first place.  What's much more likely is that
# the initial run of this script was somehow interrupted in the middle before it
# could get rid of the _r .keep file itself in which case it's better to get rid of
# it now to avoid keeping something around that would perturb our nice and neat gc
rm -f objects/pack/pack-*_r.keep
# We will add .keep files for _u packs if and when we run phase III
# Otherwise they need to not have any .keep files during phases I and II
rm -f objects/pack/pack-*_u.keep

# We need to make sure that any non-Girocco (barely tolerated) Git object creation
# activity will be able to "freshen" the pack containing a pre-existing object
# that's being written.  This really should not be necessary as the pre-receive
# hook should make sure this takes place for any incoming pushes.
# However, do it here anyway just in case.
make_packs_ugw objects/pack

# This is only effective with Git v2.3.5 and later and it will only matter when
# we are using one of the "internal_rev_list" modes of pack-objects
# (the combine-packs.sh script never uses any of those modes)
# The "git repack" and "git prune" commands always sets this internally themselves
# It makes no difference if there's no repository corruption
GIT_REF_PARANOIA=1 && export GIT_REF_PARANOIA

# All of the options we might want to use with pack-objects were supported
# at some point prior to Git version v1.6.6 which is the minimum version that
# Girocco now requires.  Except for one (--use-bitmap-index).  Several of them
# are "boiler plate" options we always want to use so we bundle them up here.
pkopt="--delta-base-offset --keep-true-parents --non-empty --all-progress-implied"
# We want to use --include-tag, but before Git v2.10.1 it would leave out
# "middle" tags (e.g. a tag of a tag of a commit would omit the tagged tag)
# See http://repo.or.cz/git.git/b773ddea2cd3b08c for details
# ("pack-objects: walk tag chains for --include-tag", 2016-09-07, v2.10.1)
# This is not a free check as it matches all refs against refs/tags/ then
# peels all the annotated tags and checks for inclusion.  The situation in
# which it would add a tag that was not already included by a reachability
# trace that included tag starting points can only occur if a new tag gets
# pushed during gc pointing to something that would have been packed anyway.
# But, it could happen and, really, campared to gc as a whole it's not that
# expensive to perform (provided we do not get an unconnected pack).
[ -z "$var_have_git_2101" ] || pkopt="$pkopt --include-tag"
pkopt="$pkopt ${quiet:---progress} $packopts"

# The git pack-objects command only supports bitmaps if all objects are being
# packed (the "--all" option) and the "--stdout" option is NOT being used.
# Additionally, while packing, if any encountered reachable objects are
# determined to be "not wanted" then no bitmap index will be written anyway.
# While it is theoretically possible that a project with a non-empty alternates
# file ends up packing all objects (because it does not actually use any of the
# objects found in the alternates), it's very unlikely.  And, in the unlikely
# event that did occur, clients would see a message about only using one bitmap
# because Git can only use one bitmap at a time and at least one of the
# alternates is bound to have a bitmap.  Therefore if we see a non-empty
# alternates file, we disable writing bitmaps which avoids the warning and any
# possibility of a client warning as well.  Also if we are running anything
# before Git v2.1.0 (the effective version for repack.writeBitmaps=true) then
# we also always disable bitmap writing.
wbmopt=
[ -z "$var_have_git_210" ] || wbmopt="--write-bitmap-index"
# More recent versions of pack-objects have optimizations when not using the
# --local option.  If we do not have any alternates it's a pointless option.
# If we do have alternates we need to skip writing a bitmap and we cannot
# have a bundle since it must contain all objects.
if [ -n "$isfork" ]; then
        lclopt="--local"
        wbmopt=
        makebndl=
else
        lclopt=
        makebndl=1
fi

#
## Phase I
#

wbmstr=
[ -n "$wbmopt" ] || wbmstr=" (bitmaps disabled)"
progress "~ [$proj] running primary full gc pack-objects$wbmstr"

gotforks=
! has_forks_with_alternates "$proj" || gotforks=1

# To avoid "Push Pack Redux" (see README-GC), after collecting the initial
# preexisting non-keep pack list, we rename them so that an incoming push
# pack cannot possibly experience a pack name collision.  Git does not require
# use of the "default" pack names, simply that the proper extensions are used.
# We rename to insert an "_r" just before the extension to avoid "Push Pack Redux"
# name collisions.  Later on we may create an "unreachable" pack for hard-linking
# down into forks and it will have an "_u" inserted just before its extension.
packlist="$(list_packs -C objects/pack --exclude-no-idx --exclude-keep --quiet .)" || :
oldpacks=
for oldpack in $packlist; do
        oldpack="${oldpack#pack-}"
        oldpack="${oldpack%.pack}"
        [ -f "objects/pack/pack-$oldpack.pack" ] || {
                echo >&2 "[$proj] unable to list old pack files"
                exit 1
        }
        if [ "${oldpack#*[!0-9a-fA-F]}" != "$oldpack" ]; then
                # names not exclusively hexadecimal do not need renaming
                oldpacks="${oldpacks:+$oldpacks }$oldpack"
                continue
        fi
        rename_pack "objects/pack/pack-$oldpack" "objects/pack/pack-${oldpack}_r" || {
                echo >&2 "[$proj] unable to rename old pack files"
                exit 1
        }
        # If the oldpack has a .keep now it means a "Push Pack Redux" is actually
        # in progress at this moment and we need to .keep the renamed pack,
        # otherwise no "Push Pack Redux" has started yet or it has already finished.
        # In either case we're okay because if it's just finished then all ref
        # changes have already been made so we don't need a .keep and we will
        # see the ref changes and grab all the objects via a reachability trace.
        # If it hasn't started yet that's okay because we're done moving that
        # name so a complete pack will appear under the old name that we'll
        # leave alone.
        if [ -f "objects/pack/pack-$oldpack.keep" ]; then
                echo "Push Pack Redux" >"objects/pack/pack-${oldpack}_r.keep"
        else
                oldpacks="${oldpacks:+$oldpacks }${oldpack}_r"
        fi
done

# We wish to keep deltas from our last full pack so if we're not redeltaing
# then make sure the .pack associated with the .bitmap has a newer mod time
# (If there is no .bitmap then touch the pack with the most objects instead.)
if [ -z "$newdeltas" ]; then
        bmpack="$(list_packs --exclude-no-bitmap --exclude-no-idx --max-matches 1 objects/pack)"
        [ -n "$bmpack" ] || bmpack="$(list_packs --exclude-no-idx --max-matches 1 --object-limit -1 --include-boundary objects/pack)"
        if [ -n "$bmpack" ] && [ -f "$bmpack" ] && [ -s "$bmpack" ]; then
                sleep 1
                touch -c "$bmpack" 2>/dev/null || :
                # We must touch .gc_in_progress here to avoid $bmpack looking
                # like it's been "freshened" when redundant packs are removed
                # It's okay if they have the same mod time, but POSIX does not
                # guarantee an ordering for the "touching" that occurs which is
                # why this must be a separate command but needs no "sleep 1"
                touch .gc_in_progress
        fi
fi

# Now we need to make sure that any "freshening" that takes place will actually
# result in a "newer" modification time than the .gc_in_progress file now has
sleep 1

# We run git pack-objects from the repack subdirectory so we can force
# optimized packs to be generated even for repositories that do not have any
# tagged commits
packs="$(git --git-dir=repack pack-objects </dev/null \
        $pkopt --all $newdeltas $lclopt ${wbmopt:---honor-pack-keep} $@ repack/alt/pack/pack)"
vcnt packcnt $packs
[ $packcnt -eq 1 ] || makebndl=

#
## Phase II
#

progress "~ [$proj] running supplementary gc pack-objects"

# Add the "supplementary" refs
compute_extra_reachables >>repack/packed-refs

# Subtract the primary refs
GIT_ALTERNATE_OBJECT_DIRECTORIES="$PWD/repack/alt"
export GIT_ALTERNATE_OBJECT_DIRECTORIES

# For this one we MUST use --local and MUST NOT use --write-bitmap-index
# However, if there is a "logs" subdirectory we need to use --reflog
# We do add it, just in case, if the linked working trees dir is present
# We do not add --indexed-objects as that requires v2.2.0 and it's unclear
# if it properly includes linked working tree index files or not.  The
# above compute_extra_reachables has already included all index trees (thereby
# providing proper --indexed-objects support for all Git versions) making the
# option completely unnecessary.
rflopt=
! [ -d logs ] && ! [ -d worktrees ] || rflopt=--reflog
spacks="$(git --git-dir=repack pack-objects </dev/null \
        $pkopt --honor-pack-keep --all $rflopt $newdeltas --local $@ repack/alt/pack/pack)"

#
## Phase III
#

# There's nothing to do for Phase III unless we have forks that refer to our
# project from their alternates file
hlpacks=
upacks=
if [ -n "$gotforks" ]; then

        progress "~ [$proj] running keep-unreachable gc pack-objects for forks"

        # If we are a fork, any pre-existing _u packs need to have a .keep
        # for this phase and be added to the hlpacks list otherwise (we are
        # not a fork) pre-existing _u packs are anomalies to be treated like
        # regular non-_u packs
        if [ -n "$isfork" ]; then
                for upack in $(find -L objects/pack -mindepth 1 -maxdepth 1 -name "pack-$octet20*_u.pack" -print); do
                        upack="${upack%.pack}"
                        [ -e "$upack.keep" ] || echo "unreachable" >"$upack.keep"
                        hlpacks="${hlpacks:+$hlpacks }${upack#objects/pack/pack-}"
                done
        fi
        # Using either --no-reuse-delta or --no-reuse-object together with the
        # --keep-unreachable option is a very, very, very bad idea when good
        # packs are the desired outcome.  If newdeltas are being generated
        # then we pack to a temp name, and use combine-packs.sh to get a better
        # pack as the result to avoid making a bad --keep-unreachable pack
        pfx=
        [ -z "$newdeltas" ] || pfx="ku"
        upacks="$(git --git-dir=repack pack-objects </dev/null \
                $pkopt --honor-pack-keep --all $rflopt --keep-unreachable --local $@ repack/alt/pack/${pfx}pack)"
        if [ -n "$upacks" ] && [ -n "$newdeltas" ]; then
                progress "~ [$proj] rebuilding keep-unreachable pack deltas"
                oldupacks="$upacks"
                upacks="$(
                        printf "repack/alt/pack/${pfx}pack-%s.pack\n" $oldupacks |
                        run_combine_packs --names --weak-naming --non-empty --all-progress-implied ${quiet:---progress} \
                                $packopts $newdeltas $@ repack/alt/pack/pack)"
                eval rm -f "$(printf \""repack/alt/pack/${pfx}pack-%s.*"\"" " $oldupacks)"
        fi
        for upack in $upacks; do
                rename_pack "repack/alt/pack/pack-$upack" "repack/alt/pack/pack-${upack}_u"
        done
        rm -f objects/pack/pack-*_u.keep
        [ -z "$hlpacks" ] && [ -z "$upacks" ] ||
        progress "~ [$proj] hard-linking keep-unreachable pack(s) into immediate child forks"

        # We have to update the lastparentgc time in the child forks even if they do not get any
        # new "unreachable packs" because they need to run gc just in case the parent now has some
        # objects that used to only be in the child so they can be removed from the child.
        # For example, a "patch" might be developed first in a fork and then later accepted into
        # the parent in which case the objects making up the patch in the child fork are now
        # redundant (since they're now in the parent as well) and need to be removed from the
        # child fork which can only happen if the child fork runs gc.
        lastparentgc="$(date "$datefmt")"

        # It is enough to copy objects just one level down and get_repo_list
        # takes a regular expression (which is automatically prefixed with '^')
        # so we can easily match forks exactly one level down from this project
        forkdir="$proj"
        get_repo_list "$forkdir/[^/:][^/:]*:" |
        while read fork; do
                # Ignore forks that do not exist or are symbolic links
                ! [ -L "$cfg_reporoot/$fork.git" ] && [ -d "$cfg_reporoot/$fork.git" ] ||
                        continue
                # Or do not have a non-zero length alternates file
                [ -s "$cfg_reporoot/$fork.git/objects/info/alternates" ] ||
                        continue
                runupdate=
                # Match hlpacks in parent project if any
                if [ -n "$hlpacks" ]; then
                        mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
                        eval ln -f "$(printf '"objects/pack/pack-%s.pack" ' $hlpacks)" \
                                "$(printf '"objects/pack/pack-%s.idx" ' $hlpacks)" \
                                '"$cfg_reporoot/$fork.git/objects/pack/"'
                        runupdate=1
                fi
                # Match upacks in repack/alt area if any
                if [ -n "$upacks" ]; then
                        mkdir -p "$cfg_reporoot/$fork.git/objects/pack"
                        eval ln -f "$(printf '"repack/alt/pack/pack-%s_u.pack" ' $upacks)" \
                                "$(printf '"repack/alt/pack/pack-%s_u.idx" ' $upacks)" \
                                '"$cfg_reporoot/$fork.git/objects/pack/"'
                        runupdate=1
                fi
                if ! [ -e "$cfg_reporoot/$fork.git/.needsgc" ]; then
                        # Trigger a mini gc in the fork if it now has too many packs
                        packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep "$cfg_reporoot/$fork.git/objects/pack")" || :
                        if [ -n "$packs" ] && [ "$packs" -ge 20 ]; then
                                >"$cfg_reporoot/$fork.git/.needsgc"
                        fi
                fi
                [ -z "$runupdate" ] || git --git-dir="$cfg_reporoot/$fork.git" update-server-info
                # Update the fork's lastparentgc date (must be more recent than $gcstart)
                git --git-dir="$cfg_reporoot/$fork.git" config gitweb.lastparentgc "$lastparentgc"
        done
fi

# Now move any primary/supplementary packs back into objects/pack
# then drop any "unfreshened" redundant packs and clear repack/alt

# First make sure the primary pack(s) have the most recent mod time
[ -z "$packs" ] || printf 'repack/alt/pack/pack-%s.pack\n' $packs | xargs touch -c 2>/dev/null || :

# Move the packs into place
for pack in $packs $spacks; do
        rename_pack "repack/alt/pack/pack-$pack" "objects/pack/pack-$pack"
done

# It's possible that one of the $oldpacks had a .bitmap, got renamed (along
# with its .bitmap) and then got "freshened" causing us to not remove it
# However, if $wbmopt is set we most likely now have TWO .bitmap packs!
# This can produce ugly warnings we don't want and possibly get the wrong
# bitmap used since only one .bitmap file can ever be used by Git.
# If this has happened, the .bitmap we want to discard will always have
# an _r infix so we can just zap any such now since it will leave the pack.
[ -z "$wbmopt" ] || rm -f objects/pack/pack-*_r.bitmap || :

# Remove the redundant packs that have not since been "freshened"
# This does not completely eliminate the race condition window (Girocco's own
# activites -- gc/fetch/receive are immune to the race) but it substantially
# shrinks it down to just the time after the find but before the following rm
>repack/oldpacks
[ -z "$oldpacks" ] ||
printf 'objects/pack/pack-%s.pack\n' $oldpacks |
LC_ALL=C sort >repack/oldpacks
find -L objects/pack -maxdepth 1 -type f -name "pack-$octet20*.pack" -newer .gc_in_progress -print |
LC_ALL=C sort >repack/freshened
deadpacks="$(LC_ALL=C join -v 1 repack/oldpacks repack/freshened | LC_ALL=C sed 's/\.pack$//')"
[ -z "$deadpacks" ] ||
eval echo "$(printf '"%s".* ' $deadpacks)" | xargs rm -f || :

# No need for this anymore
rm -rf repack/alt objects/pack/repack
unset GIT_ALTERNATE_OBJECT_DIRECTORIES

#
## Phase IV
#

progress "~ [$proj] running gc prune-packed"

# We do not want the redundant packs or any new "--keep-unreachable" pack(s) to be
# present while running prune-packed.  We try to guarantee that any loose object
# that's unreachable persists for at least one $Girocco::Config::min_gc_interval
# (not withstanding administrator interference to force earlier gc to occur).
# If we were to include the redundant/keep-unreachable pack(s) when running
# prune-packed and a loose unreachable object happened to be duplicated in one
# of them we would end up removing it too soon and void our guarantee.
git prune-packed $quiet

progress "~ [$proj] running loose objects gc pack-objects"

# Although Git v2.10.0 and later support a --pack-loose-unreachable option,
# we MUST NOT use it for these reasons:
#  1) We're not interested in expensive "unreachable" at this point, only "loose"
#  2) It produces simply horrid packs about 3.8x times larger than they should be
#  3) We don't require anything more than Git v1.6.6
lpacks="$(run_combine_packs </dev/null --names --loose --weak-naming --non-empty --all-progress-implied ${quiet:---progress} $packopts $newdeltas $@)"

if [ -n "$lpacks" ]; then
        # Make sure any primary pack(s) have a more recent mod time than "unreachable" objects packs
        if [ -n "$packs" ]; then
                sleep 1
                printf 'objects/pack/pack-%s.pack\n' $packs | xargs touch -c 2>/dev/null || :
        fi
        # We need to identify these packs later so we don't combine_packs them
        for objpack in $lpacks; do
                rename_pack "objects/pack/pack-$objpack" "objects/pack/pack-${objpack}_o" || :
        done
        # Finally zap the corresponding loose objects
        progress "~ [$proj] running packed loose objects gc prune-packed"
        git prune-packed $quiet
fi

! [ -e .gc_failed ] || exit 1
# These, if they exist, are now meaningless and need to be removed
rm -f gfi-packs .needsgc .svnpack .svnpackgc

# Make sure this stays up to date
git update-server-info

# We must make loose objects group writable so that they
# can be freshened by other pushers.  Technically we need only do this for
# push projects but to enable mirror projects to be more easily converted to
# push projects, we go ahead and do it for all projects.
# By the time we get here we really shouldn't have any of these, but just in case.
{ find -L objects/$octet -type f -name "$octet19*" -exec chmod ug+w '{}' + || :; } 2>/dev/null

# darcs:// mirrors have a xxx.log file that will grow endlessly
# if this is a mirror and the file exists, shorten it to 10000 lines
# also take this opportunity to optimize the darcs repo
if ! [ -e .nofetch ] && [ -n "$cfg_mirror" ]; then
        url="$(config_get baseurl)" || :
        case "$url" in darcs://*)
                if [ -n "$cfg_mirror_darcs" ]; then
                        url="${url%/}"
                        basedarcs="$(basename "${url#darcs:/}")"
                        if [ -f "$basedarcs.log" ]; then
                                tail -n 10000 "$basedarcs.log" >"$basedarcs.log.$$"
                                mv -f "$basedarcs.log.$$" "$basedarcs.log"
                        fi
                        if [ -d "$basedarcs.darcs" ]; then
                                (
                                        cd "$basedarcs.darcs"
                                        # without show_progress suppress non-error output
                                        [ -n "$show_progress" ] || exec >/dev/null
                                        # Note that this does not optimize _darcs/inventories/ :(
                                        darcs optimize || :
                                )
                        fi
                fi
        esac
fi

# Create a matching .bndl header file for the all-in-one pack we just created
# but only if we're not a fork (otherwise the bundle would not be complete)
# and we are running at least Git version 1.7.2 (pack_is_complete always fails otherwise)
if [ -n "$makebndl" ] && [ -n "$var_have_git_172" ]; then
        # There should only be one pack in $packs but do some checking...
        # The one we just created will have a .idx and will NOT have a .keep
        progress "~ [$proj] creating downloadble bundle header"
        pkbase=
        pkhead=
        IFS= read -r curhead <repack/HEAD.orig || :
        if
                [ -s "objects/pack/pack-$packs.pack" ] &&
                [ -s "objects/pack/pack-$packs.idx" ] &&
                ! [ -e "objects/pack/pack-$packs.keep" ] &&
                pkhead="$(pack_is_complete "$PWD/objects/pack/pack-$packs.pack" \
                        "$PWD/repack/packed-refs.orig" "$curhead")"
        then
                pkbase="objects/pack/pack-$packs"
        fi
        if [ -n "$pkbase" ] && [ -n "$pkhead" ]; then
                {
                        symref=
                        case "$curhead" in "ref: refs/"?*|"ref:refs/"?*|"refs/"?*)
                                symref="${curhead#ref:}"
                                symref="${symref# }"
                        esac
                        bndlurl=
                        [ -z "$cfg_httpbundleurl" ] || bndlurl=" url=$cfg_httpbundleurl/$proj.git/clone.bundle"
                        echo "# v2 git bundle"
                        LC_ALL=C sed -ne "/^$octet20$hexdig* refs\/[^ $tab]*\$/ p" <repack/packed-refs.orig
                        if [ -n "$symref" ]; then
                                printf "$pkhead HEAD\0symref=HEAD:%s%s\n" "$symref" "$bndlurl"
                        else
                                if [ -n "$bndlurl" ]; then
                                        printf "$pkhead HEAD\0%s\n" "${bndlurl:# }"
                                else
                                        echo "$pkhead HEAD"
                                fi
                        fi
                        echo ""
                } >"$pkbase.bndl"
                bndletag="$("$cfg_basedir/bin/rangecgi" --etag -m 1 "$pkbase.bndl" "$pkbase.pack")" || :
                bndlsha="$(printf '%s' "$bndletag" | git hash-object --stdin)" || :
                if [ -n "$bndletag" ]; then
                        case "$bndlsha" in $octet20*)
                                bndlshatrailer="${bndlsha#????????}"
                                bndlshaprefix="${bndlsha%$bndlshatrailer}"
                                bndlname="$(TZ=UTC date +%Y%m%d_%H%M%S)-${bndlshaprefix:-0}"
                                [ -d bundles ] || mkdir bundles
                                echo "${pkbase#objects/pack/}.bndl" >"bundles/$bndlname"
                                echo "${pkbase#objects/pack/}.pack" >>"bundles/$bndlname"
                                ln -s -f -n "$bndlname" bundles/latest
                        esac
                fi
        fi
fi

# Record the size of this repo as the sum of its clone packed-refs + *.pack sizes as 1024-byte blocks
eval "reposizek=$(( $(
        echo 0 $(du -k repack/packed-refs.orig $(printf 'objects/pack/pack-%s.pack ' $packs) 2>/dev/null |
                LC_ALL=C awk '{print $1}') |
        LC_ALL=C sed -e 's/ / + /g') ))"
config_set_raw girocco.reposizek "${reposizek:-0}"

# Now we're finally done with this
rm -rf repack

# We didn't used to do anything about rerere or worktrees but we're
# trying to make nice with linked working trees these days :)
# Maybe even non-bare repositories too, but *shush* about those ;)
if [ -n "$var_have_git_250" ] && [ -d worktrees ]; then
        # The value "3.months.ago" is hard-coded into gc.c rather than
        # having the default be in worktree.c so we must provide it if
        # we get nothing out of the gc.worktreePruneExpire config item
        # Prior to Git v2.6.0 the config item was gc.pruneworktreesexpire
        # however we just always use the newer name no matter what Git version
        expiry="$(git config --get gc.worktreePruneExpire 2>/dev/null)" || :
        eval git worktree prune --expire '"${expiry:-3.months.ago}"' "${quiet:+>/dev/null 2>&1}" || :
fi
# git rerere does it right and handles its own default/config'd expiration values
! [ -d rr-cache ] || eval git rerere gc "${quiet:+>/dev/null 2>&1}" || :

# We use $gcstart here to avoid a race where a push occurs during the gc itself
# and the next future gc could be incorrectly skipped if we used the current
# timestamp here instead
config_set lastgc "$gcstart"
rm -f "$lockf"

progress "- [$proj] garbage check ($(date))"