From d190e3d480381be599261eda4ddb8581c41cd824 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Thu, 14 Jul 2016 09:33:58 -0700 Subject: [PATCH] combine-packs.sh: update to latest Exits with failure if any input objects are not present in the current git directory without the new --ignore-missing-objects option. Supports an objects subdirectory that is a symlink to the real location. Supports the GIT_OBJECT_DIRECTORY environment variable (but only if the new --envok option is given). Shows the exit status of any failed command. Uses -t " " with sort to correctly match join usage. Avoids SIGPIPE on sort output piped to join -. Suppresses unwanted extraneous objects when isolated trees are present. Orders commits properly rather than using input order. Avoids "exec command ..." as some shells screw that up. Signed-off-by: Kyle J. McKay --- jobd/combine-packs.sh | 121 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 94 insertions(+), 27 deletions(-) diff --git a/jobd/combine-packs.sh b/jobd/combine-packs.sh index 92e34cf..a0321e8 100755 --- a/jobd/combine-packs.sh +++ b/jobd/combine-packs.sh @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Version 1.1.8 +# Version 1.1.13 USAGE=' printf "%s\n" path-to-pack[.idx|.pack] ... | @@ -32,14 +32,28 @@ NOTE: The following options MUST be given before any pack-objects options: --ignore-missing silently ignore input pack file names that do not exist + --ignore-missing-objects + silently ignore missing objects (explicit objects when + using --objects otherwise those contained in input packs) + --objects input is a list of object hash id values instead of packs + --envok allow use of GIT_OBJECT_DIRECTORY otherwise it is an error + to run combine-packs.sh with GIT_OBJECT_DIRECTORY set + If --replace is given, ALL packs to be combined MUST be located in the objects/pack subdirectory of the current git directory AND the output pack base MUST also be omitted (meaning it defaults to objects/pack/pack). -Note that if --objects is used then --replace and --ignore-missing are invalid - and any missing input objects are always silently ignored. +Note that if --objects is used then --replace and --ignore-missing are invalid. + +Unless --ignore-missing-objects is given, any input objects (either given + explicitly when using --objects otherwise those contained in the input packs) + that are not present in the current git directory (respecting the value of + GIT_OBJECT_DIRECTORY if --envok is given) or its alternate object + directories, if any, will cause combine-packs to fail. + With this option any such objects are SILENTLY SKIPPED and do NOT appear in + the output pack(s)! A 40-char hex sha1 is taken to be objects/pack/pack-.idx relative to the current git directory (as output by `git rev-parse --git-dir`). @@ -50,11 +64,17 @@ If a does not exist and contains no "/" characters then it is Packs to be combined MUST have an associated .idx file. The pack-base-name may be a relative path name and if so, is ALWAYS relative - to the current git directory. + to the current git directory regardless of any GIT_OBJECT_DIRECTORY setting. If not given, then the pack-base-name defaults to objects/pack/pack relative to the current git directory. +If GIT_OBJECT_DIRECTORY is set to a non-default location (and the --envok flag + is given to allow it) then everywhere above where it says "objects/" is + effectively replaced with the full absolute path to "$GIT_OBJECT_DIRECTORY/". + And, obviously, that location is no longer necessarily a subdirectory of the + current git directory either. + Note that --delta-base-offset is ALWAYS passed to git pack-objects but it is the ONLY option that is automatically passed (but remember that --reuse-delta and --reuse-object are IMPLIED and must be explicitly disabled if desired). @@ -76,7 +96,7 @@ cp_pid=$$ perlprog=' #!/usr/bin/perl -#line 80 "combine-packs.sh" +#line 100 "combine-packs.sh" use strict; use warnings; @@ -124,12 +144,14 @@ xargs() { command xargs $xargs_r "$@"; } td= zap= +gd= +gdo= cleanup_on_exit() { ewf= [ -n "$td" ] && [ -e "$td/success" ] || ewf=1 [ -z "$td" ] || ! [ -e "$td" ] || rm -rf "$td" || : - [ -z "$gd" -o -z "$zap" ] || command find "$gd/objects/pack" -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || : + [ -z "$gdo" -o -z "$zap" ] || command find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || : [ -z "$ewf" ] || echo "combine_packs: exiting with failure" >&2 || : } @@ -152,14 +174,17 @@ die() { # This extra indirection shouldn't be necessary, but it is for some broken sh # in order for a failure to not prematurely exit die_on_fail with set -e active do_command() ( - LC_ALL=C exec command "$@" + # some shells do not handle "exec command ..." properly but just a + # plain "exec ..." has the same semantics so "command" is omitted here + LC_ALL=C exec "$@" ) die_on_fail() { - if ! do_command "$@"; then + do_command "$@" || { + _ec=$? [ -z "$td" ] || >"$td/failed" || : - die "failed command: $*" - fi + die "failed command ($_ec): $*" + } } # These commands may be the non-final member of a pipe and @@ -182,6 +207,8 @@ names= ignoremiss= objectlist= dozap= +envok= +missok= while [ $# -ge 1 ]; do case "$1" in --names) @@ -196,6 +223,10 @@ while [ $# -ge 1 ]; do case "$1" in ignoremiss=1 shift ;; + --ignore-missing-objects) + missok=1 + shift + ;; -h|--help) printf '%s' "${USAGE#?}" trap - EXIT @@ -205,6 +236,10 @@ while [ $# -ge 1 ]; do case "$1" in objectlist=1 shift ;; + --envok) + envok=1 + shift + ;; *) break ;; @@ -227,8 +262,25 @@ gcfbo= if [ $gvmaj -gt 2 ] || [ $gvmaj -eq 2 -a $gvmin -ge 6 ]; then gcfbo=--buffer fi -gd="$(cd "$gd" && pwd -P)" || die "cd failed: $gd" -[ -d "$gd/objects/pack" ] || die "no such directory: $gd/objects/pack" +tmp="$gd" +gd="$(cd "$gd" && pwd -P)" || die "cd failed: $tmp" +if [ "${GIT_OBJECT_DIRECTORY+set}" = "set" ] && [ -z "$envok" ]; then + # GIT_OBJECT_DIRECTORY may only be set to $gd/objects without --envok + godok= + if [ -n "$GIT_OBJECT_DIRECTORY" ] && [ -d "$GIT_OBJECT_DIRECTORY" ] && \ + [ -d "$gd/objects" ] && godfp="$(cd "$GIT_OBJECT_DIRECTORY" && pwd -P)" && \ + gdofp="$(cd "$gd/objects" && pwd -P)" && [ -n "$godfp" ] && [ -n "$gdofp" ] && \ + [ "$gdofp" = "$godfp" ]; then + godok=1 + fi + if [ -z "$godok" ]; then + die "GIT_OBJECT_DIRECTORY set to non-default location without --envok" + fi +fi +gdo="${GIT_OBJECT_DIRECTORY:-$gd/objects}" +tmp="$gdo" +gdo="$(cd "$gdo" && pwd -P)" || die "cd failed: $tmp" +[ -d "$gdo/pack" ] || die "no such directory: $gdo/pack" zap="$dozap" lastarg= @@ -261,20 +313,21 @@ fi if [ $nonopts -eq 1 ]; then packbase="$lastarg" else - packbase="$gd/objects/pack/pack" + packbase="$gdo/pack/pack" fi pbd="$(dirname "$packbase")" [ -e "$pbd" -a -d "$pbd" ] || die "no such directory: $packbase" packbase="$(cd "$(dirname "$packbase")" && pwd -P)/$(basename "$packbase")" pbd="$(dirname "$packbase")" [ -e "$pbd" -a -d "$pbd" ] || die "internal failure realpathing: $packbase" +packbasecheck="$packbase" case "$packbase" in "$gd"/?*) packbase="${packbase#$gd/}" esac [ $nonopts -eq 1 ] || packbasearg="$packbase" [ -z "$zap" -o -n "$packbasearg" ] || die "--replace does not allow specifying pack-base" -if [ -n "$zap" ] && [ "$(dirname "$packbase")" != "objects/pack" ]; then - die "--replace and pack base dir not /objects/pack" >&2 +if [ -n "$zap" ] && [ "$(dirname "$packbasecheck")" != "$gdo/pack" ] ; then + die "--replace and pack base dir not /pack" >&2 fi td="$(mktemp -d "$gd/cmbnpcks-XXXXXX")" @@ -285,9 +338,11 @@ packok="$td/packok" popid="$td/popid" success="$td/success" cm="$tdmin/commits" +cmo="$tdmin/ordered" tg="$tdmin/tags" tr="$tdmin/trees" bl="$tdmin/blobs" +ms="$tdmin/missing" trbl="$tdmin/treesblobs" named="$tdmin/named" @@ -295,7 +350,7 @@ get_pack_base() { _name="$1" case "$_name" in $octet20) - _name="$gd/objects/pack/pack-$_name" + _name="$gdo/pack/pack-$_name" ;; *.idx) _name="${_name%.idx}" @@ -306,7 +361,7 @@ get_pack_base() { esac if ! [ -e "$_name.idx" -o -e "$_name.pack" ]; then case "$_name" in */*) :;; *) - _name="$gd/objects/pack/$_name" + _name="$gdo/pack/$_name" esac fi if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then @@ -317,11 +372,12 @@ get_pack_base() { if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then die "internal failure realpathing: $1" >&2 fi + _namecheck="$_name" case "$(dirname "$_name")" in "$gd"/?*) _name="${_name#$gd/}" esac - if [ -n "$zap" ] && [ "$(dirname "$_name")" != "objects/pack" ]; then - die "--replace and pack not in /objects/pack: $1" >&2 + if [ -n "$zap" ] && [ "$(dirname "$_namecheck")" != "$gdo/pack" ]; then + die "--replace and pack not in /pack: $1" >&2 fi echo "$_name" return 0 @@ -346,12 +402,13 @@ move_aside() { origdir="$PWD" cd "$gd" >"$cm" +>"$cmo" >"$tr" >"$bl" if [ -n "$objectlist" ]; then git cat-file $gcfbo --batch-check='%(objectname) %(objecttype)' else - [ -z "$zap" ] || find objects/pack -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || : + [ -z "$zap" ] || find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || : while IFS=': ' read -r packraw junk; do pack="$(cd "$origdir" && get_pack_base "$packraw" || die "no such pack: $packraw")" if [ -n "$pack" ]; then @@ -365,21 +422,31 @@ fi | awk '{ else if ($2=="blob") print $1 >"'"$bl"'" else if ($2=="commit") print $1 >"'"$cm"'" else if ($2=="tag") print $1 >"'"$tg"'" + else if ($2=="missing") print $1 >"'"$ms"'" }' | sort -u >"$tr" -cat "$tr" "$bl" | sort -u >"$trbl" +[ -n "$missok" ] || ! [ -s "$ms" ] || die "missing" $(wc -l <"$ms") "object(s)" +echo "g" | cat "$tr" "$bl" - | sort -u >"$trbl" git rev-list --no-walk --objects --stdin <"$cm" | -awk '{print NR " " $0}' | -sort -k2,2 | +awk '{ + if ($1!=$0) print NR " " $0 + else print $0 >"'"$cmo"'" +}' | +sort -t " " -k2,2 | join -t " " -1 2 - "$trbl" >"$named" pocmd='git pack-objects --delta-base-offset "$@"' [ -z "$packbasearg" ] || pocmd="$pocmd \"${packbasearg}tmp\"" { - cat "$cm" + cat "$cmo" ! [ -s "$tg" ] || git cat-file $gcfbo --batch <"$tg" | perl -e "$perlprog" - sort -k2,2n <"$named" | + sort -t " " -k2,2n <"$named" | sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/' join -t " " -v 1 "$tr" "$named" | - git rev-list --no-walk --objects --stdin + git rev-list --no-walk --objects --stdin | + awk '{print NR " " $0}' | + sort -t " " -k2,2 | + join -t " " -1 2 - "$trbl" | + sort -t " " -k2,2n | + sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/' cat "$bl" >"$listok" } | { @@ -401,7 +468,7 @@ while read -r newpack; do done [ $? -eq 0 -a ! -e "$failed" -a -e "$listok" -a -e "$packok" ] || die "unspecified failure" if [ -n "$zap" ]; then - find objects/pack -maxdepth 1 -type f -name "*.$zap" -print | + find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print | while read -r remove; do rm -f "${remove%.$zap}".* done -- 2.11.4.GIT