3 # combine-packs.sh -- combine Git pack files
4 # Copyright (C) 2016 Kyle J. McKay. All rights reserved
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
22 printf "%s\n" path-to-pack[.idx|.pack] ... |
23 combine-packs [option]... [pack-objects option]... [pack-base-name]
25 NOTE: The following options MUST be given before any pack-objects options:
27 --replace on success, remove the input packs, see note below
28 (but any input packs with a .keep are never removed)
30 --names output the 40-char hex sha1 plus "\n" to stdout for each
31 newly created pack(s), if any
33 --ignore-missing silently ignore input pack file names that do not exist
35 --ignore-missing-objects
36 silently ignore missing objects (explicit objects when
37 using --objects otherwise those contained in input packs)
39 --objects input is a list of object hash id values instead of packs
41 --envok allow use of GIT_OBJECT_DIRECTORY otherwise it is an error
42 to run combine-packs.sh with GIT_OBJECT_DIRECTORY set
44 If --replace is given, ALL packs to be combined MUST be located in
45 the objects/pack subdirectory of the current git directory AND the output
46 pack base MUST also be omitted (meaning it defaults to objects/pack/pack).
48 Note that if --objects is used then --replace and --ignore-missing are invalid.
50 Unless --ignore-missing-objects is given, any input objects (either given
51 explicitly when using --objects otherwise those contained in the input packs)
52 that are not present in the current git directory (respecting the value of
53 GIT_OBJECT_DIRECTORY if --envok is given) or its alternate object
54 directories, if any, will cause combine-packs to fail.
55 With this option any such objects are SILENTLY SKIPPED and do NOT appear in
58 A 40-char hex sha1 is taken to be objects/pack/pack-<sha-1>.idx relative to
59 the current git directory (as output by `git rev-parse --git-dir`).
61 If a <pack-name> does not exist and contains no "/" characters then it is
62 retried as objects/pack/<pack-name> instead.
64 Packs to be combined MUST have an associated .idx file.
66 The pack-base-name may be a relative path name and if so, is ALWAYS relative
67 to the current git directory regardless of any GIT_OBJECT_DIRECTORY setting.
69 If not given, then the pack-base-name defaults to objects/pack/pack
70 relative to the current git directory.
72 If GIT_OBJECT_DIRECTORY is set to a non-default location (and the --envok flag
73 is given to allow it) then everywhere above where it says "objects/" is
74 effectively replaced with the full absolute path to "$GIT_OBJECT_DIRECTORY/".
75 And, obviously, that location is no longer necessarily a subdirectory of the
76 current git directory either.
78 Note that --delta-base-offset is ALWAYS passed to git pack-objects but it is
79 the ONLY option that is automatically passed (but remember that --reuse-delta
80 and --reuse-object are IMPLIED and must be explicitly disabled if desired).
82 The options --revs, --unpacked, --all, --reflog, --indexed-objects and
83 --stdout are forbidden. Although --keep-true-parents is allowed it should
84 not have any effect at all. Using --incremental is recommended only for
85 wizards or with --objects as in most other cases it will result in an empty
88 WARNING: the move_aside logic currently only works when pack-base-name is
94 # $$ should be the same in subshells, but just in case, remember it
99 #line 100 "combine-packs.sh"
106 while ($count >= 32768) {
107 read(STDIN, $x, 32768);
110 read(STDIN, $x, $count) if $count;
116 if (/^([0-9a-fA-F]+) ([^ ]+) ([0-9]+)$/) {
117 my ($h, $t, $l) = ($1, $2, $3);
120 discard(1 + $l), next unless $2 eq "tag";
123 $count += length($_);
126 $tn = $1 if /^tag ([^ ]+)$/;
127 $te = $1 if /^tagger [^>]+> ([0-9]+)/;
130 discard(1 + $l - $count);
131 push(@tags, [$te, "$h $tn\n"]);
134 print map($$_[1], sort({$$b[0] <=> $$a[0]} @tags));
137 # On some broken platforms running xargs without -r and empty input runs the command
138 xargs_r
="$(: | command xargs echo -r)"
140 # Some platforms' broken xargs runs the command always at least once even if
141 # there's no input unless given a special option. Automatically supply the
142 # option on those platforms by providing an xargs function.
143 xargs() { command xargs $xargs_r "$@"; }
152 [ -n "$td" ] && [ -e "$td/success" ] || ewf
=1
153 [ -z "$td" ] ||
! [ -e "$td" ] ||
rm -rf "$td" ||
:
154 [ -z "$gdo" -o -z "$zap" ] ||
command find "$gdo/pack" -maxdepth 1 -type f
-name "*.$zap" -print0 |
xargs -0 rm -f ||
:
155 [ -z "$ewf" ] ||
echo "combine_packs: exiting with failure" >&2 ||
:
158 trap cleanup_on_exit EXIT
165 echo "combine-packs: fatal: $*" >&2 ||
:
166 # In case we are in a sub shell force the entire command to exit
167 # The trap on TERM will make sure cleanup still happens in this case
169 [ -z "$td" ] ||
[ ! -s "$td/popid" ] || extrapid
=$
(cat "$td/popid" ||
:)
170 kill $cp_pid $extrapid ||
:
174 # This extra indirection shouldn't be necessary, but it is for some broken sh
175 # in order for a failure to not prematurely exit die_on_fail with set -e active
177 # some shells do not handle "exec command ..." properly but just a
178 # plain "exec ..." has the same semantics so "command" is omitted here
185 [ -z "$td" ] ||
>"$td/failed" ||
:
186 die
"failed command ($_ec): $*"
190 # These commands may be the non-final member of a pipe and
191 # MUST NOT be allowed to silently fail without consequence
192 awk() { die_on_fail
awk "$@"; }
193 cat() { die_on_fail
cat "$@"; }
194 cut
() { die_on_fail cut
"$@"; }
195 find() { die_on_fail
find "$@"; }
196 git
() { die_on_fail git
"$@"; }
197 join() { die_on_fail
join "$@"; }
198 perl
() { die_on_fail perl
"$@"; }
199 sed() { die_on_fail
sed "$@"; }
200 sort() { die_on_fail
sort "$@"; }
202 octet
='[0-9a-f][0-9a-f]'
203 octet4
="$octet$octet$octet$octet"
204 octet20
="$octet4$octet4$octet4$octet4$octet4"
213 while [ $# -ge 1 ]; do case "$1" in
226 --ignore-missing-objects)
231 printf '%s' "${USAGE#?}"
247 [ -z "$ignoremiss$dozap" -o -z "$objectlist" ] || die
"invalid options"
249 # Always make sure we get the specified objects
250 GIT_NO_REPLACE_OBJECTS
=1
251 export GIT_NO_REPLACE_OBJECTS
252 gd
="$(git rev-parse --git-dir)"
254 gv
="${gv#[Gg]it version }"
256 IFS
=.
read -r gvmaj gvmin gvpat
<<EOT
259 : ${gvmaj:=0} ${gvmin:=0} ${gvpat:=0}
260 # gcfbo is Git Cat-File --Buffer Option :)
262 if [ $gvmaj -gt 2 ] ||
[ $gvmaj -eq 2 -a $gvmin -ge 6 ]; then
266 gd
="$(cd "$gd" && pwd -P)" || die
"cd failed: $tmp"
267 if [ "${GIT_OBJECT_DIRECTORY+set}" = "set" ] && [ -z "$envok" ]; then
268 # GIT_OBJECT_DIRECTORY may only be set to $gd/objects without --envok
270 if [ -n "$GIT_OBJECT_DIRECTORY" ] && [ -d "$GIT_OBJECT_DIRECTORY" ] && \
271 [ -d "$gd/objects" ] && godfp
="$(cd "$GIT_OBJECT_DIRECTORY" && pwd -P)" && \
272 gdofp
="$(cd "$gd/objects
" && pwd -P)" && [ -n "$godfp" ] && [ -n "$gdofp" ] && \
273 [ "$gdofp" = "$godfp" ]; then
276 if [ -z "$godok" ]; then
277 die
"GIT_OBJECT_DIRECTORY set to non-default location without --envok"
280 gdo
="${GIT_OBJECT_DIRECTORY:-$gd/objects}"
282 gdo
="$(cd "$gdo" && pwd -P)" || die
"cd failed: $tmp"
283 [ -d "$gdo/pack" ] || die
"no such directory: $gdo/pack"
295 --replace|
--names|
--ignore-missing|
-h|
--help|
--objects)
296 die
"invalid options"
298 --revs|
--unpacked|
--all|
--reflog|
--indexed-objects)
299 die
"forbidden pack-objects options"
306 nonopts
=$
(( $nonopts + 1 ))
309 if [ $# -gt 0 ] && [ $nonopts -gt 1 ] ||
[ $nonopts -eq 1 -a -n "$lastargopt" ] || \
310 [ $nonopts -eq 1 -a -z "$lastarg" ]; then
311 die
"invalid options"
313 if [ $nonopts -eq 1 ]; then
316 packbase
="$gdo/pack/pack"
318 pbd
="$(dirname "$packbase")"
319 [ -e "$pbd" -a -d "$pbd" ] || die
"no such directory: $packbase"
320 packbase
="$(cd "$
(dirname "$packbase")" && pwd -P)/$(basename "$packbase")"
321 pbd
="$(dirname "$packbase")"
322 [ -e "$pbd" -a -d "$pbd" ] || die
"internal failure realpathing: $packbase"
323 packbasecheck
="$packbase"
324 case "$packbase" in "$gd"/?
*)
325 packbase
="${packbase#$gd/}"
327 [ $nonopts -eq 1 ] || packbasearg
="$packbase"
328 [ -z "$zap" -o -n "$packbasearg" ] || die
"--replace does not allow specifying pack-base"
329 if [ -n "$zap" ] && [ "$(dirname "$packbasecheck")" != "$gdo/pack" ] ; then
330 die
"--replace and pack base dir not <git-dir-objects>/pack" >&2
333 td
="$(mktemp -d "$gd/cmbnpcks-XXXXXX
")"
334 tdmin
="$(basename "$td")"
339 success
="$td/success"
346 trbl
="$tdmin/treesblobs"
353 _name
="$gdo/pack/pack-$_name"
356 _name
="${_name%.idx}"
359 _name
="${_name%.pack}"
362 if ! [ -e "$_name.idx" -o -e "$_name.pack" ]; then
363 case "$_name" in */*) :;; *)
364 _name
="$gdo/pack/$_name"
367 if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
368 [ -z "$ignoremiss" ] ||
return 0
369 die
"no such pack found matching: $1" >&2
371 _name
="$(cd "$
(dirname "$_name")" && pwd -P)/$(basename "$_name")"
372 if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
373 die
"internal failure realpathing: $1" >&2
376 case "$(dirname "$_name")" in "$gd"/?
*)
377 _name
="${_name#$gd/}"
379 if [ -n "$zap" ] && [ "$(dirname "$_namecheck")" != "$gdo/pack" ]; then
380 die
"--replace and pack not in <git-dir-objects>/pack: $1" >&2
386 # add "old" prefix to passed in existing files, but be careful to hard-link
387 # ALL the files to be renamed to the renamed name BEFORE removing anything
391 ln -f "$_f" "$(dirname "$_f")/old$(basename "$_f")"
394 if [ -f "$_f" ]; then
408 if [ -n "$objectlist" ]; then
409 git cat-file
$gcfbo --batch-check='%(objectname) %(objecttype)'
411 [ -z "$zap" ] ||
find "$gdo/pack" -maxdepth 1 -type f
-name "*.$zap" -print0 |
xargs -0 rm -f ||
:
412 while IFS
=': ' read -r packraw junk
; do
413 pack
="$(cd "$origdir" && get_pack_base "$packraw" || die "no such pack
: $packraw")"
414 if [ -n "$pack" ]; then
415 [ -z "$zap" ] ||
[ -e "$pack.keep" ] ||
>"$pack.$zap"
416 git show-index
<"$pack.idx"
418 done | cut
-d ' ' -f 2 |
419 git cat-file
$gcfbo --batch-check='%(objectname) %(objecttype)'
421 if ($2=="tree") print $1
422 else if ($2=="blob") print $1 >"'"$bl"'"
423 else if ($2=="commit") print $1 >"'"$cm"'"
424 else if ($2=="tag") print $1 >"'"$tg"'"
425 else if ($2=="missing") print $1 >"'"$ms"'"
427 [ -n "$missok" ] ||
! [ -s "$ms" ] || die
"missing" $
(wc -l <"$ms") "object(s)"
428 echo "g" |
cat "$tr" "$bl" - |
sort -u >"$trbl"
429 git rev-list
--no-walk --objects --stdin <"$cm" |
431 if ($1!=$0) print NR " " $0
432 else print $0 >"'"$cmo"'"
435 join -t " " -1 2 - "$trbl" >"$named"
436 pocmd
='git pack-objects --delta-base-offset "$@"'
437 [ -z "$packbasearg" ] || pocmd
="$pocmd \"${packbasearg}tmp\""
440 ! [ -s "$tg" ] || git cat-file
$gcfbo --batch <"$tg" | perl
-e "$perlprog"
441 sort -t " " -k2,2n
<"$named" |
442 sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/'
443 join -t " " -v 1 "$tr" "$named" |
444 git rev-list
--no-walk --objects --stdin |
445 awk '{print NR " " $0}' |
447 join -t " " -1 2 - "$trbl" |
449 sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/'
453 sh
-c 'echo $$ >"$1"; pocmd="$2"; shift; shift; eval "exec $pocmd"' sh
"$popid" "$pocmd" "$@" ||
{
455 die
"git pack-objects failed"
460 while read -r newpack
; do
461 if [ -n "$packbasearg" ]; then
462 move_aside
"$packbasearg"-$newpack.
*
463 ln -f "${packbasearg}tmp"-$newpack.pack
"$packbasearg"-$newpack.pack
464 ln -f "${packbasearg}tmp"-$newpack.idx
"$packbasearg"-$newpack.idx
465 rm -f "${packbasearg}tmp"-$newpack.
*
467 [ -z "$names" ] ||
echo "$newpack"
469 [ $?
-eq 0 -a ! -e "$failed" -a -e "$listok" -a -e "$packok" ] || die
"unspecified failure"
470 if [ -n "$zap" ]; then
471 find "$gdo/pack" -maxdepth 1 -type f
-name "*.$zap" -print |
472 while read -r remove
; do
473 rm -f "${remove%.$zap}".
*