3 # combine-packs.sh -- combine Git pack files
4 # Copyright (C) 2016 Kyle J. McKay. All rights reserved
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
22 printf "%s\n" path-to-pack[.idx|.pack] ... |
23 combine-packs [option]... [pack-objects option]... [pack-base-name]
25 NOTE: The following options MUST be given before any pack-objects options:
27 --replace on success, remove the input packs, see note below
28 (but any input packs with a .keep are never removed)
30 --names output the 40-char hex sha1 plus "\n" to stdout for each
31 newly created pack(s), if any
33 --ignore-missing silently ignore input pack file names that do not exist
35 --objects input is a list of object hash id values instead of packs
37 If --replace is given, ALL packs to be combined MUST be located in
38 the objects/pack subdirectory of the current git directory AND the output
39 pack base MUST also be omitted (meaning it defaults to objects/pack/pack).
41 Note that if --objects is used then --replace and --ignore-missing are invalid
42 and any missing input objects are always silently ignored.
44 A 40-char hex sha1 is taken to be objects/pack/pack-<sha-1>.idx relative to
45 the current git directory (as output by `git rev-parse --git-dir`).
47 If a <pack-name> does not exist and contains no "/" characters then it is
48 retried as objects/pack/<pack-name> instead.
50 Packs to be combined MUST have an associated .idx file.
52 The pack-base-name may be a relative path name and if so, is ALWAYS relative
53 to the current git directory.
55 If not given, then the pack-base-name defaults to objects/pack/pack
56 relative to the current git directory.
58 Note that --delta-base-offset is ALWAYS passed to git pack-objects but it is
59 the ONLY option that is automatically passed (but remember that --reuse-delta
60 and --reuse-object are IMPLIED and must be explicitly disabled if desired).
62 The options --revs, --unpacked, --all, --reflog, --indexed-objects and
63 --stdout are forbidden. Although --keep-true-parents is allowed it should
64 not have any effect at all. Using --incremental is recommended only for
65 wizards or with --objects as in most other cases it will result in an empty
68 WARNING: the move_aside logic currently only works when pack-base-name is
74 # $$ should be the same in subshells, but just in case, remember it
79 #line 80 "combine-packs.sh"
86 while ($count >= 32768) {
87 read(STDIN, $x, 32768);
90 read(STDIN, $x, $count) if $count;
96 if (/^([0-9a-fA-F]+) ([^ ]+) ([0-9]+)$/) {
97 my ($h, $t, $l) = ($1, $2, $3);
100 discard(1 + $l), next unless $2 eq "tag";
103 $count += length($_);
106 $tn = $1 if /^tag ([^ ]+)$/;
107 $te = $1 if /^tagger [^>]+> ([0-9]+)/;
110 discard(1 + $l - $count);
111 push(@tags, [$te, "$h $tn\n"]);
114 print map($$_[1], sort({$$b[0] <=> $$a[0]} @tags));
117 # On some broken platforms running xargs without -r and empty input runs the command
118 xargs_r
="$(: | command xargs echo -r)"
120 # Some platforms' broken xargs runs the command always at least once even if
121 # there's no input unless given a special option. Automatically supply the
122 # option on those platforms by providing an xargs function.
123 xargs() { command xargs $xargs_r "$@"; }
130 [ -n "$td" ] && [ -e "$td/success" ] || ewf
=1
131 [ -z "$td" ] ||
! [ -e "$td" ] ||
rm -rf "$td" ||
:
132 [ -z "$gd" -o -z "$zap" ] ||
command find "$gd/objects/pack" -maxdepth 1 -type f
-name "*.$zap" -print0 |
xargs -0 rm -f ||
:
133 [ -z "$ewf" ] ||
echo "combine_packs: exiting with failure" >&2 ||
:
136 trap cleanup_on_exit EXIT
143 echo "combine-packs: fatal: $*" >&2 ||
:
144 # In case we are in a sub shell force the entire command to exit
145 # The trap on TERM will make sure cleanup still happens in this case
147 [ -z "$td" ] ||
[ ! -s "$td/popid" ] || extrapid
=$
(cat "$td/popid" ||
:)
148 kill $cp_pid $extrapid ||
:
152 # This extra indirection shouldn't be necessary, but it is for some broken sh
153 # in order for a failure to not prematurely exit die_on_fail with set -e active
155 LC_ALL
=C
exec command "$@"
159 if ! do_command
"$@"; then
160 [ -z "$td" ] ||
>"$td/failed" ||
:
161 die
"failed command: $*"
165 # These commands may be the non-final member of a pipe and
166 # MUST NOT be allowed to silently fail without consequence
167 awk() { die_on_fail
awk "$@"; }
168 cat() { die_on_fail
cat "$@"; }
169 cut
() { die_on_fail cut
"$@"; }
170 find() { die_on_fail
find "$@"; }
171 git
() { die_on_fail git
"$@"; }
172 join() { die_on_fail
join "$@"; }
173 perl
() { die_on_fail perl
"$@"; }
174 sed() { die_on_fail
sed "$@"; }
175 sort() { die_on_fail
sort "$@"; }
177 octet
='[0-9a-f][0-9a-f]'
178 octet4
="$octet$octet$octet$octet"
179 octet20
="$octet4$octet4$octet4$octet4$octet4"
186 while [ $# -ge 1 ]; do case "$1" in
200 printf '%s' "${USAGE#?}"
212 [ -z "$ignoremiss$zap" -o -z "$objectlist" ] || die
"invalid options"
214 gd
="$(git rev-parse --git-dir)"
215 gd
="$(cd "$gd" && pwd -P)" || die
"cd failed: $gd"
216 [ -d "$gd/objects/pack" ] || die
"no such directory: $gd/objects/pack"
228 --replace|
--names|
--ignore-missing|
-h|
--help|
--objects)
229 die
"invalid options"
231 --revs|
--unpacked|
--all|
--reflog|
--indexed-objects)
232 die
"forbidden pack-objects options"
239 nonopts
=$
(( $nonopts + 1 ))
242 if [ $# -gt 0 ] && [ $nonopts -gt 1 ] ||
[ $nonopts -eq 1 -a -n "$lastargopt" ] || \
243 [ $nonopts -eq 1 -a -z "$lastarg" ]; then
244 die
"invalid options"
246 if [ $nonopts -eq 1 ]; then
249 packbase
="$gd/objects/pack/pack"
251 pbd
="$(dirname "$packbase")"
252 [ -e "$pbd" -a -d "$pbd" ] || die
"no such directory: $packbase"
253 packbase
="$(cd "$
(dirname "$packbase")" && pwd -P)/$(basename "$packbase")"
254 pbd
="$(dirname "$packbase")"
255 [ -e "$pbd" -a -d "$pbd" ] || die
"internal failure realpathing: $packbase"
256 case "$packbase" in "$gd"/?
*)
257 packbase
="${packbase#$gd/}"
259 [ $nonopts -eq 1 ] || packbasearg
="$packbase"
260 [ -z "$zap" -o -n "$packbasearg" ] || die
"--replace does not allow specifying pack-base"
261 if [ -n "$zap" ] && [ "$(dirname "$packbase")" != "objects/pack" ]; then
262 die
"--replace and pack base dir not <git-dir>/objects/pack" >&2
265 td
="$(mktemp -d "$gd/cmbnpcks-XXXXXX
")"
266 tdmin
="$(basename "$td")"
271 success
="$td/success"
276 trbl
="$tdmin/treesblobs"
283 _name
="$gd/objects/pack/pack-$_name"
286 _name
="${_name%.idx}"
289 _name
="${_name%.pack}"
292 if ! [ -e "$_name.idx" -o -e "$_name.pack" ]; then
293 case "$_name" in */*) :;; *)
294 _name
="$gd/objects/pack/$_name"
297 if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
298 [ -z "$ignoremiss" ] ||
return 0
299 die
"no such pack found matching: $1" >&2
301 _name
="$(cd "$
(dirname "$_name")" && pwd -P)/$(basename "$_name")"
302 if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
303 die
"internal failure realpathing: $1" >&2
305 case "$(dirname "$_name")" in "$gd"/?
*)
306 _name
="${_name#$gd/}"
308 if [ -n "$zap" ] && [ "$(dirname "$_name")" != "objects/pack" ]; then
309 die
"--replace and pack not in <git-dir>/objects/pack: $1" >&2
315 # add "old" prefix to passed in existing files, but be careful to hard-link
316 # ALL the files to be renamed to the renamed name BEFORE removing anything
320 ln -f "$_f" "$(dirname "$_f")/old$(basename "$_f")"
323 if [ -f "$_f" ]; then
336 if [ -n "$objectlist" ]; then
337 git cat-file
--batch-check='%(objectname) %(objecttype)'
339 [ -z "$zap" ] ||
find objects
/pack
-maxdepth 1 -type f
-name "*.$zap" -print0 |
xargs -0 rm -f ||
:
340 while IFS
=': ' read -r packraw junk
; do
341 pack
="$(cd "$origdir" && get_pack_base "$packraw" || die "no such pack
: $packraw")"
342 if [ -n "$pack" ]; then
343 [ -z "$zap" ] ||
[ -e "$pack.keep" ] ||
>"$pack.$zap"
344 git show-index
<"$pack.idx"
346 done | cut
-d ' ' -f 2 |
347 git cat-file
--batch-check='%(objectname) %(objecttype)'
349 if ($2=="tree") print $1
350 else if ($2=="blob") print $1 >"'"$bl"'"
351 else if ($2=="commit") print $1 >"'"$cm"'"
352 else if ($2=="tag") print $1 >"'"$tg"'"
354 cat "$tr" "$bl" |
sort -u >"$trbl"
355 git rev-list
--no-walk --objects --stdin <"$cm" |
356 awk '{print NR " " $0}' |
358 join -t " " -1 2 - "$trbl" >"$named"
359 pocmd
='git pack-objects --delta-base-offset "$@"'
360 [ -z "$packbasearg" ] || pocmd
="$pocmd \"${packbasearg}tmp\""
363 ! [ -s "$tg" ] || git cat-file
--batch <"$tg" | perl
-e "$perlprog"
364 sort -k2,2n
<"$named" |
365 sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/'
366 join -t " " -v 1 "$tr" "$named" |
367 git rev-list
--no-walk --objects --stdin
371 sh
-c 'echo $$ >"$1"; pocmd="$2"; shift; shift; eval "exec $pocmd"' sh
"$popid" "$pocmd" "$@" ||
{
373 die
"git pack-objects failed"
378 while read -r newpack
; do
379 if [ -n "$packbasearg" ]; then
380 move_aside
"$packbasearg"-$newpack.
*
381 ln -f "${packbasearg}tmp"-$newpack.pack
"$packbasearg"-$newpack.pack
382 ln -f "${packbasearg}tmp"-$newpack.idx
"$packbasearg"-$newpack.idx
383 rm -f "${packbasearg}tmp"-$newpack.
*
385 [ -z "$names" ] ||
echo "$newpack"
387 [ $?
-eq 0 -a ! -e "$failed" -a -e "$listok" -a -e "$packok" ] || die
"unspecified failure"
388 if [ -n "$zap" ]; then
389 find objects
/pack
-maxdepth 1 -type f
-name "*.$zap" -print |
390 while read -r remove
; do
391 rm -f "${remove%.$zap}".
*