jobd/combine-packs.sh

   1 #!/bin/sh
   2
   3 # combine-packs.sh -- combine Git pack files
   4 # Copyright (C) 2016 Kyle J. McKay.  All rights reserved
   5
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 # Version 1.1.15
  20
  21 USAGE='
  22 printf "%s\n" path-to-pack[.idx|.pack] ... |
  23 combine-packs [option]... [pack-objects option]... [pack-base-name]
  24
  25 NOTE: The following options MUST be given before any pack-objects options:
  26
  27   --replace         on success, remove the input packs, see note below
  28                     (but any input packs with a .keep are never removed)
  29
  30   --names           output the 40-char hex sha1 plus "\n" to stdout for each
  31                     newly created pack(s), if any
  32
  33   --ignore-missing  silently ignore input pack file names that do not exist
  34
  35   --ignore-missing-objects
  36                     silently ignore missing objects (explicit objects when
  37                     using --objects otherwise those contained in input packs)
  38
  39   --objects         input is a list of object hash id values instead of packs
  40
  41   --envok           allow use of GIT_OBJECT_DIRECTORY otherwise it is an error
  42                     to run combine-packs.sh with GIT_OBJECT_DIRECTORY set
  43
  44 If --replace is given, ALL packs to be combined MUST be located in
  45   the objects/pack subdirectory of the current git directory AND the output
  46   pack base MUST also be omitted (meaning it defaults to objects/pack/pack).
  47
  48 Note that if --objects is used then --replace and --ignore-missing are invalid.
  49
  50 Unless --ignore-missing-objects is given, any input objects (either given
  51   explicitly when using --objects otherwise those contained in the input packs)
  52   that are not present in the current git directory (respecting the value of
  53   GIT_OBJECT_DIRECTORY if --envok is given) or its alternate object
  54   directories, if any, will cause combine-packs to fail.
  55   With this option any such objects are SILENTLY SKIPPED and do NOT appear in
  56   the output pack(s)!
  57
  58 A 40-char hex sha1 is taken to be objects/pack/pack-<sha-1>.idx relative to
  59   the current git directory (as output by `git rev-parse --git-dir`).
  60
  61 If a <pack-name> does not exist and contains no "/" characters then it is
  62   retried as objects/pack/<pack-name> instead.
  63
  64 Packs to be combined MUST have an associated .idx file.
  65
  66 The pack-base-name may be a relative path name and if so, is ALWAYS relative
  67   to the current git directory regardless of any GIT_OBJECT_DIRECTORY setting.
  68
  69 If not given, then the pack-base-name defaults to objects/pack/pack
  70   relative to the current git directory.
  71
  72 If GIT_OBJECT_DIRECTORY is set to a non-default location (and the --envok flag
  73   is given to allow it) then everywhere above where it says "objects/" is
  74   effectively replaced with the full absolute path to "$GIT_OBJECT_DIRECTORY/".
  75   And, obviously, that location is no longer necessarily a subdirectory of the
  76   current git directory either.
  77
  78 Note that --delta-base-offset is ALWAYS passed to git pack-objects but it is
  79   the ONLY option that is automatically passed (but remember that --reuse-delta
  80   and --reuse-object are IMPLIED and must be explicitly disabled if desired).
  81
  82 The options --revs, --unpacked, --all, --reflog, --indexed-objects and
  83   --stdout are forbidden.  Although --keep-true-parents is allowed it should
  84   not have any effect at all.  Using --incremental is recommended only for
  85   wizards or with --objects as in most other cases it will result in an empty
  86   pack being output.
  87
  88 WARNING: the move_aside logic currently only works when pack-base-name is
  89          completely omitted!
  90 '
  91
  92 set -e
  93
  94 # $$ should be the same in subshells, but just in case, remember it
  95 cp_pid=$$
  96
  97 perlprog='
  98 #!/usr/bin/perl
  99 #line 100 "combine-packs.sh"
 100 use strict;
 101 use warnings;
 102
 103 sub discard {
 104   my $count = shift;
 105   my $x = "";
 106   while ($count >= 32768) {
 107     read(STDIN, $x, 32768);
 108     $count -= 32768;
 109   }
 110   read(STDIN, $x, $count) if $count;
 111 }
 112
 113 my @tags = ();
 114 binmode STDIN;
 115 while (<STDIN>) {
 116   if (/^([0-9a-fA-F]+) ([^ ]+) ([0-9]+)$/) {
 117     my ($h, $t, $l) = ($1, $2, $3);
 118     my $te = 0;
 119     my $tn = "";
 120     discard(1 + $l), next unless $2 eq "tag";
 121     my $count = 0;
 122     while (<STDIN>) {
 123       $count += length($_);
 124       chomp;
 125       last if /^$/;
 126       $tn = $1 if /^tag ([^ ]+)$/;
 127       $te = $1 if /^tagger [^>]+> ([0-9]+)/;
 128       last if $tn && $te;
 129     }
 130     discard(1 + $l - $count);
 131     push(@tags, [$te, "$h $tn\n"]);
 132   }
 133 }
 134 print map($$_[1], sort({$$b[0] <=> $$a[0]} @tags));
 135 '
 136
 137 # On some broken platforms running xargs without -r and empty input runs the command
 138 xargs_r="$(: | command xargs echo -r)"
 139
 140 # Some platforms' broken xargs runs the command always at least once even if
 141 # there's no input unless given a special option.  Automatically supply the
 142 # option on those platforms by providing an xargs function.
 143 xargs() { command xargs $xargs_r "$@"; }
 144
 145 td=
 146 zap=
 147 gd=
 148 gdo=
 149
 150 cleanup_on_exit() {
 151         ewf=
 152         [ -n "$td" ] && [ -e "$td/success" ] || ewf=1
 153         [ -z "$td" ] || ! [ -e "$td" ] || rm -rf "$td" || :
 154         [ -z "$gdo" -o -z "$zap" ] || command find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || :
 155         [ -z "$ewf" ] || echo "combine_packs: exiting with failure" >&2 || :
 156 }
 157
 158 trap cleanup_on_exit EXIT
 159 trap 'exit 129' HUP
 160 trap 'exit 130' INT
 161 trap 'exit 131' QUIT
 162 trap 'exit 143' TERM
 163
 164 die() {
 165         echo "combine-packs: fatal: $*" >&2 || :
 166         # In case we are in a sub shell force the entire command to exit
 167         # The trap on TERM will make sure cleanup still happens in this case
 168         extrapid=
 169         [ -z "$td" ] || [ ! -s "$td/popid" ] || extrapid=$(cat "$td/popid" || :)
 170         kill $cp_pid $extrapid || :
 171         exit 1
 172 }
 173
 174 # This extra indirection shouldn't be necessary, but it is for some broken sh
 175 # in order for a failure to not prematurely exit die_on_fail with set -e active
 176 do_command() (
 177         # some shells do not handle "exec command ..." properly but just a
 178         # plain "exec ..." has the same semantics so "command" is omitted here
 179         LC_ALL=C exec "$@"
 180 )
 181
 182 die_on_fail() {
 183         do_command "$@" || {
 184                 _ec=$?
 185                 [ -z "$td" ] || >"$td/failed" || :
 186                 die "failed command ($_ec): $*"
 187         }
 188 }
 189
 190 # These commands may be the non-final member of a pipe and
 191 # MUST NOT be allowed to silently fail without consequence
 192 awk()  { die_on_fail awk  "$@"; }
 193 cat()  { die_on_fail cat  "$@"; }
 194 cut()  { die_on_fail cut  "$@"; }
 195 find() { die_on_fail find "$@"; }
 196 git()  { die_on_fail git  "$@"; }
 197 join() { die_on_fail join "$@"; }
 198 perl() { die_on_fail perl "$@"; }
 199 sed()  { die_on_fail sed  "$@"; }
 200 sort() { die_on_fail sort "$@"; }
 201
 202 octet='[0-9a-f][0-9a-f]'
 203 octet4="$octet$octet$octet$octet"
 204 octet20="$octet4$octet4$octet4$octet4$octet4"
 205
 206 names=
 207 ignoremiss=
 208 objectlist=
 209 dozap=
 210 envok=
 211 missok=
 212
 213 while [ $# -ge 1 ]; do case "$1" in
 214         --names)
 215                 names=1
 216                 shift
 217                 ;;
 218         --replace)
 219                 dozap="zap-$$"
 220                 shift
 221                 ;;
 222         --ignore-missing)
 223                 ignoremiss=1
 224                 shift
 225                 ;;
 226         --ignore-missing-objects)
 227                 missok=1
 228                 shift
 229                 ;;
 230         -h|--help)
 231                 printf '%s' "${USAGE#?}"
 232                 trap - EXIT
 233                 exit 0
 234                 ;;
 235         --objects)
 236                 objectlist=1
 237                 shift
 238                 ;;
 239         --envok)
 240                 envok=1
 241                 shift
 242                 ;;
 243         *)
 244                 break
 245                 ;;
 246 esac; done
 247 [ -z "$ignoremiss$dozap" -o -z "$objectlist" ] || die "invalid options"
 248
 249 # Always make sure we get the specified objects
 250 GIT_NO_REPLACE_OBJECTS=1
 251 export GIT_NO_REPLACE_OBJECTS
 252 gd="$(git rev-parse --git-dir)"
 253 gv="$(git version)"
 254 gv="${gv#[Gg]it version }"
 255 gv="${gv%%[!0-9.]*}"
 256 IFS=. read -r gvmaj gvmin gvpat <<EOT
 257 $gv
 258 EOT
 259 : ${gvmaj:=0} ${gvmin:=0} ${gvpat:=0}
 260 # gcfbo is Git Cat-File --Buffer Option :)
 261 gcfbo=
 262 if [ $gvmaj -gt 2 ] || [ $gvmaj -eq 2 -a $gvmin -ge 6 ]; then
 263         gcfbo=--buffer
 264 fi
 265 tmp="$gd"
 266 gd="$(cd "$gd" && pwd -P)" || die "cd failed: $tmp"
 267 if [ "${GIT_OBJECT_DIRECTORY+set}" = "set" ] && [ -z "$envok" ]; then
 268         # GIT_OBJECT_DIRECTORY may only be set to $gd/objects without --envok
 269         godok=
 270         if [ -n "$GIT_OBJECT_DIRECTORY" ] && [ -d "$GIT_OBJECT_DIRECTORY" ] && \
 271            [ -d "$gd/objects" ] && godfp="$(cd "$GIT_OBJECT_DIRECTORY" && pwd -P)" && \
 272            gdofp="$(cd "$gd/objects" && pwd -P)" && [ -n "$godfp" ] && [ -n "$gdofp" ] && \
 273            [ "$gdofp" = "$godfp" ]; then
 274                 godok=1
 275         fi
 276         if [ -z "$godok" ]; then
 277                 die "GIT_OBJECT_DIRECTORY set to non-default location without --envok"
 278         fi
 279 fi
 280 gdo="${GIT_OBJECT_DIRECTORY:-$gd/objects}"
 281 tmp="$gdo"
 282 gdo="$(cd "$gdo" && pwd -P)" || die "cd failed: $tmp"
 283 [ -d "$gdo/pack" ] || die "no such directory: $gdo/pack"
 284 zap="$dozap"
 285
 286 lastarg=
 287 lastargopt=
 288 packbase=
 289 packbasearg=
 290 nonopts=0
 291 for arg; do
 292         lastarg="$arg"
 293         lastargopt=1
 294         case "$arg" in
 295         --replace|--names|--ignore-missing|-h|--help|--objects)
 296                 die "invalid options"
 297                 ;;
 298         --revs|--unpacked|--all|--reflog|--indexed-objects)
 299                 die "forbidden pack-objects options"
 300                 ;;
 301         -*)
 302                 :
 303                 ;;
 304         *)
 305                 lastargopt=
 306                 nonopts=$(( $nonopts + 1 ))
 307         esac
 308 done
 309 if [ $# -gt 0 ] && [ $nonopts -gt 1 ] || [ $nonopts -eq 1 -a -n "$lastargopt" ] || \
 310    [ $nonopts -eq 1 -a -z "$lastarg" ]; then
 311         die "invalid options"
 312 fi
 313 if [ $nonopts -eq 1 ]; then
 314         packbase="$lastarg"
 315 else
 316         packbase="$gdo/pack/pack"
 317 fi
 318 pbd="$(dirname "$packbase")"
 319 [ -e "$pbd" -a -d "$pbd" ] || die "no such directory: $packbase"
 320 packbase="$(cd "$(dirname "$packbase")" && pwd -P)/$(basename "$packbase")"
 321 pbd="$(dirname "$packbase")"
 322 [ -e "$pbd" -a -d "$pbd" ] || die "internal failure realpathing: $packbase"
 323 packbasecheck="$packbase"
 324 case "$packbase" in "$gd"/?*)
 325         packbase="${packbase#$gd/}"
 326 esac
 327 [ $nonopts -eq 1 ] || packbasearg="$packbase"
 328 [ -z "$zap" -o -n "$packbasearg" ] || die "--replace does not allow specifying pack-base"
 329 if [ -n "$zap" ] && [ "$(dirname "$packbasecheck")" != "$gdo/pack" ] ; then
 330         die "--replace and pack base dir not <git-dir-objects>/pack" >&2
 331 fi
 332
 333 td="$(mktemp -d "$gd/cmbnpcks-XXXXXX")"
 334 tdmin="$(basename "$td")"
 335 failed="$td/failed"
 336 listok="$td/listok"
 337 packok="$td/packok"
 338 popid="$td/popid"
 339 success="$td/success"
 340 cm="$tdmin/commits"
 341 cmo="$tdmin/ordered"
 342 tg="$tdmin/tags"
 343 tr="$tdmin/trees"
 344 bl="$tdmin/blobs"
 345 ms="$tdmin/missing"
 346 trbl="$tdmin/treesblobs"
 347 named="$tdmin/named"
 348 named2="$tdmin/named2"
 349
 350 get_pack_base() {
 351         _name="$1"
 352         case "$_name" in
 353         $octet20)
 354                 _name="$gdo/pack/pack-$_name"
 355                 ;;
 356         *.idx)
 357                 _name="${_name%.idx}"
 358                 ;;
 359         *.pack)
 360                 _name="${_name%.pack}"
 361                 ;;
 362         esac
 363         if ! [ -e "$_name.idx" -o -e "$_name.pack" ]; then
 364                 case "$_name" in */*) :;; *)
 365                         _name="$gdo/pack/$_name"
 366                 esac
 367         fi
 368         if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
 369                 [ -z "$ignoremiss" ] || return 0
 370                 die "no such pack found matching: $1" >&2
 371         fi
 372         _name="$(cd "$(dirname "$_name")" && pwd -P)/$(basename "$_name")"
 373         if ! [ -f "$_name.idx" -a -s "$_name.idx" -a -f "$_name.pack" -a -s "$_name.pack" ]; then
 374                 die "internal failure realpathing: $1" >&2
 375         fi
 376         _namecheck="$_name"
 377         case "$(dirname "$_name")" in "$gd"/?*)
 378                 _name="${_name#$gd/}"
 379         esac
 380         if [ -n "$zap" ] && [ "$(dirname "$_namecheck")" != "$gdo/pack" ]; then
 381                 die "--replace and pack not in <git-dir-objects>/pack: $1" >&2
 382         fi
 383         echo "$_name"
 384         return 0
 385 }
 386
 387 # add "old" prefix to passed in existing files, but be careful to hard-link
 388 # ALL the files to be renamed to the renamed name BEFORE removing anything
 389 move_aside() {
 390         for _f; do
 391                 ! [ -f "$_f" ] || \
 392                 ln -f "$_f" "$(dirname "$_f")/old$(basename "$_f")"
 393         done
 394         for _f; do
 395                 if [ -f "$_f" ]; then
 396                         rm -f "$_f"
 397                         ! test -f "$_f"
 398                 fi
 399         done
 400         return 0
 401 }
 402
 403 origdir="$PWD"
 404 cd "$gd"
 405 >"$cm"
 406 >"$cmo"
 407 >"$tr"
 408 >"$bl"
 409 if [ -n "$objectlist" ]; then
 410         git cat-file $gcfbo --batch-check='%(objectname) %(objecttype)'
 411 else
 412         [ -z "$zap" ] || find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print0 | xargs -0 rm -f || :
 413         while IFS=': ' read -r packraw junk; do
 414                 pack="$(cd "$origdir" && get_pack_base "$packraw" || die "no such pack: $packraw")"
 415                 if [ -n "$pack" ]; then
 416                         [ -z "$zap" ] || [ -e "$pack.keep" ] || >"$pack.$zap"
 417                         git show-index <"$pack.idx"
 418                 fi
 419         done | cut -d ' ' -f 2 |
 420         git cat-file $gcfbo --batch-check='%(objectname) %(objecttype)'
 421 fi | awk '{
 422   if ($2=="tree") print $1
 423   else if ($2=="blob") print $1 >"'"$bl"'"
 424   else if ($2=="commit") print $1 >"'"$cm"'"
 425   else if ($2=="tag") print $1 >"'"$tg"'"
 426   else if ($2=="missing") print $1 >"'"$ms"'"
 427 }' | sort -u >"$tr"
 428 [ -n "$missok" ] || ! [ -s "$ms" ] || die "missing" $(wc -l <"$ms") "object(s)"
 429 echo "g" | cat "$tr" "$bl" - | sort -u >"$trbl"
 430 git rev-list --no-walk --objects --stdin <"$cm" |
 431 awk '{
 432   if ($1!=$0) print NR " " $0
 433   else print $0 >"'"$cmo"'"
 434 }' |
 435 sort -t " " -k2,2 |
 436 join -t " " -1 2 - "$trbl" >"$named"
 437 join -t " " -v 1 "$tr" "$named" |
 438 git rev-list --no-walk --objects --stdin |
 439 awk '{print NR " " $0}' |
 440 sort -t " " -k2,2 |
 441 join -t " " -1 2 - "$trbl" >"$named2"
 442 pocmd='git pack-objects --delta-base-offset "$@"'
 443 [ -z "$packbasearg" ] || pocmd="$pocmd \"${packbasearg}tmp\""
 444 {
 445         cat "$cmo"
 446         ! [ -s "$tg" ] || git cat-file $gcfbo --batch <"$tg" | perl -e "$perlprog"
 447         {
 448                 join -t " " "$named" "$tr" |
 449                 sort -t " " -k2,2n
 450                 join -t " " "$named2" "$tr" |
 451                 sort -t " " -k2,2n
 452         } | sed -e 's/\([^ ][^ ]*\) [^ ][^ ]*/\1/'
 453         {
 454                 join -t " " -v 1 "$named" "$tr" |
 455                 sort -t " " -k2,2n
 456                 join -t " " -v 1 "$named2" "$tr" |
 457                 sort -t " " -k2,2n
 458         } | awk -F '[ ]' '{
 459                 if (NF >= 3) {
 460                         nm = substr($0, length($1) + length($2) + 3)
 461                         sfx = nm
 462                         gsub(/[\t\n\013\f\r ]+/, "", sfx)
 463                         if (length(sfx)) {
 464                                 if (length(sfx) > 16) sfx = substr(sfx, length(sfx) - 15)
 465                                 else if (length(sfx) < 16) sfx = sprintf("%16s", sfx)
 466                                 split(sfx, c, "")
 467                                 r = c[16] c[15] c[14] c[13] c[12] c[11] c[10] c[9] c[8] c[7] c[6] c[5] c[4] c[3] c[2] c[1]
 468                                 sub(/[ ]+$/, "", r)
 469                                 print NR " " $1 " " r " " nm
 470                         } else print NR " " $1 "  " nm
 471                 } else print NR " " $1 "  "
 472         }' | sort -t " " -k3,3 -k1,1n | awk -F '[ ]' '{
 473                 if (NF >= 4) {
 474                         nm = substr($0, length($1) + length($2) + length($3) + 4)
 475                         print $2 " " nm
 476                 } else print $2 " "
 477         }'
 478         sort -u "$bl"
 479         >"$listok"
 480 } | {
 481         sh -c 'echo $$ >"$1"; pocmd="$2"; shift; shift; eval "exec $pocmd"' sh "$popid" "$pocmd" "$@" || {
 482                 rm -f "$popid"
 483                 die "git pack-objects failed"
 484         }
 485         rm -f "$popid"
 486         >"$packok"
 487 } |
 488 while read -r newpack; do
 489         if [ -n "$packbasearg" ]; then
 490                 move_aside "$packbasearg"-$newpack.*
 491                 ln -f "${packbasearg}tmp"-$newpack.pack "$packbasearg"-$newpack.pack
 492                 ln -f "${packbasearg}tmp"-$newpack.idx "$packbasearg"-$newpack.idx
 493                 rm -f "${packbasearg}tmp"-$newpack.*
 494         fi
 495         [ -z "$names" ] || echo "$newpack"
 496 done
 497 [ $? -eq 0 -a ! -e "$failed" -a -e "$listok" -a -e "$packok" ] || die "unspecified failure"
 498 if [ -n "$zap" ]; then
 499         find "$gdo/pack" -maxdepth 1 -type f -name "*.$zap" -print |
 500         while read -r remove; do
 501                 rm -f "${remove%.$zap}".*
 502         done
 503 fi
 504 >"$success"