t/t5300-pack-object.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2005 Junio C Hamano
   4 #
   5
   6 test_description='git pack-object
   7
   8 '
   9 . ./test-lib.sh
  10
  11 test_expect_success 'setup' '
  12         rm -f .git/index* &&
  13         perl -e "print \"a\" x 4096;" >a &&
  14         perl -e "print \"b\" x 4096;" >b &&
  15         perl -e "print \"c\" x 4096;" >c &&
  16         test-tool genrandom "seed a" 2097152 >a_big &&
  17         test-tool genrandom "seed b" 2097152 >b_big &&
  18         git update-index --add a a_big b b_big c &&
  19         cat c >d && echo foo >>d && git update-index --add d &&
  20         tree=$(git write-tree) &&
  21         commit=$(git commit-tree $tree </dev/null) &&
  22         {
  23                 echo $tree &&
  24                 echo $commit &&
  25                 git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)        .*/\\1/"
  26         } >obj-list &&
  27         {
  28                 git diff-tree --root -p $commit &&
  29                 while read object
  30                 do
  31                         t=$(git cat-file -t $object) &&
  32                         git cat-file $t $object || return 1
  33                 done <obj-list
  34         } >expect
  35 '
  36
  37 test_expect_success 'setup pack-object <stdin' '
  38         git init pack-object-stdin &&
  39         test_commit -C pack-object-stdin one &&
  40         test_commit -C pack-object-stdin two
  41
  42 '
  43
  44 test_expect_success 'pack-object <stdin parsing: basic [|--revs]' '
  45         cat >in <<-EOF &&
  46         $(git -C pack-object-stdin rev-parse one)
  47         EOF
  48
  49         git -C pack-object-stdin pack-objects basic-stdin <in &&
  50         idx=$(echo pack-object-stdin/basic-stdin-*.idx) &&
  51         git show-index <"$idx" >actual &&
  52         test_line_count = 1 actual &&
  53
  54         git -C pack-object-stdin pack-objects --revs basic-stdin-revs <in &&
  55         idx=$(echo pack-object-stdin/basic-stdin-revs-*.idx) &&
  56         git show-index <"$idx" >actual &&
  57         test_line_count = 3 actual
  58 '
  59
  60 test_expect_success 'pack-object <stdin parsing: [|--revs] bad line' '
  61         cat >in <<-EOF &&
  62         $(git -C pack-object-stdin rev-parse one)
  63         garbage
  64         $(git -C pack-object-stdin rev-parse two)
  65         EOF
  66
  67         sed "s/^> //g" >err.expect <<-EOF &&
  68         fatal: expected object ID, got garbage:
  69         >  garbage
  70
  71         EOF
  72         test_must_fail git -C pack-object-stdin pack-objects bad-line-stdin <in 2>err.actual &&
  73         test_cmp err.expect err.actual &&
  74
  75         cat >err.expect <<-EOF &&
  76         fatal: bad revision '"'"'garbage'"'"'
  77         EOF
  78         test_must_fail git -C pack-object-stdin pack-objects --revs bad-line-stdin-revs <in 2>err.actual &&
  79         test_cmp err.expect err.actual
  80 '
  81
  82 test_expect_success 'pack-object <stdin parsing: [|--revs] empty line' '
  83         cat >in <<-EOF &&
  84         $(git -C pack-object-stdin rev-parse one)
  85
  86         $(git -C pack-object-stdin rev-parse two)
  87         EOF
  88
  89         sed -e "s/^> //g" -e "s/Z$//g" >err.expect <<-EOF &&
  90         fatal: expected object ID, got garbage:
  91         >  Z
  92
  93         EOF
  94         test_must_fail git -C pack-object-stdin pack-objects empty-line-stdin <in 2>err.actual &&
  95         test_cmp err.expect err.actual &&
  96
  97         git -C pack-object-stdin pack-objects --revs empty-line-stdin-revs <in &&
  98         idx=$(echo pack-object-stdin/empty-line-stdin-revs-*.idx) &&
  99         git show-index <"$idx" >actual &&
 100         test_line_count = 3 actual
 101 '
 102
 103 test_expect_success 'pack-object <stdin parsing: [|--revs] with --stdin' '
 104         cat >in <<-EOF &&
 105         $(git -C pack-object-stdin rev-parse one)
 106         $(git -C pack-object-stdin rev-parse two)
 107         EOF
 108
 109         # There is the "--stdin-packs is incompatible with --revs"
 110         # test below, but we should make sure that the revision.c
 111         # --stdin is not picked up
 112         cat >err.expect <<-EOF &&
 113         fatal: disallowed abbreviated or ambiguous option '"'"'stdin'"'"'
 114         EOF
 115         test_must_fail git -C pack-object-stdin pack-objects stdin-with-stdin-option --stdin <in 2>err.actual &&
 116         test_cmp err.expect err.actual &&
 117
 118         test_must_fail git -C pack-object-stdin pack-objects --stdin --revs stdin-with-stdin-option-revs 2>err.actual <in &&
 119         test_cmp err.expect err.actual
 120 '
 121
 122 test_expect_success 'pack-object <stdin parsing: --stdin-packs handles garbage' '
 123         cat >in <<-EOF &&
 124         $(git -C pack-object-stdin rev-parse one)
 125         $(git -C pack-object-stdin rev-parse two)
 126         EOF
 127
 128         # That we get "two" and not "one" has to do with OID
 129         # ordering. It happens to be the same here under SHA-1 and
 130         # SHA-256. See commentary in pack-objects.c
 131         cat >err.expect <<-EOF &&
 132         fatal: could not find pack '"'"'$(git -C pack-object-stdin rev-parse two)'"'"'
 133         EOF
 134         test_must_fail git \
 135                 -C pack-object-stdin \
 136                 pack-objects stdin-with-stdin-option --stdin-packs \
 137                 <in 2>err.actual &&
 138         test_cmp err.expect err.actual
 139 '
 140
 141 # usage: check_deltas <stderr_from_pack_objects> <cmp_op> <nr_deltas>
 142 # e.g.: check_deltas stderr -gt 0
 143 check_deltas() {
 144         deltas=$(perl -lne '/delta (\d+)/ and print $1' "$1") &&
 145         shift &&
 146         if ! test "$deltas" "$@"
 147         then
 148                 echo >&2 "unexpected number of deltas (compared $delta $*)"
 149                 return 1
 150         fi
 151 }
 152
 153 test_expect_success 'pack without delta' '
 154         packname_1=$(git pack-objects --progress --window=0 test-1 \
 155                         <obj-list 2>stderr) &&
 156         check_deltas stderr = 0
 157 '
 158
 159 test_expect_success 'pack-objects with bogus arguments' '
 160         test_must_fail git pack-objects --window=0 test-1 blah blah <obj-list
 161 '
 162
 163 check_unpack () {
 164         local packname="$1" &&
 165         local object_list="$2" &&
 166         local git_config="$3" &&
 167         test_when_finished "rm -rf git2" &&
 168         git $git_config init --bare git2 &&
 169         (
 170                 git $git_config -C git2 unpack-objects -n <"$packname".pack &&
 171                 git $git_config -C git2 unpack-objects <"$packname".pack &&
 172                 git $git_config -C git2 cat-file --batch-check="%(objectname)"
 173         ) <"$object_list" >current &&
 174         cmp "$object_list" current
 175 }
 176
 177 test_expect_success 'unpack without delta' '
 178         check_unpack test-1-${packname_1} obj-list
 179 '
 180
 181 BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch'
 182
 183 test_expect_success 'unpack without delta (core.fsyncmethod=batch)' '
 184         check_unpack test-1-${packname_1} obj-list "$BATCH_CONFIGURATION"
 185 '
 186
 187 test_expect_success 'pack with REF_DELTA' '
 188         packname_2=$(git pack-objects --progress test-2 <obj-list 2>stderr) &&
 189         check_deltas stderr -gt 0
 190 '
 191
 192 test_expect_success 'unpack with REF_DELTA' '
 193         check_unpack test-2-${packname_2} obj-list
 194 '
 195
 196 test_expect_success 'unpack with REF_DELTA (core.fsyncmethod=batch)' '
 197        check_unpack test-2-${packname_2} obj-list "$BATCH_CONFIGURATION"
 198 '
 199
 200 test_expect_success 'pack with OFS_DELTA' '
 201         packname_3=$(git pack-objects --progress --delta-base-offset test-3 \
 202                         <obj-list 2>stderr) &&
 203         check_deltas stderr -gt 0
 204 '
 205
 206 test_expect_success 'unpack with OFS_DELTA' '
 207         check_unpack test-3-${packname_3} obj-list
 208 '
 209
 210 test_expect_success 'unpack with OFS_DELTA (core.fsyncmethod=batch)' '
 211         check_unpack test-3-${packname_3} obj-list "$BATCH_CONFIGURATION"
 212 '
 213
 214 test_expect_success 'compare delta flavors' '
 215         perl -e '\''
 216                 defined($_ = -s $_) or die for @ARGV;
 217                 exit 1 if $ARGV[0] <= $ARGV[1];
 218         '\'' test-2-$packname_2.pack test-3-$packname_3.pack
 219 '
 220
 221 check_use_objects () {
 222         test_when_finished "rm -rf git2" &&
 223         git init --bare git2 &&
 224         cp "$1".pack "$1".idx git2/objects/pack &&
 225         (
 226                 cd git2 &&
 227                 git diff-tree --root -p $commit &&
 228                 while read object
 229                 do
 230                         t=$(git cat-file -t $object) &&
 231                         git cat-file $t $object || exit 1
 232                 done
 233         ) <obj-list >current &&
 234         cmp expect current
 235 }
 236
 237 test_expect_success 'use packed objects' '
 238         check_use_objects test-1-${packname_1}
 239 '
 240
 241 test_expect_success 'use packed deltified (REF_DELTA) objects' '
 242         check_use_objects test-2-${packname_2}
 243 '
 244
 245 test_expect_success 'use packed deltified (OFS_DELTA) objects' '
 246         check_use_objects test-3-${packname_3}
 247 '
 248
 249 test_expect_success 'survive missing objects/pack directory' '
 250         (
 251                 rm -fr missing-pack &&
 252                 mkdir missing-pack &&
 253                 cd missing-pack &&
 254                 git init &&
 255                 GOP=.git/objects/pack &&
 256                 rm -fr $GOP &&
 257                 git index-pack --stdin --keep=test <../test-3-${packname_3}.pack &&
 258                 test -f $GOP/pack-${packname_3}.pack &&
 259                 cmp $GOP/pack-${packname_3}.pack ../test-3-${packname_3}.pack &&
 260                 test -f $GOP/pack-${packname_3}.idx &&
 261                 cmp $GOP/pack-${packname_3}.idx ../test-3-${packname_3}.idx &&
 262                 test -f $GOP/pack-${packname_3}.keep
 263         )
 264 '
 265
 266 test_expect_success 'verify pack' '
 267         git verify-pack test-1-${packname_1}.idx \
 268                 test-2-${packname_2}.idx \
 269                 test-3-${packname_3}.idx
 270 '
 271
 272 test_expect_success 'verify pack -v' '
 273         git verify-pack -v test-1-${packname_1}.idx \
 274                 test-2-${packname_2}.idx \
 275                 test-3-${packname_3}.idx
 276 '
 277
 278 test_expect_success 'verify-pack catches mismatched .idx and .pack files' '
 279         cat test-1-${packname_1}.idx >test-3.idx &&
 280         cat test-2-${packname_2}.pack >test-3.pack &&
 281         if git verify-pack test-3.idx
 282         then false
 283         else :;
 284         fi
 285 '
 286
 287 test_expect_success 'verify-pack catches a corrupted pack signature' '
 288         cat test-1-${packname_1}.pack >test-3.pack &&
 289         echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=2 &&
 290         if git verify-pack test-3.idx
 291         then false
 292         else :;
 293         fi
 294 '
 295
 296 test_expect_success 'verify-pack catches a corrupted pack version' '
 297         cat test-1-${packname_1}.pack >test-3.pack &&
 298         echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=7 &&
 299         if git verify-pack test-3.idx
 300         then false
 301         else :;
 302         fi
 303 '
 304
 305 test_expect_success 'verify-pack catches a corrupted type/size of the 1st packed object data' '
 306         cat test-1-${packname_1}.pack >test-3.pack &&
 307         echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=12 &&
 308         if git verify-pack test-3.idx
 309         then false
 310         else :;
 311         fi
 312 '
 313
 314 test_expect_success 'verify-pack catches a corrupted sum of the index file itself' '
 315         l=$(wc -c <test-3.idx) &&
 316         l=$(expr $l - 20) &&
 317         cat test-1-${packname_1}.pack >test-3.pack &&
 318         printf "%20s" "" | dd of=test-3.idx count=20 bs=1 conv=notrunc seek=$l &&
 319         if git verify-pack test-3.pack
 320         then false
 321         else :;
 322         fi
 323 '
 324
 325 test_expect_success 'build pack index for an existing pack' '
 326         cat test-1-${packname_1}.pack >test-3.pack &&
 327         git index-pack -o tmp.idx test-3.pack &&
 328         cmp tmp.idx test-1-${packname_1}.idx &&
 329
 330         git index-pack --promisor=message test-3.pack &&
 331         cmp test-3.idx test-1-${packname_1}.idx &&
 332         echo message >expect &&
 333         test_cmp expect test-3.promisor &&
 334
 335         cat test-2-${packname_2}.pack >test-3.pack &&
 336         git index-pack -o tmp.idx test-2-${packname_2}.pack &&
 337         cmp tmp.idx test-2-${packname_2}.idx &&
 338
 339         git index-pack test-3.pack &&
 340         cmp test-3.idx test-2-${packname_2}.idx &&
 341
 342         cat test-3-${packname_3}.pack >test-3.pack &&
 343         git index-pack -o tmp.idx test-3-${packname_3}.pack &&
 344         cmp tmp.idx test-3-${packname_3}.idx &&
 345
 346         git index-pack test-3.pack &&
 347         cmp test-3.idx test-3-${packname_3}.idx &&
 348
 349         cat test-1-${packname_1}.pack >test-4.pack &&
 350         rm -f test-4.keep &&
 351         git index-pack --keep=why test-4.pack &&
 352         cmp test-1-${packname_1}.idx test-4.idx &&
 353         test -f test-4.keep &&
 354
 355         :
 356 '
 357
 358 test_expect_success 'unpacking with --strict' '
 359
 360         for j in a b c d e f g
 361         do
 362                 for i in 0 1 2 3 4 5 6 7 8 9
 363                 do
 364                         o=$(echo $j$i | git hash-object -w --stdin) &&
 365                         echo "100644 $o 0 $j$i" || return 1
 366                 done
 367         done >LIST &&
 368         rm -f .git/index &&
 369         git update-index --index-info <LIST &&
 370         LIST=$(git write-tree) &&
 371         rm -f .git/index &&
 372         head -n 10 LIST | git update-index --index-info &&
 373         LI=$(git write-tree) &&
 374         rm -f .git/index &&
 375         tail -n 10 LIST | git update-index --index-info &&
 376         ST=$(git write-tree) &&
 377         git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
 378         PACK5=$( git pack-objects test-5 <actual ) &&
 379         PACK6=$( test_write_lines "$LIST" "$LI" "$ST" | git pack-objects test-6 ) &&
 380         test_create_repo test-5 &&
 381         (
 382                 cd test-5 &&
 383                 git unpack-objects --strict <../test-5-$PACK5.pack &&
 384                 git ls-tree -r $LIST &&
 385                 git ls-tree -r $LI &&
 386                 git ls-tree -r $ST
 387         ) &&
 388         test_create_repo test-6 &&
 389         (
 390                 # tree-only into empty repo -- many unreachables
 391                 cd test-6 &&
 392                 test_must_fail git unpack-objects --strict <../test-6-$PACK6.pack
 393         ) &&
 394         (
 395                 # already populated -- no unreachables
 396                 cd test-5 &&
 397                 git unpack-objects --strict <../test-6-$PACK6.pack
 398         )
 399 '
 400
 401 test_expect_success 'index-pack with --strict' '
 402
 403         for j in a b c d e f g
 404         do
 405                 for i in 0 1 2 3 4 5 6 7 8 9
 406                 do
 407                         o=$(echo $j$i | git hash-object -w --stdin) &&
 408                         echo "100644 $o 0 $j$i" || return 1
 409                 done
 410         done >LIST &&
 411         rm -f .git/index &&
 412         git update-index --index-info <LIST &&
 413         LIST=$(git write-tree) &&
 414         rm -f .git/index &&
 415         head -n 10 LIST | git update-index --index-info &&
 416         LI=$(git write-tree) &&
 417         rm -f .git/index &&
 418         tail -n 10 LIST | git update-index --index-info &&
 419         ST=$(git write-tree) &&
 420         git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
 421         PACK5=$( git pack-objects test-5 <actual ) &&
 422         PACK6=$( test_write_lines "$LIST" "$LI" "$ST" | git pack-objects test-6 ) &&
 423         test_create_repo test-7 &&
 424         (
 425                 cd test-7 &&
 426                 git index-pack --strict --stdin <../test-5-$PACK5.pack &&
 427                 git ls-tree -r $LIST &&
 428                 git ls-tree -r $LI &&
 429                 git ls-tree -r $ST
 430         ) &&
 431         test_create_repo test-8 &&
 432         (
 433                 # tree-only into empty repo -- many unreachables
 434                 cd test-8 &&
 435                 test_must_fail git index-pack --strict --stdin <../test-6-$PACK6.pack
 436         ) &&
 437         (
 438                 # already populated -- no unreachables
 439                 cd test-7 &&
 440                 git index-pack --strict --stdin <../test-6-$PACK6.pack
 441         )
 442 '
 443
 444 test_expect_success 'honor pack.packSizeLimit' '
 445         git config pack.packSizeLimit 3m &&
 446         packname_10=$(git pack-objects test-10 <obj-list) &&
 447         test 2 = $(ls test-10-*.pack | wc -l)
 448 '
 449
 450 test_expect_success 'verify resulting packs' '
 451         git verify-pack test-10-*.pack
 452 '
 453
 454 test_expect_success 'tolerate packsizelimit smaller than biggest object' '
 455         git config pack.packSizeLimit 1 &&
 456         packname_11=$(git pack-objects test-11 <obj-list) &&
 457         test 5 = $(ls test-11-*.pack | wc -l)
 458 '
 459
 460 test_expect_success 'verify resulting packs' '
 461         git verify-pack test-11-*.pack
 462 '
 463
 464 test_expect_success 'set up pack for non-repo tests' '
 465         # make sure we have a pack with no matching index file
 466         cp test-1-*.pack foo.pack
 467 '
 468
 469 test_expect_success 'index-pack --stdin complains of non-repo' '
 470         nongit test_must_fail git index-pack --object-format=$(test_oid algo) --stdin <foo.pack &&
 471         test_path_is_missing non-repo/.git
 472 '
 473
 474 test_expect_success 'index-pack <pack> works in non-repo' '
 475         nongit git index-pack --object-format=$(test_oid algo) ../foo.pack &&
 476         test_path_is_file foo.idx
 477 '
 478
 479 test_expect_success 'index-pack --strict <pack> works in non-repo' '
 480         rm -f foo.idx &&
 481         nongit git index-pack --strict --object-format=$(test_oid algo) ../foo.pack &&
 482         test_path_is_file foo.idx
 483 '
 484
 485 test_expect_success !PTHREADS,!FAIL_PREREQS \
 486         'index-pack --threads=N or pack.threads=N warns when no pthreads' '
 487         test_must_fail git index-pack --threads=2 2>err &&
 488         grep ^warning: err >warnings &&
 489         test_line_count = 1 warnings &&
 490         grep -F "no threads support, ignoring --threads=2" err &&
 491
 492         test_must_fail git -c pack.threads=2 index-pack 2>err &&
 493         grep ^warning: err >warnings &&
 494         test_line_count = 1 warnings &&
 495         grep -F "no threads support, ignoring pack.threads" err &&
 496
 497         test_must_fail git -c pack.threads=2 index-pack --threads=4 2>err &&
 498         grep ^warning: err >warnings &&
 499         test_line_count = 2 warnings &&
 500         grep -F "no threads support, ignoring --threads=4" err &&
 501         grep -F "no threads support, ignoring pack.threads" err
 502 '
 503
 504 test_expect_success !PTHREADS,!FAIL_PREREQS \
 505         'pack-objects --threads=N or pack.threads=N warns when no pthreads' '
 506         git pack-objects --threads=2 --stdout --all </dev/null >/dev/null 2>err &&
 507         grep ^warning: err >warnings &&
 508         test_line_count = 1 warnings &&
 509         grep -F "no threads support, ignoring --threads" err &&
 510
 511         git -c pack.threads=2 pack-objects --stdout --all </dev/null >/dev/null 2>err &&
 512         grep ^warning: err >warnings &&
 513         test_line_count = 1 warnings &&
 514         grep -F "no threads support, ignoring pack.threads" err &&
 515
 516         git -c pack.threads=2 pack-objects --threads=4 --stdout --all </dev/null >/dev/null 2>err &&
 517         grep ^warning: err >warnings &&
 518         test_line_count = 2 warnings &&
 519         grep -F "no threads support, ignoring --threads" err &&
 520         grep -F "no threads support, ignoring pack.threads" err
 521 '
 522
 523 test_expect_success 'pack-objects in too-many-packs mode' '
 524         GIT_TEST_FULL_IN_PACK_ARRAY=1 git repack -ad &&
 525         git fsck
 526 '
 527
 528 test_expect_success 'setup: fake a SHA1 hash collision' '
 529         git init corrupt &&
 530         (
 531                 cd corrupt &&
 532                 long_a=$(git hash-object -w ../a | sed -e "s!^..!&/!") &&
 533                 long_b=$(git hash-object -w ../b | sed -e "s!^..!&/!") &&
 534                 test -f .git/objects/$long_b &&
 535                 cp -f   .git/objects/$long_a \
 536                         .git/objects/$long_b
 537         )
 538 '
 539
 540 test_expect_success 'make sure index-pack detects the SHA1 collision' '
 541         (
 542                 cd corrupt &&
 543                 test_must_fail git index-pack -o ../bad.idx ../test-3.pack 2>msg &&
 544                 test_i18ngrep "SHA1 COLLISION FOUND" msg
 545         )
 546 '
 547
 548 test_expect_success 'make sure index-pack detects the SHA1 collision (large blobs)' '
 549         (
 550                 cd corrupt &&
 551                 test_must_fail git -c core.bigfilethreshold=1 index-pack -o ../bad.idx ../test-3.pack 2>msg &&
 552                 test_i18ngrep "SHA1 COLLISION FOUND" msg
 553         )
 554 '
 555
 556 test_expect_success 'prefetch objects' '
 557         rm -rf server client &&
 558
 559         git init server &&
 560         test_config -C server uploadpack.allowanysha1inwant 1 &&
 561         test_config -C server uploadpack.allowfilter 1 &&
 562         test_config -C server protocol.version 2 &&
 563
 564         echo one >server/one &&
 565         git -C server add one &&
 566         git -C server commit -m one &&
 567         git -C server branch one_branch &&
 568
 569         echo two_a >server/two_a &&
 570         echo two_b >server/two_b &&
 571         git -C server add two_a two_b &&
 572         git -C server commit -m two &&
 573
 574         echo three >server/three &&
 575         git -C server add three &&
 576         git -C server commit -m three &&
 577         git -C server branch three_branch &&
 578
 579         # Clone, fetch "two" with blobs excluded, and re-push it. This requires
 580         # the client to have the blobs of "two" - verify that these are
 581         # prefetched in one batch.
 582         git clone --filter=blob:none --single-branch -b one_branch \
 583                 "file://$(pwd)/server" client &&
 584         test_config -C client protocol.version 2 &&
 585         TWO=$(git -C server rev-parse three_branch^) &&
 586         git -C client fetch --filter=blob:none origin "$TWO" &&
 587         GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
 588         grep "fetch> done" trace >donelines &&
 589         test_line_count = 1 donelines
 590 '
 591
 592 test_expect_success 'negative window clamps to 0' '
 593         git pack-objects --progress --window=-1 neg-window <obj-list 2>stderr &&
 594         check_deltas stderr = 0
 595 '
 596
 597 test_done