The second batch
[git.git] / t / t5300-pack-object.sh
blob61e2be2903d344563fe2ebea718de962e60d7b78
1 #!/bin/sh
3 # Copyright (c) 2005 Junio C Hamano
6 test_description='git pack-object
9 . ./test-lib.sh
11 test_expect_success 'setup' '
12 rm -f .git/index* &&
13 perl -e "print \"a\" x 4096;" >a &&
14 perl -e "print \"b\" x 4096;" >b &&
15 perl -e "print \"c\" x 4096;" >c &&
16 test-tool genrandom "seed a" 2097152 >a_big &&
17 test-tool genrandom "seed b" 2097152 >b_big &&
18 git update-index --add a a_big b b_big c &&
19 cat c >d && echo foo >>d && git update-index --add d &&
20 tree=$(git write-tree) &&
21 commit=$(git commit-tree $tree </dev/null) &&
23 echo $tree &&
24 echo $commit &&
25 git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/"
26 } >obj-list &&
28 git diff-tree --root -p $commit &&
29 while read object
31 t=$(git cat-file -t $object) &&
32 git cat-file $t $object || return 1
33 done <obj-list
34 } >expect
37 test_expect_success 'setup pack-object <stdin' '
38 git init pack-object-stdin &&
39 test_commit -C pack-object-stdin one &&
40 test_commit -C pack-object-stdin two
44 test_expect_success 'pack-object <stdin parsing: basic [|--revs]' '
45 cat >in <<-EOF &&
46 $(git -C pack-object-stdin rev-parse one)
47 EOF
49 git -C pack-object-stdin pack-objects basic-stdin <in &&
50 idx=$(echo pack-object-stdin/basic-stdin-*.idx) &&
51 git show-index <"$idx" >actual &&
52 test_line_count = 1 actual &&
54 git -C pack-object-stdin pack-objects --revs basic-stdin-revs <in &&
55 idx=$(echo pack-object-stdin/basic-stdin-revs-*.idx) &&
56 git show-index <"$idx" >actual &&
57 test_line_count = 3 actual
60 test_expect_success 'pack-object <stdin parsing: [|--revs] bad line' '
61 cat >in <<-EOF &&
62 $(git -C pack-object-stdin rev-parse one)
63 garbage
64 $(git -C pack-object-stdin rev-parse two)
65 EOF
67 sed "s/^> //g" >err.expect <<-EOF &&
68 fatal: expected object ID, got garbage:
69 > garbage
71 EOF
72 test_must_fail git -C pack-object-stdin pack-objects bad-line-stdin <in 2>err.actual &&
73 test_cmp err.expect err.actual &&
75 cat >err.expect <<-EOF &&
76 fatal: bad revision '"'"'garbage'"'"'
77 EOF
78 test_must_fail git -C pack-object-stdin pack-objects --revs bad-line-stdin-revs <in 2>err.actual &&
79 test_cmp err.expect err.actual
82 test_expect_success 'pack-object <stdin parsing: [|--revs] empty line' '
83 cat >in <<-EOF &&
84 $(git -C pack-object-stdin rev-parse one)
86 $(git -C pack-object-stdin rev-parse two)
87 EOF
89 sed -e "s/^> //g" -e "s/Z$//g" >err.expect <<-EOF &&
90 fatal: expected object ID, got garbage:
91 > Z
93 EOF
94 test_must_fail git -C pack-object-stdin pack-objects empty-line-stdin <in 2>err.actual &&
95 test_cmp err.expect err.actual &&
97 git -C pack-object-stdin pack-objects --revs empty-line-stdin-revs <in &&
98 idx=$(echo pack-object-stdin/empty-line-stdin-revs-*.idx) &&
99 git show-index <"$idx" >actual &&
100 test_line_count = 3 actual
103 test_expect_success 'pack-object <stdin parsing: [|--revs] with --stdin' '
104 cat >in <<-EOF &&
105 $(git -C pack-object-stdin rev-parse one)
106 $(git -C pack-object-stdin rev-parse two)
109 # There is the "--stdin-packs is incompatible with --revs"
110 # test below, but we should make sure that the revision.c
111 # --stdin is not picked up
112 cat >err.expect <<-EOF &&
113 fatal: disallowed abbreviated or ambiguous option '"'"'stdin'"'"'
115 test_must_fail git -C pack-object-stdin pack-objects stdin-with-stdin-option --stdin <in 2>err.actual &&
116 test_cmp err.expect err.actual &&
118 test_must_fail git -C pack-object-stdin pack-objects --stdin --revs stdin-with-stdin-option-revs 2>err.actual <in &&
119 test_cmp err.expect err.actual
122 test_expect_success 'pack-object <stdin parsing: --stdin-packs handles garbage' '
123 cat >in <<-EOF &&
124 $(git -C pack-object-stdin rev-parse one)
125 $(git -C pack-object-stdin rev-parse two)
128 # That we get "two" and not "one" has to do with OID
129 # ordering. It happens to be the same here under SHA-1 and
130 # SHA-256. See commentary in pack-objects.c
131 cat >err.expect <<-EOF &&
132 fatal: could not find pack '"'"'$(git -C pack-object-stdin rev-parse two)'"'"'
134 test_must_fail git \
135 -C pack-object-stdin \
136 pack-objects stdin-with-stdin-option --stdin-packs \
137 <in 2>err.actual &&
138 test_cmp err.expect err.actual
141 # usage: check_deltas <stderr_from_pack_objects> <cmp_op> <nr_deltas>
142 # e.g.: check_deltas stderr -gt 0
143 check_deltas() {
144 deltas=$(perl -lne '/delta (\d+)/ and print $1' "$1") &&
145 shift &&
146 if ! test "$deltas" "$@"
147 then
148 echo >&2 "unexpected number of deltas (compared $delta $*)"
149 return 1
153 test_expect_success 'pack without delta' '
154 packname_1=$(git pack-objects --progress --window=0 test-1 \
155 <obj-list 2>stderr) &&
156 check_deltas stderr = 0
159 test_expect_success 'pack-objects with bogus arguments' '
160 test_must_fail git pack-objects --window=0 test-1 blah blah <obj-list
163 check_unpack () {
164 local packname="$1" &&
165 local object_list="$2" &&
166 local git_config="$3" &&
167 test_when_finished "rm -rf git2" &&
168 git $git_config init --bare git2 &&
170 git $git_config -C git2 unpack-objects -n <"$packname".pack &&
171 git $git_config -C git2 unpack-objects <"$packname".pack &&
172 git $git_config -C git2 cat-file --batch-check="%(objectname)"
173 ) <"$object_list" >current &&
174 cmp "$object_list" current
177 test_expect_success 'unpack without delta' '
178 check_unpack test-1-${packname_1} obj-list
181 BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch'
183 test_expect_success 'unpack without delta (core.fsyncmethod=batch)' '
184 check_unpack test-1-${packname_1} obj-list "$BATCH_CONFIGURATION"
187 test_expect_success 'pack with REF_DELTA' '
188 packname_2=$(git pack-objects --progress test-2 <obj-list 2>stderr) &&
189 check_deltas stderr -gt 0
192 test_expect_success 'unpack with REF_DELTA' '
193 check_unpack test-2-${packname_2} obj-list
196 test_expect_success 'unpack with REF_DELTA (core.fsyncmethod=batch)' '
197 check_unpack test-2-${packname_2} obj-list "$BATCH_CONFIGURATION"
200 test_expect_success 'pack with OFS_DELTA' '
201 packname_3=$(git pack-objects --progress --delta-base-offset test-3 \
202 <obj-list 2>stderr) &&
203 check_deltas stderr -gt 0
206 test_expect_success 'unpack with OFS_DELTA' '
207 check_unpack test-3-${packname_3} obj-list
210 test_expect_success 'unpack with OFS_DELTA (core.fsyncmethod=batch)' '
211 check_unpack test-3-${packname_3} obj-list "$BATCH_CONFIGURATION"
214 test_expect_success 'compare delta flavors' '
215 perl -e '\''
216 defined($_ = -s $_) or die for @ARGV;
217 exit 1 if $ARGV[0] <= $ARGV[1];
218 '\'' test-2-$packname_2.pack test-3-$packname_3.pack
221 check_use_objects () {
222 test_when_finished "rm -rf git2" &&
223 git init --bare git2 &&
224 cp "$1".pack "$1".idx git2/objects/pack &&
226 cd git2 &&
227 git diff-tree --root -p $commit &&
228 while read object
230 t=$(git cat-file -t $object) &&
231 git cat-file $t $object || exit 1
232 done
233 ) <obj-list >current &&
234 cmp expect current
237 test_expect_success 'use packed objects' '
238 check_use_objects test-1-${packname_1}
241 test_expect_success 'use packed deltified (REF_DELTA) objects' '
242 check_use_objects test-2-${packname_2}
245 test_expect_success 'use packed deltified (OFS_DELTA) objects' '
246 check_use_objects test-3-${packname_3}
249 test_expect_success 'survive missing objects/pack directory' '
251 rm -fr missing-pack &&
252 mkdir missing-pack &&
253 cd missing-pack &&
254 git init &&
255 GOP=.git/objects/pack &&
256 rm -fr $GOP &&
257 git index-pack --stdin --keep=test <../test-3-${packname_3}.pack &&
258 test -f $GOP/pack-${packname_3}.pack &&
259 cmp $GOP/pack-${packname_3}.pack ../test-3-${packname_3}.pack &&
260 test -f $GOP/pack-${packname_3}.idx &&
261 cmp $GOP/pack-${packname_3}.idx ../test-3-${packname_3}.idx &&
262 test -f $GOP/pack-${packname_3}.keep
266 test_expect_success 'verify pack' '
267 git verify-pack test-1-${packname_1}.idx \
268 test-2-${packname_2}.idx \
269 test-3-${packname_3}.idx
272 test_expect_success 'verify pack -v' '
273 git verify-pack -v test-1-${packname_1}.idx \
274 test-2-${packname_2}.idx \
275 test-3-${packname_3}.idx
278 test_expect_success 'verify-pack catches mismatched .idx and .pack files' '
279 cat test-1-${packname_1}.idx >test-3.idx &&
280 cat test-2-${packname_2}.pack >test-3.pack &&
281 if git verify-pack test-3.idx
282 then false
283 else :;
287 test_expect_success 'verify-pack catches a corrupted pack signature' '
288 cat test-1-${packname_1}.pack >test-3.pack &&
289 echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=2 &&
290 if git verify-pack test-3.idx
291 then false
292 else :;
296 test_expect_success 'verify-pack catches a corrupted pack version' '
297 cat test-1-${packname_1}.pack >test-3.pack &&
298 echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=7 &&
299 if git verify-pack test-3.idx
300 then false
301 else :;
305 test_expect_success 'verify-pack catches a corrupted type/size of the 1st packed object data' '
306 cat test-1-${packname_1}.pack >test-3.pack &&
307 echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=12 &&
308 if git verify-pack test-3.idx
309 then false
310 else :;
314 test_expect_success 'verify-pack catches a corrupted sum of the index file itself' '
315 l=$(wc -c <test-3.idx) &&
316 l=$(expr $l - 20) &&
317 cat test-1-${packname_1}.pack >test-3.pack &&
318 printf "%20s" "" | dd of=test-3.idx count=20 bs=1 conv=notrunc seek=$l &&
319 if git verify-pack test-3.pack
320 then false
321 else :;
325 test_expect_success 'build pack index for an existing pack' '
326 cat test-1-${packname_1}.pack >test-3.pack &&
327 git index-pack -o tmp.idx test-3.pack &&
328 cmp tmp.idx test-1-${packname_1}.idx &&
330 git index-pack --promisor=message test-3.pack &&
331 cmp test-3.idx test-1-${packname_1}.idx &&
332 echo message >expect &&
333 test_cmp expect test-3.promisor &&
335 cat test-2-${packname_2}.pack >test-3.pack &&
336 git index-pack -o tmp.idx test-2-${packname_2}.pack &&
337 cmp tmp.idx test-2-${packname_2}.idx &&
339 git index-pack test-3.pack &&
340 cmp test-3.idx test-2-${packname_2}.idx &&
342 cat test-3-${packname_3}.pack >test-3.pack &&
343 git index-pack -o tmp.idx test-3-${packname_3}.pack &&
344 cmp tmp.idx test-3-${packname_3}.idx &&
346 git index-pack test-3.pack &&
347 cmp test-3.idx test-3-${packname_3}.idx &&
349 cat test-1-${packname_1}.pack >test-4.pack &&
350 rm -f test-4.keep &&
351 git index-pack --keep=why test-4.pack &&
352 cmp test-1-${packname_1}.idx test-4.idx &&
353 test -f test-4.keep &&
358 test_expect_success 'unpacking with --strict' '
360 for j in a b c d e f g
362 for i in 0 1 2 3 4 5 6 7 8 9
364 o=$(echo $j$i | git hash-object -w --stdin) &&
365 echo "100644 $o 0 $j$i" || return 1
366 done
367 done >LIST &&
368 rm -f .git/index &&
369 git update-index --index-info <LIST &&
370 LIST=$(git write-tree) &&
371 rm -f .git/index &&
372 head -n 10 LIST | git update-index --index-info &&
373 LI=$(git write-tree) &&
374 rm -f .git/index &&
375 tail -n 10 LIST | git update-index --index-info &&
376 ST=$(git write-tree) &&
377 git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
378 PACK5=$( git pack-objects test-5 <actual ) &&
379 PACK6=$( test_write_lines "$LIST" "$LI" "$ST" | git pack-objects test-6 ) &&
380 test_create_repo test-5 &&
382 cd test-5 &&
383 git unpack-objects --strict <../test-5-$PACK5.pack &&
384 git ls-tree -r $LIST &&
385 git ls-tree -r $LI &&
386 git ls-tree -r $ST
387 ) &&
388 test_create_repo test-6 &&
390 # tree-only into empty repo -- many unreachables
391 cd test-6 &&
392 test_must_fail git unpack-objects --strict <../test-6-$PACK6.pack
393 ) &&
395 # already populated -- no unreachables
396 cd test-5 &&
397 git unpack-objects --strict <../test-6-$PACK6.pack
401 test_expect_success 'index-pack with --strict' '
403 for j in a b c d e f g
405 for i in 0 1 2 3 4 5 6 7 8 9
407 o=$(echo $j$i | git hash-object -w --stdin) &&
408 echo "100644 $o 0 $j$i" || return 1
409 done
410 done >LIST &&
411 rm -f .git/index &&
412 git update-index --index-info <LIST &&
413 LIST=$(git write-tree) &&
414 rm -f .git/index &&
415 head -n 10 LIST | git update-index --index-info &&
416 LI=$(git write-tree) &&
417 rm -f .git/index &&
418 tail -n 10 LIST | git update-index --index-info &&
419 ST=$(git write-tree) &&
420 git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
421 PACK5=$( git pack-objects test-5 <actual ) &&
422 PACK6=$( test_write_lines "$LIST" "$LI" "$ST" | git pack-objects test-6 ) &&
423 test_create_repo test-7 &&
425 cd test-7 &&
426 git index-pack --strict --stdin <../test-5-$PACK5.pack &&
427 git ls-tree -r $LIST &&
428 git ls-tree -r $LI &&
429 git ls-tree -r $ST
430 ) &&
431 test_create_repo test-8 &&
433 # tree-only into empty repo -- many unreachables
434 cd test-8 &&
435 test_must_fail git index-pack --strict --stdin <../test-6-$PACK6.pack
436 ) &&
438 # already populated -- no unreachables
439 cd test-7 &&
440 git index-pack --strict --stdin <../test-6-$PACK6.pack
444 test_expect_success 'setup for --strict and --fsck-objects downgrading fsck msgs' '
445 git init strict &&
447 cd strict &&
448 test_commit first hello &&
449 cat >commit <<-EOF &&
450 tree $(git rev-parse HEAD^{tree})
451 parent $(git rev-parse HEAD)
452 author A U Thor
453 committer A U Thor
455 commit: this is a commit with bad emails
458 git hash-object --literally -t commit -w --stdin <commit >commit_list &&
459 git pack-objects test <commit_list >pack-name
463 test_with_bad_commit () {
464 must_fail_arg="$1" &&
465 must_pass_arg="$2" &&
467 cd strict &&
468 test_must_fail git index-pack "$must_fail_arg" "test-$(cat pack-name).pack" &&
469 git index-pack "$must_pass_arg" "test-$(cat pack-name).pack"
473 test_expect_success 'index-pack with --strict downgrading fsck msgs' '
474 test_with_bad_commit --strict --strict="missingEmail=ignore"
477 test_expect_success 'index-pack with --fsck-objects downgrading fsck msgs' '
478 test_with_bad_commit --fsck-objects --fsck-objects="missingEmail=ignore"
481 test_expect_success 'cleanup for --strict and --fsck-objects downgrading fsck msgs' '
482 rm -rf strict
485 test_expect_success 'honor pack.packSizeLimit' '
486 git config pack.packSizeLimit 3m &&
487 packname_10=$(git pack-objects test-10 <obj-list) &&
488 test 2 = $(ls test-10-*.pack | wc -l)
491 test_expect_success 'verify resulting packs' '
492 git verify-pack test-10-*.pack
495 test_expect_success 'tolerate packsizelimit smaller than biggest object' '
496 git config pack.packSizeLimit 1 &&
497 packname_11=$(git pack-objects test-11 <obj-list) &&
498 test 5 = $(ls test-11-*.pack | wc -l)
501 test_expect_success 'verify resulting packs' '
502 git verify-pack test-11-*.pack
505 test_expect_success 'set up pack for non-repo tests' '
506 # make sure we have a pack with no matching index file
507 cp test-1-*.pack foo.pack
510 test_expect_success 'index-pack --stdin complains of non-repo' '
511 nongit test_must_fail git index-pack --object-format=$(test_oid algo) --stdin <foo.pack &&
512 test_path_is_missing non-repo/.git
515 test_expect_success 'index-pack <pack> works in non-repo' '
516 nongit git index-pack --object-format=$(test_oid algo) ../foo.pack &&
517 test_path_is_file foo.idx
520 test_expect_success 'index-pack --strict <pack> works in non-repo' '
521 rm -f foo.idx &&
522 nongit git index-pack --strict --object-format=$(test_oid algo) ../foo.pack &&
523 test_path_is_file foo.idx
526 test_expect_success !PTHREADS,!FAIL_PREREQS \
527 'index-pack --threads=N or pack.threads=N warns when no pthreads' '
528 test_must_fail git index-pack --threads=2 2>err &&
529 grep ^warning: err >warnings &&
530 test_line_count = 1 warnings &&
531 grep -F "no threads support, ignoring --threads=2" err &&
533 test_must_fail git -c pack.threads=2 index-pack 2>err &&
534 grep ^warning: err >warnings &&
535 test_line_count = 1 warnings &&
536 grep -F "no threads support, ignoring pack.threads" err &&
538 test_must_fail git -c pack.threads=2 index-pack --threads=4 2>err &&
539 grep ^warning: err >warnings &&
540 test_line_count = 2 warnings &&
541 grep -F "no threads support, ignoring --threads=4" err &&
542 grep -F "no threads support, ignoring pack.threads" err
545 test_expect_success !PTHREADS,!FAIL_PREREQS \
546 'pack-objects --threads=N or pack.threads=N warns when no pthreads' '
547 git pack-objects --threads=2 --stdout --all </dev/null >/dev/null 2>err &&
548 grep ^warning: err >warnings &&
549 test_line_count = 1 warnings &&
550 grep -F "no threads support, ignoring --threads" err &&
552 git -c pack.threads=2 pack-objects --stdout --all </dev/null >/dev/null 2>err &&
553 grep ^warning: err >warnings &&
554 test_line_count = 1 warnings &&
555 grep -F "no threads support, ignoring pack.threads" err &&
557 git -c pack.threads=2 pack-objects --threads=4 --stdout --all </dev/null >/dev/null 2>err &&
558 grep ^warning: err >warnings &&
559 test_line_count = 2 warnings &&
560 grep -F "no threads support, ignoring --threads" err &&
561 grep -F "no threads support, ignoring pack.threads" err
564 test_expect_success 'pack-objects in too-many-packs mode' '
565 GIT_TEST_FULL_IN_PACK_ARRAY=1 git repack -ad &&
566 git fsck
569 test_expect_success 'setup: fake a SHA1 hash collision' '
570 git init corrupt &&
572 cd corrupt &&
573 long_a=$(git hash-object -w ../a | sed -e "s!^..!&/!") &&
574 long_b=$(git hash-object -w ../b | sed -e "s!^..!&/!") &&
575 test -f .git/objects/$long_b &&
576 cp -f .git/objects/$long_a \
577 .git/objects/$long_b
581 test_expect_success 'make sure index-pack detects the SHA1 collision' '
583 cd corrupt &&
584 test_must_fail git index-pack -o ../bad.idx ../test-3.pack 2>msg &&
585 test_grep "SHA1 COLLISION FOUND" msg
589 test_expect_success 'make sure index-pack detects the SHA1 collision (large blobs)' '
591 cd corrupt &&
592 test_must_fail git -c core.bigfilethreshold=1 index-pack -o ../bad.idx ../test-3.pack 2>msg &&
593 test_grep "SHA1 COLLISION FOUND" msg
597 test_expect_success 'prefetch objects' '
598 rm -rf server client &&
600 git init server &&
601 test_config -C server uploadpack.allowanysha1inwant 1 &&
602 test_config -C server uploadpack.allowfilter 1 &&
603 test_config -C server protocol.version 2 &&
605 echo one >server/one &&
606 git -C server add one &&
607 git -C server commit -m one &&
608 git -C server branch one_branch &&
610 echo two_a >server/two_a &&
611 echo two_b >server/two_b &&
612 git -C server add two_a two_b &&
613 git -C server commit -m two &&
615 echo three >server/three &&
616 git -C server add three &&
617 git -C server commit -m three &&
618 git -C server branch three_branch &&
620 # Clone, fetch "two" with blobs excluded, and re-push it. This requires
621 # the client to have the blobs of "two" - verify that these are
622 # prefetched in one batch.
623 git clone --filter=blob:none --single-branch -b one_branch \
624 "file://$(pwd)/server" client &&
625 test_config -C client protocol.version 2 &&
626 TWO=$(git -C server rev-parse three_branch^) &&
627 git -C client fetch --filter=blob:none origin "$TWO" &&
628 GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
629 grep "fetch> done" trace >donelines &&
630 test_line_count = 1 donelines
633 test_expect_success 'negative window clamps to 0' '
634 git pack-objects --progress --window=-1 neg-window <obj-list 2>stderr &&
635 check_deltas stderr = 0
638 test_done