Sync with 2.38.4
[git.git] / t / t5329-pack-objects-cruft.sh
blob303f7a5d842d36edb191bb05ec61df9a6af8d7c9
1 #!/bin/sh
3 test_description='cruft pack related pack-objects tests'
4 . ./test-lib.sh
6 objdir=.git/objects
7 packdir=$objdir/pack
9 basic_cruft_pack_tests () {
10 expire="$1"
12 test_expect_success "unreachable loose objects are packed (expire $expire)" '
13 git init repo &&
14 test_when_finished "rm -fr repo" &&
16 cd repo &&
18 test_commit base &&
19 git repack -Ad &&
20 test_commit loose &&
22 test-tool chmtime +2000 "$objdir/$(test_oid_to_path \
23 $(git rev-parse loose:loose.t))" &&
24 test-tool chmtime +1000 "$objdir/$(test_oid_to_path \
25 $(git rev-parse loose^{tree}))" &&
28 git rev-list --objects --no-object-names base..loose |
29 while read oid
31 path="$objdir/$(test_oid_to_path "$oid")" &&
32 printf "%s %d\n" "$oid" "$(test-tool chmtime --get "$path")" ||
33 echo "object list generation failed for $oid"
34 done |
35 sort -k1
36 ) >expect &&
38 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
39 cruft="$(echo $keep | git pack-objects --cruft \
40 --cruft-expiration="$expire" $packdir/pack)" &&
41 test-tool pack-mtimes "pack-$cruft.mtimes" >actual &&
43 test_cmp expect actual
47 test_expect_success "unreachable packed objects are packed (expire $expire)" '
48 git init repo &&
49 test_when_finished "rm -fr repo" &&
51 cd repo &&
53 test_commit packed &&
54 git repack -Ad &&
55 test_commit other &&
57 git rev-list --objects --no-object-names packed.. >objects &&
58 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
59 other="$(git pack-objects --delta-base-offset \
60 $packdir/pack <objects)" &&
61 git prune-packed &&
63 test-tool chmtime --get -100 "$packdir/pack-$other.pack" >expect &&
65 cruft="$(git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack <<-EOF
66 $keep
67 -pack-$other.pack
68 EOF
69 )" &&
70 test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
72 cut -d" " -f2 <actual.raw | sort -u >actual &&
74 test_cmp expect actual
78 test_expect_success "unreachable cruft objects are repacked (expire $expire)" '
79 git init repo &&
80 test_when_finished "rm -fr repo" &&
82 cd repo &&
84 test_commit packed &&
85 git repack -Ad &&
86 test_commit other &&
88 git rev-list --objects --no-object-names packed.. >objects &&
89 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
91 cruft_a="$(echo $keep | git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack)" &&
92 git prune-packed &&
93 cruft_b="$(git pack-objects --cruft --cruft-expiration="$expire" $packdir/pack <<-EOF
94 $keep
95 -pack-$cruft_a.pack
96 EOF
97 )" &&
99 test-tool pack-mtimes "pack-$cruft_a.mtimes" >expect.raw &&
100 test-tool pack-mtimes "pack-$cruft_b.mtimes" >actual.raw &&
102 sort <expect.raw >expect &&
103 sort <actual.raw >actual &&
105 test_cmp expect actual
109 test_expect_success "multiple cruft packs (expire $expire)" '
110 git init repo &&
111 test_when_finished "rm -fr repo" &&
113 cd repo &&
115 test_commit reachable &&
116 git repack -Ad &&
117 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
119 test_commit cruft &&
120 loose="$objdir/$(test_oid_to_path $(git rev-parse cruft))" &&
122 # generate three copies of the cruft object in different
123 # cruft packs, each with a unique mtime:
124 # - one expired (1000 seconds ago)
125 # - two non-expired (one 1000 seconds in the future,
126 # one 1500 seconds in the future)
127 test-tool chmtime =-1000 "$loose" &&
128 git pack-objects --cruft $packdir/pack-A <<-EOF &&
129 $keep
131 test-tool chmtime =+1000 "$loose" &&
132 git pack-objects --cruft $packdir/pack-B <<-EOF &&
133 $keep
134 -$(basename $(ls $packdir/pack-A-*.pack))
136 test-tool chmtime =+1500 "$loose" &&
137 git pack-objects --cruft $packdir/pack-C <<-EOF &&
138 $keep
139 -$(basename $(ls $packdir/pack-A-*.pack))
140 -$(basename $(ls $packdir/pack-B-*.pack))
143 # ensure the resulting cruft pack takes the most recent
144 # mtime among all copies
145 cruft="$(git pack-objects --cruft \
146 --cruft-expiration="$expire" \
147 $packdir/pack <<-EOF
148 $keep
149 -$(basename $(ls $packdir/pack-A-*.pack))
150 -$(basename $(ls $packdir/pack-B-*.pack))
151 -$(basename $(ls $packdir/pack-C-*.pack))
153 )" &&
155 test-tool pack-mtimes "$(basename $(ls $packdir/pack-C-*.mtimes))" >expect.raw &&
156 test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
158 sort expect.raw >expect &&
159 sort actual.raw >actual &&
160 test_cmp expect actual
164 test_expect_success "cruft packs tolerate missing trees (expire $expire)" '
165 git init repo &&
166 test_when_finished "rm -fr repo" &&
168 cd repo &&
170 test_commit reachable &&
171 test_commit cruft &&
173 tree="$(git rev-parse cruft^{tree})" &&
175 git reset --hard reachable &&
176 git tag -d cruft &&
177 git reflog expire --all --expire=all &&
179 # remove the unreachable tree, but leave the commit
180 # which has it as its root tree intact
181 rm -fr "$objdir/$(test_oid_to_path "$tree")" &&
183 git repack -Ad &&
184 basename $(ls $packdir/pack-*.pack) >in &&
185 git pack-objects --cruft --cruft-expiration="$expire" \
186 $packdir/pack <in
190 test_expect_success "cruft packs tolerate missing blobs (expire $expire)" '
191 git init repo &&
192 test_when_finished "rm -fr repo" &&
194 cd repo &&
196 test_commit reachable &&
197 test_commit cruft &&
199 blob="$(git rev-parse cruft:cruft.t)" &&
201 git reset --hard reachable &&
202 git tag -d cruft &&
203 git reflog expire --all --expire=all &&
205 # remove the unreachable blob, but leave the commit (and
206 # the root tree of that commit) intact
207 rm -fr "$objdir/$(test_oid_to_path "$blob")" &&
209 git repack -Ad &&
210 basename $(ls $packdir/pack-*.pack) >in &&
211 git pack-objects --cruft --cruft-expiration="$expire" \
212 $packdir/pack <in
217 basic_cruft_pack_tests never
218 basic_cruft_pack_tests 2.weeks.ago
220 test_expect_success 'cruft tags rescue tagged objects' '
221 git init repo &&
222 test_when_finished "rm -fr repo" &&
224 cd repo &&
226 test_commit packed &&
227 git repack -Ad &&
229 test_commit tagged &&
230 git tag -a annotated -m tag &&
232 git rev-list --objects --no-object-names packed.. >objects &&
233 while read oid
235 test-tool chmtime -1000 \
236 "$objdir/$(test_oid_to_path $oid)" || exit 1
237 done <objects &&
239 test-tool chmtime -500 \
240 "$objdir/$(test_oid_to_path $(git rev-parse annotated))" &&
242 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
243 cruft="$(echo $keep | git pack-objects --cruft \
244 --cruft-expiration=750.seconds.ago \
245 $packdir/pack)" &&
246 test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
247 cut -f1 -d" " <actual.raw | sort >actual &&
250 cat objects &&
251 git rev-parse annotated
252 ) >expect.raw &&
253 sort <expect.raw >expect &&
255 test_cmp expect actual &&
256 cat actual
260 test_expect_success 'cruft commits rescue parents, trees' '
261 git init repo &&
262 test_when_finished "rm -fr repo" &&
264 cd repo &&
266 test_commit packed &&
267 git repack -Ad &&
269 test_commit old &&
270 test_commit new &&
272 git rev-list --objects --no-object-names packed..new >objects &&
273 while read object
275 test-tool chmtime -1000 \
276 "$objdir/$(test_oid_to_path $object)" || exit 1
277 done <objects &&
278 test-tool chmtime +500 "$objdir/$(test_oid_to_path \
279 $(git rev-parse HEAD))" &&
281 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
282 cruft="$(echo $keep | git pack-objects --cruft \
283 --cruft-expiration=750.seconds.ago \
284 $packdir/pack)" &&
285 test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
287 cut -d" " -f1 <actual.raw | sort >actual &&
288 sort <objects >expect &&
290 test_cmp expect actual
294 test_expect_success 'cruft trees rescue sub-trees, blobs' '
295 git init repo &&
296 test_when_finished "rm -fr repo" &&
298 cd repo &&
300 test_commit packed &&
301 git repack -Ad &&
303 mkdir -p dir/sub &&
304 echo foo >foo &&
305 echo bar >dir/bar &&
306 echo baz >dir/sub/baz &&
308 test_tick &&
309 git add . &&
310 git commit -m "pruned" &&
312 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD))" &&
313 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD^{tree}))" &&
314 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD:foo))" &&
315 test-tool chmtime -500 "$objdir/$(test_oid_to_path $(git rev-parse HEAD:dir))" &&
316 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD:dir/bar))" &&
317 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD:dir/sub))" &&
318 test-tool chmtime -1000 "$objdir/$(test_oid_to_path $(git rev-parse HEAD:dir/sub/baz))" &&
320 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
321 cruft="$(echo $keep | git pack-objects --cruft \
322 --cruft-expiration=750.seconds.ago \
323 $packdir/pack)" &&
324 test-tool pack-mtimes "pack-$cruft.mtimes" >actual.raw &&
325 cut -f1 -d" " <actual.raw | sort >actual &&
327 git rev-parse HEAD:dir HEAD:dir/bar HEAD:dir/sub HEAD:dir/sub/baz >expect.raw &&
328 sort <expect.raw >expect &&
330 test_cmp expect actual
334 test_expect_success 'expired objects are pruned' '
335 git init repo &&
336 test_when_finished "rm -fr repo" &&
338 cd repo &&
340 test_commit packed &&
341 git repack -Ad &&
343 test_commit pruned &&
345 git rev-list --objects --no-object-names packed..pruned >objects &&
346 while read object
348 test-tool chmtime -1000 \
349 "$objdir/$(test_oid_to_path $object)" || exit 1
350 done <objects &&
352 keep="$(basename "$(ls $packdir/pack-*.pack)")" &&
353 cruft="$(echo $keep | git pack-objects --cruft \
354 --cruft-expiration=750.seconds.ago \
355 $packdir/pack)" &&
357 test-tool pack-mtimes "pack-$cruft.mtimes" >actual &&
358 test_must_be_empty actual
362 test_expect_success 'repack --cruft generates a cruft pack' '
363 git init repo &&
364 test_when_finished "rm -fr repo" &&
366 cd repo &&
368 test_commit reachable &&
369 git branch -M main &&
370 git checkout --orphan other &&
371 test_commit unreachable &&
373 git checkout main &&
374 git branch -D other &&
375 git tag -d unreachable &&
376 # objects are not cruft if they are contained in the reflogs
377 git reflog expire --all --expire=all &&
379 git rev-list --objects --all --no-object-names >reachable.raw &&
380 git cat-file --batch-all-objects --batch-check="%(objectname)" >objects &&
381 sort <reachable.raw >reachable &&
382 comm -13 reachable objects >unreachable &&
384 git repack --cruft -d &&
386 cruft=$(basename $(ls $packdir/pack-*.mtimes) .mtimes) &&
387 pack=$(basename $(ls $packdir/pack-*.pack | grep -v $cruft) .pack) &&
389 git show-index <$packdir/$pack.idx >actual.raw &&
390 cut -f2 -d" " actual.raw | sort >actual &&
391 test_cmp reachable actual &&
393 git show-index <$packdir/$cruft.idx >actual.raw &&
394 cut -f2 -d" " actual.raw | sort >actual &&
395 test_cmp unreachable actual
399 test_expect_success 'loose objects mtimes upsert others' '
400 git init repo &&
401 test_when_finished "rm -fr repo" &&
403 cd repo &&
405 test_commit reachable &&
406 git repack -Ad &&
407 git branch -M main &&
409 git checkout --orphan other &&
410 test_commit cruft &&
411 # incremental repack, leaving existing objects loose (so
412 # they can be "freshened")
413 git repack &&
415 tip="$(git rev-parse cruft)" &&
416 path="$objdir/$(test_oid_to_path "$tip")" &&
417 test-tool chmtime --get +1000 "$path" >expect &&
419 git checkout main &&
420 git branch -D other &&
421 git tag -d cruft &&
422 git reflog expire --all --expire=all &&
424 git repack --cruft -d &&
426 mtimes="$(basename $(ls $packdir/pack-*.mtimes))" &&
427 test-tool pack-mtimes "$mtimes" >actual.raw &&
428 grep "$tip" actual.raw | cut -d" " -f2 >actual &&
429 test_cmp expect actual
433 test_expect_success 'expiring cruft objects with git gc' '
434 git init repo &&
435 test_when_finished "rm -fr repo" &&
437 cd repo &&
439 test_commit reachable &&
440 git branch -M main &&
441 git checkout --orphan other &&
442 test_commit unreachable &&
444 git checkout main &&
445 git branch -D other &&
446 git tag -d unreachable &&
447 # objects are not cruft if they are contained in the reflogs
448 git reflog expire --all --expire=all &&
450 git rev-list --objects --all --no-object-names >reachable.raw &&
451 git cat-file --batch-all-objects --batch-check="%(objectname)" >objects &&
452 sort <reachable.raw >reachable &&
453 comm -13 reachable objects >unreachable &&
455 # Write a cruft pack containing all unreachable objects.
456 git gc --cruft --prune="01-01-1980" &&
458 mtimes=$(ls .git/objects/pack/pack-*.mtimes) &&
459 test_path_is_file $mtimes &&
461 # Prune all unreachable objects from the cruft pack.
462 git gc --cruft --prune=now &&
464 git cat-file --batch-all-objects --batch-check="%(objectname)" >objects &&
466 comm -23 unreachable objects >removed &&
467 test_cmp unreachable removed &&
468 test_path_is_missing $mtimes
472 test_expect_success 'cruft packs are not included in geometric repack' '
473 git init repo &&
474 test_when_finished "rm -fr repo" &&
476 cd repo &&
478 test_commit reachable &&
479 git repack -Ad &&
480 git branch -M main &&
482 git checkout --orphan other &&
483 test_commit cruft &&
484 git repack -d &&
486 git checkout main &&
487 git branch -D other &&
488 git tag -d cruft &&
489 git reflog expire --all --expire=all &&
491 git repack --cruft &&
493 find $packdir -type f | sort >before &&
494 git repack --geometric=2 -d &&
495 find $packdir -type f | sort >after &&
497 test_cmp before after
501 test_expect_success 'repack --geometric collects once-cruft objects' '
502 git init repo &&
503 test_when_finished "rm -fr repo" &&
505 cd repo &&
507 test_commit reachable &&
508 git repack -Ad &&
509 git branch -M main &&
511 git checkout --orphan other &&
512 git rm -rf . &&
513 test_commit --no-tag cruft &&
514 cruft="$(git rev-parse HEAD)" &&
516 git checkout main &&
517 git branch -D other &&
518 git reflog expire --all --expire=all &&
520 # Pack the objects created in the previous step into a cruft
521 # pack. Intentionally leave loose copies of those objects
522 # around so we can pick them up in a subsequent --geometric
523 # reapack.
524 git repack --cruft &&
526 # Now make those objects reachable, and ensure that they are
527 # packed into the new pack created via a --geometric repack.
528 git update-ref refs/heads/other $cruft &&
530 # Without this object, the set of unpacked objects is exactly
531 # the set of objects already in the cruft pack. Tweak that set
532 # to ensure we do not overwrite the cruft pack entirely.
533 test_commit reachable2 &&
535 find $packdir -name "pack-*.idx" | sort >before &&
536 git repack --geometric=2 -d &&
537 find $packdir -name "pack-*.idx" | sort >after &&
540 git rev-list --objects --no-object-names $cruft &&
541 git rev-list --objects --no-object-names reachable..reachable2
542 } >want.raw &&
543 sort want.raw >want &&
545 pack=$(comm -13 before after) &&
546 git show-index <$pack >objects.raw &&
548 cut -d" " -f2 objects.raw | sort >got &&
550 test_cmp want got
554 test_expect_success 'cruft repack with no reachable objects' '
555 git init repo &&
556 test_when_finished "rm -fr repo" &&
558 cd repo &&
560 test_commit base &&
561 git repack -ad &&
563 base="$(git rev-parse base)" &&
565 git for-each-ref --format="delete %(refname)" >in &&
566 git update-ref --stdin <in &&
567 git reflog expire --all --expire=all &&
568 rm -fr .git/index &&
570 git repack --cruft -d &&
572 git cat-file -t $base
576 test_expect_success 'cruft repack ignores --max-pack-size' '
577 git init max-pack-size &&
579 cd max-pack-size &&
580 test_commit base &&
581 # two cruft objects which exceed the maximum pack size
582 test-tool genrandom foo 1048576 | git hash-object --stdin -w &&
583 test-tool genrandom bar 1048576 | git hash-object --stdin -w &&
584 git repack --cruft --max-pack-size=1M &&
585 find $packdir -name "*.mtimes" >cruft &&
586 test_line_count = 1 cruft &&
587 test-tool pack-mtimes "$(basename "$(cat cruft)")" >objects &&
588 test_line_count = 2 objects
592 test_expect_success 'cruft repack ignores pack.packSizeLimit' '
594 cd max-pack-size &&
595 # repack everything back together to remove the existing cruft
596 # pack (but to keep its objects)
597 git repack -adk &&
598 git -c pack.packSizeLimit=1M repack --cruft &&
599 # ensure the same post condition is met when --max-pack-size
600 # would otherwise be inferred from the configuration
601 find $packdir -name "*.mtimes" >cruft &&
602 test_line_count = 1 cruft &&
603 test-tool pack-mtimes "$(basename "$(cat cruft)")" >objects &&
604 test_line_count = 2 objects
608 test_expect_success 'cruft repack respects repack.cruftWindow' '
609 git init repo &&
610 test_when_finished "rm -fr repo" &&
612 cd repo &&
614 test_commit base &&
616 GIT_TRACE2_EVENT=$(pwd)/event.trace \
617 git -c pack.window=1 -c repack.cruftWindow=2 repack \
618 --cruft --window=3 &&
620 grep "pack-objects.*--window=2.*--cruft" event.trace
624 test_expect_success 'cruft repack respects --window by default' '
625 git init repo &&
626 test_when_finished "rm -fr repo" &&
628 cd repo &&
630 test_commit base &&
632 GIT_TRACE2_EVENT=$(pwd)/event.trace \
633 git -c pack.window=2 repack --cruft --window=3 &&
635 grep "pack-objects.*--window=3.*--cruft" event.trace
639 test_expect_success 'cruft repack respects --quiet' '
640 git init repo &&
641 test_when_finished "rm -fr repo" &&
643 cd repo &&
645 test_commit base &&
646 GIT_PROGRESS_DELAY=0 git repack --cruft --quiet 2>err &&
647 test_must_be_empty err
651 test_expect_success 'cruft --local drops unreachable objects' '
652 git init alternate &&
653 git init repo &&
654 test_when_finished "rm -fr alternate repo" &&
656 test_commit -C alternate base &&
657 # Pack all objects in alterate so that the cruft repack in "repo" sees
658 # the object it dropped due to `--local` as packed. Otherwise this
659 # object would not appear packed anywhere (since it is not packed in
660 # alternate and likewise not part of the cruft pack in the other repo
661 # because of `--local`).
662 git -C alternate repack -ad &&
665 cd repo &&
667 object="$(git -C ../alternate rev-parse HEAD:base.t)" &&
668 git -C ../alternate cat-file -p $object >contents &&
670 # Write some reachable objects and two unreachable ones: one
671 # that the alternate has and another that is unique.
672 test_commit other &&
673 git hash-object -w -t blob contents &&
674 cruft="$(echo cruft | git hash-object -w -t blob --stdin)" &&
676 ( cd ../alternate/.git/objects && pwd ) \
677 >.git/objects/info/alternates &&
679 test_path_is_file $objdir/$(test_oid_to_path $cruft) &&
680 test_path_is_file $objdir/$(test_oid_to_path $object) &&
682 git repack -d --cruft --local &&
684 test-tool pack-mtimes "$(basename $(ls $packdir/pack-*.mtimes))" \
685 >objects &&
686 ! grep $object objects &&
687 grep $cruft objects
691 test_expect_success 'MIDX bitmaps tolerate reachable cruft objects' '
692 git init repo &&
693 test_when_finished "rm -fr repo" &&
695 cd repo &&
697 test_commit reachable &&
698 test_commit cruft &&
699 unreachable="$(git rev-parse cruft)" &&
701 git reset --hard $unreachable^ &&
702 git tag -d cruft &&
703 git reflog expire --all --expire=all &&
705 git repack --cruft -d &&
707 # resurrect the unreachable object via a new commit. the
708 # new commit will get selected for a bitmap, but be
709 # missing one of its parents from the selected packs.
710 git reset --hard $unreachable &&
711 test_commit resurrect &&
713 git repack --write-midx --write-bitmap-index --geometric=2 -d
717 test_expect_success 'cruft objects are freshend via loose' '
718 git init repo &&
719 test_when_finished "rm -fr repo" &&
721 cd repo &&
723 echo "cruft" >contents &&
724 blob="$(git hash-object -w -t blob contents)" &&
725 loose="$objdir/$(test_oid_to_path $blob)" &&
727 test_commit base &&
729 git repack --cruft -d &&
731 test_path_is_missing "$loose" &&
732 test-tool pack-mtimes "$(basename "$(ls $packdir/pack-*.mtimes)")" >cruft &&
733 grep "$blob" cruft &&
735 # write the same object again
736 git hash-object -w -t blob contents &&
738 test_path_is_file "$loose"
742 test_done