t/t7704-repack-cruft.sh

   1 #!/bin/sh
   2
   3 test_description='git repack works correctly'
   4
   5 TEST_PASSES_SANITIZE_LEAK=true
   6 . ./test-lib.sh
   7
   8 objdir=.git/objects
   9 packdir=$objdir/pack
  10
  11 test_expect_success '--expire-to stores pruned objects (now)' '
  12         git init expire-to-now &&
  13         (
  14                 cd expire-to-now &&
  15
  16                 git branch -M main &&
  17
  18                 test_commit base &&
  19
  20                 git checkout -b cruft &&
  21                 test_commit --no-tag cruft &&
  22
  23                 git rev-list --objects --no-object-names main..cruft >moved.raw &&
  24                 sort moved.raw >moved.want &&
  25
  26                 git rev-list --all --objects --no-object-names >expect.raw &&
  27                 sort expect.raw >expect &&
  28
  29                 git checkout main &&
  30                 git branch -D cruft &&
  31                 git reflog expire --all --expire=all &&
  32
  33                 git init --bare expired.git &&
  34                 git repack -d \
  35                         --cruft --cruft-expiration="now" \
  36                         --expire-to="expired.git/objects/pack/pack" &&
  37
  38                 expired="$(ls expired.git/objects/pack/pack-*.idx)" &&
  39                 test_path_is_file "${expired%.idx}.mtimes" &&
  40
  41                 # Since the `--cruft-expiration` is "now", the effective
  42                 # behavior is to move _all_ unreachable objects out to
  43                 # the location in `--expire-to`.
  44                 git show-index <$expired >expired.raw &&
  45                 cut -d" " -f2 expired.raw | sort >expired.objects &&
  46                 git rev-list --all --objects --no-object-names \
  47                         >remaining.objects &&
  48
  49                 # ...in other words, the combined contents of this
  50                 # repository and expired.git should be the same as the
  51                 # set of objects we started with.
  52                 sort expired.objects remaining.objects >actual &&
  53                 test_cmp expect actual &&
  54
  55                 # The "moved" objects (i.e., those in expired.git)
  56                 # should be the same as the cruft objects which were
  57                 # expired in the previous step.
  58                 test_cmp moved.want expired.objects
  59         )
  60 '
  61
  62 test_expect_success '--expire-to stores pruned objects (5.minutes.ago)' '
  63         git init expire-to-5.minutes.ago &&
  64         (
  65                 cd expire-to-5.minutes.ago &&
  66
  67                 git branch -M main &&
  68
  69                 test_commit base &&
  70
  71                 # Create two classes of unreachable objects, one which
  72                 # is older than 5 minutes (stale), and another which is
  73                 # newer (recent).
  74                 for kind in stale recent
  75                 do
  76                         git checkout -b $kind main &&
  77                         test_commit --no-tag $kind || return 1
  78                 done &&
  79
  80                 git rev-list --objects --no-object-names main..stale >in &&
  81                 stale="$(git pack-objects $objdir/pack/pack <in)" &&
  82                 mtime="$(test-tool chmtime --get =-600 $objdir/pack/pack-$stale.pack)" &&
  83
  84                 # expect holds the set of objects we expect to find in
  85                 # this repository after repacking
  86                 git rev-list --objects --no-object-names recent >expect.raw &&
  87                 sort expect.raw >expect &&
  88
  89                 # moved.want holds the set of objects we expect to find
  90                 # in expired.git
  91                 git rev-list --objects --no-object-names main..stale >out &&
  92                 sort out >moved.want &&
  93
  94                 git checkout main &&
  95                 git branch -D stale recent &&
  96                 git reflog expire --all --expire=all &&
  97                 git prune-packed &&
  98
  99                 git init --bare expired.git &&
 100                 git repack -d \
 101                         --cruft --cruft-expiration=5.minutes.ago \
 102                         --expire-to="expired.git/objects/pack/pack" &&
 103
 104                 # Some of the remaining objects in this repository are
 105                 # unreachable, so use `cat-file --batch-all-objects`
 106                 # instead of `rev-list` to get their names
 107                 git cat-file --batch-all-objects --batch-check="%(objectname)" \
 108                         >remaining.objects &&
 109                 sort remaining.objects >actual &&
 110                 test_cmp expect actual &&
 111
 112                 (
 113                         cd expired.git &&
 114
 115                         expired="$(ls objects/pack/pack-*.mtimes)" &&
 116                         test-tool pack-mtimes $(basename $expired) >out &&
 117                         cut -d" " -f1 out | sort >../moved.got &&
 118
 119                         # Ensure that there are as many objects with the
 120                         # expected mtime as were moved to expired.git.
 121                         #
 122                         # In other words, ensure that the recorded
 123                         # mtimes of any moved objects was written
 124                         # correctly.
 125                         grep " $mtime$" out >matching &&
 126                         test_line_count = $(wc -l <../moved.want) matching
 127                 ) &&
 128                 test_cmp moved.want moved.got
 129         )
 130 '
 131
 132 generate_random_blob() {
 133         test-tool genrandom "$@" >blob &&
 134         git hash-object -w -t blob blob &&
 135         rm blob
 136 }
 137
 138 pack_random_blob () {
 139         generate_random_blob "$@" &&
 140         git repack -d -q >/dev/null
 141 }
 142
 143 generate_cruft_pack () {
 144         pack_random_blob "$@" >/dev/null &&
 145
 146         ls $packdir/pack-*.pack | xargs -n 1 basename >in &&
 147         pack="$(git pack-objects --cruft $packdir/pack <in)" &&
 148         git prune-packed &&
 149
 150         echo "$packdir/pack-$pack.mtimes"
 151 }
 152
 153 test_expect_success '--max-cruft-size creates new packs when above threshold' '
 154         git init max-cruft-size-large &&
 155         (
 156                 cd max-cruft-size-large &&
 157                 test_commit base &&
 158
 159                 foo="$(pack_random_blob foo $((1*1024*1024)))" &&
 160                 git repack --cruft -d &&
 161                 cruft_foo="$(ls $packdir/pack-*.mtimes)" &&
 162
 163                 bar="$(pack_random_blob bar $((1*1024*1024)))" &&
 164                 git repack --cruft -d --max-cruft-size=1M &&
 165                 cruft_bar="$(ls $packdir/pack-*.mtimes | grep -v $cruft_foo)" &&
 166
 167                 test-tool pack-mtimes $(basename "$cruft_foo") >foo.objects &&
 168                 test-tool pack-mtimes $(basename "$cruft_bar") >bar.objects &&
 169
 170                 grep "^$foo" foo.objects &&
 171                 test_line_count = 1 foo.objects &&
 172                 grep "^$bar" bar.objects &&
 173                 test_line_count = 1 bar.objects
 174         )
 175 '
 176
 177 test_expect_success '--max-cruft-size combines existing packs when below threshold' '
 178         git init max-cruft-size-small &&
 179         (
 180                 cd max-cruft-size-small &&
 181                 test_commit base &&
 182
 183                 foo="$(pack_random_blob foo $((1*1024*1024)))" &&
 184                 git repack --cruft -d &&
 185
 186                 bar="$(pack_random_blob bar $((1*1024*1024)))" &&
 187                 git repack --cruft -d --max-cruft-size=10M &&
 188
 189                 cruft=$(ls $packdir/pack-*.mtimes) &&
 190                 test-tool pack-mtimes $(basename "$cruft") >cruft.objects &&
 191
 192                 grep "^$foo" cruft.objects &&
 193                 grep "^$bar" cruft.objects &&
 194                 test_line_count = 2 cruft.objects
 195         )
 196 '
 197
 198 test_expect_success '--max-cruft-size combines smaller packs first' '
 199         git init max-cruft-size-consume-small &&
 200         (
 201                 cd max-cruft-size-consume-small &&
 202
 203                 test_commit base &&
 204                 git repack -ad &&
 205
 206                 cruft_foo="$(generate_cruft_pack foo 524288)" &&    # 0.5 MiB
 207                 cruft_bar="$(generate_cruft_pack bar 524288)" &&    # 0.5 MiB
 208                 cruft_baz="$(generate_cruft_pack baz 1048576)" &&   # 1.0 MiB
 209                 cruft_quux="$(generate_cruft_pack quux 1572864)" && # 1.5 MiB
 210
 211                 test-tool pack-mtimes "$(basename $cruft_foo)" >expect.raw &&
 212                 test-tool pack-mtimes "$(basename $cruft_bar)" >>expect.raw &&
 213                 sort expect.raw >expect.objects &&
 214
 215                 # repacking with `--max-cruft-size=2M` should combine
 216                 # both 0.5 MiB packs together, instead of, say, one of
 217                 # the 0.5 MiB packs with the 1.0 MiB pack
 218                 ls $packdir/pack-*.mtimes | sort >cruft.before &&
 219                 git repack -d --cruft --max-cruft-size=2M &&
 220                 ls $packdir/pack-*.mtimes | sort >cruft.after &&
 221
 222                 comm -13 cruft.before cruft.after >cruft.new &&
 223                 comm -23 cruft.before cruft.after >cruft.removed &&
 224
 225                 test_line_count = 1 cruft.new &&
 226                 test_line_count = 2 cruft.removed &&
 227
 228                 # the two smaller packs should be rolled up first
 229                 printf "%s\n" $cruft_foo $cruft_bar | sort >expect.removed &&
 230                 test_cmp expect.removed cruft.removed &&
 231
 232                 # ...and contain the set of objects rolled up
 233                 test-tool pack-mtimes "$(basename $(cat cruft.new))" >actual.raw &&
 234                 sort actual.raw >actual.objects &&
 235
 236                 test_cmp expect.objects actual.objects
 237         )
 238 '
 239
 240 test_expect_success 'setup --max-cruft-size with freshened objects' '
 241         git init max-cruft-size-freshen &&
 242         (
 243                 cd max-cruft-size-freshen &&
 244
 245                 test_commit base &&
 246                 git repack -ad &&
 247
 248                 foo="$(generate_random_blob foo 64)" &&
 249                 test-tool chmtime --get -10000 \
 250                         "$objdir/$(test_oid_to_path "$foo")" >foo.mtime &&
 251
 252                 git repack --cruft -d &&
 253
 254                 cruft="$(ls $packdir/pack-*.mtimes)" &&
 255                 test-tool pack-mtimes "$(basename $cruft)" >actual &&
 256                 echo "$foo $(cat foo.mtime)" >expect &&
 257                 test_cmp expect actual
 258         )
 259 '
 260
 261 test_expect_success '--max-cruft-size with freshened objects (loose)' '
 262         (
 263                 cd max-cruft-size-freshen &&
 264
 265                 # regenerate the object, setting its mtime to be more recent
 266                 foo="$(generate_random_blob foo 64)" &&
 267                 test-tool chmtime --get -100 \
 268                         "$objdir/$(test_oid_to_path "$foo")" >foo.mtime &&
 269
 270                 git repack --cruft -d &&
 271
 272                 cruft="$(ls $packdir/pack-*.mtimes)" &&
 273                 test-tool pack-mtimes "$(basename $cruft)" >actual &&
 274                 echo "$foo $(cat foo.mtime)" >expect &&
 275                 test_cmp expect actual
 276         )
 277 '
 278
 279 test_expect_success '--max-cruft-size with freshened objects (packed)' '
 280         (
 281                 cd max-cruft-size-freshen &&
 282
 283                 # regenerate the object and store it in a packfile,
 284                 # setting its mtime to be more recent
 285                 #
 286                 # store it alongside another cruft object so that we
 287                 # do not create an identical copy of the existing
 288                 # cruft pack (which contains $foo).
 289                 foo="$(generate_random_blob foo 64)" &&
 290                 bar="$(generate_random_blob bar 64)" &&
 291                 foo_pack="$(printf "%s\n" $foo $bar | git pack-objects $packdir/pack)" &&
 292                 git prune-packed &&
 293
 294                 test-tool chmtime --get -10 \
 295                         "$packdir/pack-$foo_pack.pack" >foo.mtime &&
 296
 297                 git repack --cruft -d &&
 298
 299                 cruft="$(ls $packdir/pack-*.mtimes)" &&
 300                 test-tool pack-mtimes "$(basename $cruft)" >actual &&
 301                 echo "$foo $(cat foo.mtime)" >expect.raw &&
 302                 echo "$bar $(cat foo.mtime)" >>expect.raw &&
 303                 sort expect.raw >expect &&
 304                 test_cmp expect actual
 305         )
 306 '
 307
 308 test_expect_success '--max-cruft-size with pruning' '
 309         git init max-cruft-size-prune &&
 310         (
 311                 cd max-cruft-size-prune &&
 312
 313                 test_commit base &&
 314                 foo="$(generate_random_blob foo $((1024*1024)))" &&
 315                 bar="$(generate_random_blob bar $((1024*1024)))" &&
 316                 baz="$(generate_random_blob baz $((1024*1024)))" &&
 317
 318                 test-tool chmtime -10000 "$objdir/$(test_oid_to_path "$foo")" &&
 319
 320                 git repack -d --cruft --max-cruft-size=1M &&
 321
 322                 # backdate the mtimes of all cruft packs to validate
 323                 # that they were rewritten as a result of pruning
 324                 ls $packdir/pack-*.mtimes | sort >cruft.before &&
 325                 for cruft in $(cat cruft.before)
 326                 do
 327                         mtime="$(test-tool chmtime --get -10000 "$cruft")" &&
 328                         echo $cruft $mtime >>mtimes || return 1
 329                 done &&
 330
 331                 # repack (and prune) with a --max-cruft-size to ensure
 332                 # that we appropriately split the resulting set of packs
 333                 git repack -d --cruft --max-cruft-size=1M \
 334                         --cruft-expiration=1000.seconds.ago &&
 335                 ls $packdir/pack-*.mtimes | sort >cruft.after &&
 336
 337                 for cruft in $(cat cruft.after)
 338                 do
 339                         old_mtime="$(grep $cruft mtimes | cut -d" " -f2)" &&
 340                         new_mtime="$(test-tool chmtime --get $cruft)" &&
 341                         test $old_mtime -lt $new_mtime || return 1
 342                 done &&
 343
 344                 test_line_count = 3 cruft.before &&
 345                 test_line_count = 2 cruft.after &&
 346                 test_must_fail git cat-file -e $foo &&
 347                 git cat-file -e $bar &&
 348                 git cat-file -e $baz
 349         )
 350 '
 351
 352 test_expect_success '--max-cruft-size ignores non-local packs' '
 353         repo="max-cruft-size-non-local" &&
 354         git init $repo &&
 355         (
 356                 cd $repo &&
 357                 test_commit base &&
 358                 generate_random_blob foo 64 &&
 359                 git repack --cruft -d
 360         ) &&
 361
 362         git clone --reference=$repo $repo $repo-alt &&
 363         (
 364                 cd $repo-alt &&
 365
 366                 test_commit other &&
 367                 generate_random_blob bar 64 &&
 368
 369                 # ensure that we do not attempt to pick up packs from
 370                 # the non-alternated repository, which would result in a
 371                 # crash
 372                 git repack --cruft --max-cruft-size=1M -d
 373         )
 374 '
 375
 376 test_expect_success 'reachable packs are preferred over cruft ones' '
 377         repo="cruft-preferred-packs" &&
 378         git init "$repo" &&
 379         (
 380                 cd "$repo" &&
 381
 382                 # This test needs to exercise careful control over when a MIDX
 383                 # is and is not written. Unset the corresponding TEST variable
 384                 # accordingly.
 385                 sane_unset GIT_TEST_MULTI_PACK_INDEX &&
 386
 387                 test_commit base &&
 388                 test_commit --no-tag cruft &&
 389
 390                 non_cruft="$(echo base | git pack-objects --revs $packdir/pack)" &&
 391                 # Write a cruft pack which both (a) sorts ahead of the non-cruft
 392                 # pack in lexical order, and (b) has an older mtime to appease
 393                 # the MIDX preferred pack selection routine.
 394                 cruft="$(echo pack-$non_cruft.pack | git pack-objects --cruft $packdir/pack-A)" &&
 395                 test-tool chmtime -1000 $packdir/pack-A-$cruft.pack &&
 396
 397                 test_commit other &&
 398                 git repack -d &&
 399
 400                 git repack --geometric 2 -d --write-midx --write-bitmap-index &&
 401
 402                 # After repacking, there are two packs left: one reachable one
 403                 # (which is the result of combining both of the existing two
 404                 # non-cruft packs), and one cruft pack.
 405                 find .git/objects/pack -type f -name "*.pack" >packs &&
 406                 test_line_count = 2 packs &&
 407
 408                 # Make sure that the pack we just wrote is marked as preferred,
 409                 # not the cruft one.
 410                 pack="$(test-tool read-midx --preferred-pack $objdir)" &&
 411                 test_path_is_missing "$packdir/$(basename "$pack" ".idx").mtimes"
 412         )
 413 '
 414
 415 test_done